This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 97bdd5b62128 feat(spark): add hudi-spark-native-bundle packaging
Apache DataFusion Comet (#19674)
97bdd5b62128 is described below
commit 97bdd5b6212853c37b421822a7739b1bbdab625f
Author: Y Ethan Guo <[email protected]>
AuthorDate: Thu Oct 22 22:17:54 2026 -0700
feat(spark): add hudi-spark-native-bundle packaging Apache DataFusion Comet
(#19674)
---
.github/workflows/java_ci_engines.yml | 43 ++--
LICENSE | 59 +++++
NOTICE | 11 +
README.md | 28 ++-
azure-pipelines-20230430.yml | 3 +-
packaging/bundle-validation/Dockerfile | 12 +-
packaging/bundle-validation/README.md | 10 +
.../base/build_flink1200hive313spark359.sh | 27 +++
packaging/bundle-validation/ci_run.sh | 28 +++
.../bundle-validation/spark_native/validate.scala | 86 ++++++++
.../utilities/hoodieapp.properties | 2 +
packaging/bundle-validation/validate.sh | 120 +++++++++-
packaging/hudi-spark-native-bundle/README.md | 123 +++++++++++
packaging/hudi-spark-native-bundle/pom.xml | 242 +++++++++++++++++++++
.../org/apache/hudi/spark/nativebundle/Main.java | 38 ++++
pom.xml | 2 +
scripts/release/deploy_staging_jars.sh | 9 +-
scripts/release/deploy_staging_jars_java17.sh | 7 +-
scripts/release/validate_staged_bundles.sh | 4 +-
19 files changed, 815 insertions(+), 39 deletions(-)
diff --git a/.github/workflows/java_ci_engines.yml
b/.github/workflows/java_ci_engines.yml
index 1502df5b38d5..ce71426c8f75 100644
--- a/.github/workflows/java_ci_engines.yml
+++ b/.github/workflows/java_ci_engines.yml
@@ -231,6 +231,20 @@ jobs:
strategy:
matrix:
include:
+ # validate.sh only runs the Flink test on a Flink 2.x image
(HUDI-8608), so the Spark,
+ # utilities, CLI and native bundles are validated on the Flink 1.x
entries.
+ - scalaProfile: 'scala-2.13'
+ flinkProfile: 'flink1.20'
+ flinkAvroVersion: '1.11.4'
+ flinkParquetVersion: '1.13.1'
+ sparkProfile: 'spark3.5'
+ sparkRuntime: 'spark3.5.1'
+ - scalaProfile: 'scala-2.12'
+ flinkProfile: 'flink1.20'
+ flinkAvroVersion: '1.11.4'
+ flinkParquetVersion: '1.13.1'
+ sparkProfile: 'spark3.5'
+ sparkRuntime: 'spark3.5.9'
- scalaProfile: 'scala-2.12'
flinkProfile: 'flink2.2'
flinkAvroVersion: '1.11.4'
@@ -275,7 +289,7 @@ jobs:
FLINK_PARQUET_VERSION: ${{ matrix.flinkParquetVersion }}
run: |
if [ "$SCALA_PROFILE" == "scala-2.13" ]; then
- mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-DdeployArtifacts=true -DskipTests=true $MVN_ARGS -pl
packaging/hudi-hadoop-mr-bundle,packaging/hudi-spark-bundle,packaging/hudi-utilities-bundle,packaging/hudi-utilities-slim-bundle,packaging/hudi-cli-bundle
-am
+ mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-DdeployArtifacts=true -DskipTests=true $MVN_ARGS -pl
packaging/hudi-hadoop-mr-bundle,packaging/hudi-spark-bundle,packaging/hudi-spark-native-bundle,packaging/hudi-utilities-bundle,packaging/hudi-utilities-slim-bundle,packaging/hudi-cli-bundle
-am
else
mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-D"$FLINK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS
# TODO remove the sudo below. It's a needed workaround as detailed
in HUDI-5708.
@@ -308,19 +322,18 @@ jobs:
strategy:
matrix:
include:
- # [CI-TRIM] to revisit for CI improvement
-# - scalaProfile: 'scala-2.13'
-# flinkProfile: 'flink1.20'
-# flinkAvroVersion: '1.11.4'
-# flinkParquetVersion: '1.13.1'
-# sparkProfile: 'spark4.0'
-# sparkRuntime: 'spark4.0.0'
-# - scalaProfile: 'scala-2.13'
-# flinkProfile: 'flink1.20'
-# flinkAvroVersion: '1.11.4'
-# flinkParquetVersion: '1.13.1'
-# sparkProfile: 'spark4.1'
-# sparkRuntime: 'spark4.1.1'
+ - scalaProfile: 'scala-2.13'
+ flinkProfile: 'flink1.20'
+ flinkAvroVersion: '1.11.4'
+ flinkParquetVersion: '1.13.1'
+ sparkProfile: 'spark4.0'
+ sparkRuntime: 'spark4.0.0'
+ - scalaProfile: 'scala-2.13'
+ flinkProfile: 'flink1.20'
+ flinkAvroVersion: '1.11.4'
+ flinkParquetVersion: '1.13.1'
+ sparkProfile: 'spark4.1'
+ sparkRuntime: 'spark4.1.1'
- scalaProfile: 'scala-2.13'
flinkProfile: 'flink1.20'
flinkAvroVersion: '1.11.4'
@@ -343,7 +356,7 @@ jobs:
FLINK_AVRO_VERSION: ${{ matrix.flinkAvroVersion }}
FLINK_PARQUET_VERSION: ${{ matrix.flinkParquetVersion }}
run: |
- mvn clean package -T 2 -Pjava17 -D"$SCALA_PROFILE"
-D"$SPARK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -pl
packaging/hudi-hadoop-mr-bundle,packaging/hudi-spark-bundle,packaging/hudi-utilities-bundle,packaging/hudi-utilities-slim-bundle,packaging/hudi-cli-bundle
-am
+ mvn clean package -T 2 -Pjava17 -D"$SCALA_PROFILE"
-D"$SPARK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -pl
packaging/hudi-hadoop-mr-bundle,packaging/hudi-spark-bundle,packaging/hudi-spark-native-bundle,packaging/hudi-utilities-bundle,packaging/hudi-utilities-slim-bundle,packaging/hudi-cli-bundle
-am
- name: Validate Bundle UI Assets
run: ./scripts/release/validate_bundle_ui_assets.sh
- name: IT - Spark4 Bundle Validation - OpenJDK 17
diff --git a/LICENSE b/LICENSE
index e31610a27293..0a29333d6efe 100644
--- a/LICENSE
+++ b/LICENSE
@@ -480,3 +480,62 @@ SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY
DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+
-------------------------------------------------------------------------------
+
+ This product includes software from Apache DataFusion Comet
+
+ * The hudi-spark-native-bundle artifacts bundle
+ org.apache.datafusion:comet-spark-spark<spark version>_<scala binary
version>, including these
+ pre-built native libraries:
+
+ org/apache/comet/linux/aarch64/libcomet.so
+ org/apache/comet/linux/amd64/libcomet.so
+ arrow_cdata_jni/aarch_64/libarrow_cdata_jni.so
+ arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib
+ arrow_cdata_jni/x86_64/libarrow_cdata_jni.so
+ arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib
+ arrow_cdata_jni/x86_64/arrow_cdata_jni.dll
+
+ The arrow_cdata_jni libraries are from Apache Arrow
(org.apache.arrow:arrow-c-data), under the
+ Apache License 2.0.
+
+ Copyright: 2020 and onwards The Apache Software Foundation
+ Home page: https://datafusion.apache.org/comet/
+ License: http://www.apache.org/licenses/LICENSE-2.0
+
+ The bundled comet-spark artifact redistributes Protocol Buffers, relocated to
+ org.apache.comet.shaded.protobuf, under the following license
+
+ Copyright 2008 Google Inc. All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+ * Neither the name of Google Inc. nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ Code generated by the Protocol Buffer compiler is owned by the owner
+ of the input file used when generating it. This code is not
+ standalone and requires a support library to be linked with it. This
+ support library is itself covered by the above license.
diff --git a/NOTICE b/NOTICE
index 623cbf3d8319..94a4d7ca400e 100644
--- a/NOTICE
+++ b/NOTICE
@@ -244,3 +244,14 @@ its NOTICE file:
| Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
| See the License for the specific language governing permissions and
limitations under the License.
+
+--------------------------------------------------------------------------------
+
+This product includes code from Apache DataFusion Comet, which includes the
following in
+its NOTICE file:
+
+ comet-spark
+ Copyright 2020 The Apache Software Foundation
+
+ This product includes software developed at
+ The Apache Software Foundation (http://www.apache.org/).
diff --git a/README.md b/README.md
index 1a29a466cc4e..ea7c7cc0300c 100644
--- a/README.md
+++ b/README.md
@@ -128,17 +128,23 @@ The default Spark 3.x version, corresponding to `spark3`
profile is
Refer to the table below for building with different Spark and Scala versions.
-| Maven build options | Expected Spark bundle jar name |
Notes |
-|:--------------------------|:---------------------------------------------|:-------------------------------------------------|
-| (empty) | hudi-spark3.5-bundle_2.12 |
For Spark 3.5.x and Scala 2.12 (default options) |
-| `-Dspark3.3` | hudi-spark3.3-bundle_2.12 |
For Spark 3.3.2+ and Scala 2.12 |
-| `-Dspark3.4` | hudi-spark3.4-bundle_2.12 |
For Spark 3.4.x and Scala 2.12 |
-| `-Dspark3.5 -Dscala-2.12` | hudi-spark3.5-bundle_2.12 |
For Spark 3.5.x and Scala 2.12 (same as default) |
-| `-Dspark3.5 -Dscala-2.13` | hudi-spark3.5-bundle_2.13 |
For Spark 3.5.x and Scala 2.13 |
-| `-Dspark4.0` | hudi-spark4.0-bundle_2.13 |
For Spark 4.0 and Scala 2.13 (Needs java 17) |
-| `-Dspark4.1` | hudi-spark4.1-bundle_2.13 |
For Spark 4.1 and Scala 2.13 (Needs java 17) |
-| `-Dspark4.2` | hudi-spark4.2-bundle_2.13 |
For Spark 4.2 and Scala 2.13 (Needs java 17) |
-| `-Dspark3` | hudi-spark3-bundle_2.12 (legacy bundle name) |
For Spark 3.5.x and Scala 2.12 |
+| Maven build options | Expected Spark bundle jar name |
Expected Spark native bundle jar name | Notes
|
+|:--------------------------|:---------------------------------------------|:--------------------------------------|:-------------------------------------------------|
+| (empty) | hudi-spark3.5-bundle_2.12 |
hudi-spark3.5-native-bundle_2.12 | For Spark 3.5.x and Scala 2.12 (default
options) |
+| `-Dspark3.3` | hudi-spark3.3-bundle_2.12 |
none | For Spark 3.3.2+ and Scala 2.12
|
+| `-Dspark3.4` | hudi-spark3.4-bundle_2.12 |
hudi-spark3.4-native-bundle_2.12 | For Spark 3.4.x and Scala 2.12
|
+| `-Dspark3.5 -Dscala-2.12` | hudi-spark3.5-bundle_2.12 |
hudi-spark3.5-native-bundle_2.12 | For Spark 3.5.x and Scala 2.12 (same as
default) |
+| `-Dspark3.5 -Dscala-2.13` | hudi-spark3.5-bundle_2.13 |
hudi-spark3.5-native-bundle_2.13 | For Spark 3.5.x and Scala 2.13
|
+| `-Dspark4.0` | hudi-spark4.0-bundle_2.13 |
hudi-spark4.0-native-bundle_2.13 | For Spark 4.0 and Scala 2.13 (Needs
java 17) |
+| `-Dspark4.1` | hudi-spark4.1-bundle_2.13 |
hudi-spark4.1-native-bundle_2.13 | For Spark 4.1 and Scala 2.13 (Needs
java 17) |
+| `-Dspark4.2` | hudi-spark4.2-bundle_2.13 |
none | For Spark 4.2 and Scala 2.13 (Needs
java 17) |
+| `-Dspark3` | hudi-spark3-bundle_2.12 (legacy bundle name) |
none | For Spark 3.5.x and Scala 2.12
|
+
+`hudi-spark-native-bundle` is `hudi-spark-bundle` plus [Apache DataFusion
Comet](https://datafusion.apache.org/comet/)
+for native vectorized execution, built by the same Maven options. Comet
publishes one artifact per
+Spark minor version, so the native bundle exists only where that artifact
does, and `-Dspark3` is a
+legacy alias that selects none. It builds on Java 11 but needs Java 17 and
Linux with glibc at runtime.
+See `packaging/hudi-spark-native-bundle/README.md`.
Hudi uses ZSTD as the default Parquet compression codec with Spark 3.5 and
newer. Spark 3.3 and 3.4
retain GZIP because Hudi's non-vectorized file-group reader uses parquet-java
1.12.x and can leak
diff --git a/azure-pipelines-20230430.yml b/azure-pipelines-20230430.yml
index 0e3667a238d7..aefb91816765 100644
--- a/azure-pipelines-20230430.yml
+++ b/azure-pipelines-20230430.yml
@@ -96,6 +96,7 @@ parameters:
- '!packaging/hudi-metaserver-server-bundle'
- '!packaging/hudi-presto-bundle'
- '!packaging/hudi-spark-bundle'
+ - '!packaging/hudi-spark-native-bundle'
- '!packaging/hudi-timeline-server-bundle'
- '!packaging/hudi-utilities-slim-bundle'
@@ -531,7 +532,7 @@ stages:
-v $(Build.SourcesDirectory):/hudi
-v /var/run/docker.sock:/var/run/docker.sock
-i
docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId)
- /bin/bash -c "MAVEN_OPTS='-Xmx8g' mvn clean install -T 1
$(MVN_OPTS_INSTALL) -Phudi-platform-service -Pthrift-gen-source
+ /bin/bash -c "MAVEN_OPTS='-Xmx8g' mvn clean install -T 1
$(MVN_OPTS_INSTALL) -Phudi-platform-service -Pthrift-gen-source
-Dhudi.spark.native.bundle.skip=true
&& mvn test $(MVN_OPTS_TEST) -Punit-tests
-Dsurefire.failIfNoSpecifiedTests=false $(JACOCO_AGENT_DESTFILE1_ARG) -pl
$(JOB10_UT_MODULES)
&& mvn test $(MVN_OPTS_TEST) -Punit-tests
$(JACOCO_AGENT_DESTFILE2_ARG) -Dtest="!TestHoodie*"
-Dsurefire.failIfNoSpecifiedTests=false -pl hudi-utilities
&& mvn test $(MVN_OPTS_TEST) -Pfunctional-tests
-Dsurefire.failIfNoSpecifiedTests=false $(JACOCO_AGENT_DESTFILE3_ARG) -pl
$(JOB10_FT_MODULES)"
diff --git a/packaging/bundle-validation/Dockerfile
b/packaging/bundle-validation/Dockerfile
index e986a5b918e3..3b47b6a1881e 100644
--- a/packaging/bundle-validation/Dockerfile
+++ b/packaging/bundle-validation/Dockerfile
@@ -30,11 +30,15 @@ RUN cp $DERBY_HOME/lib/derbyclient.jar $SPARK_HOME/jars/
RUN cp conf/spark-defaults.conf $SPARK_HOME/conf/
RUN if [[ $SPARK_HOME =~ 'spark-3.2' ]] || [[ $SPARK_HOME =~ 'spark-3.3' ]]; \
then printf "\nspark.sql.catalog.spark_catalog
org.apache.spark.sql.hudi.catalog.HoodieCatalog\n" >>
$SPARK_HOME/conf/spark-defaults.conf; fi
-RUN printf "\ntaskmanager.numberOfTaskSlots: 2\n" >>
$FLINK_HOME/conf/flink-conf.yaml
+# Flink 1.19+ ships config.yaml and gives a legacy flink-conf.yaml precedence,
so adding one that
+# holds only the slot count would start the cluster without any memory
settings.
+RUN if [ -f $FLINK_HOME/conf/config.yaml ]; \
+ then sed -i 's/numberOfTaskSlots: 1/numberOfTaskSlots: 2/'
$FLINK_HOME/conf/config.yaml; \
+ else printf "\ntaskmanager.numberOfTaskSlots: 2\n" >>
$FLINK_HOME/conf/flink-conf.yaml; fi
RUN printf "\nlocalhost\n" >> $FLINK_HOME/conf/workers
# install maven
-RUN wget -t=3
https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.12/apache-maven-3.9.12-bin.tar.gz
+RUN wget -t 3
https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.12/apache-maven-3.9.12-bin.tar.gz
RUN mkdir -p /usr/share/maven
RUN tar xzvf apache-maven-3.9.12-bin.tar.gz -C /usr/share/maven
ENV MAVEN_HOME=/usr/share/maven/apache-maven-3.9.12
@@ -54,5 +58,5 @@ ENV HDFS_SECONDARYNAMENODE_USER root
ENV YARN_RESOURCEMANAGER_USER root
ENV YARN_NODEMANAGER_USER root
-# for RocksDb
-RUN apk add --no-cache libstdc++
+# for RocksDb on base images that install packages with apk; the glibc base
image already carries libstdc++
+RUN if command -v apk > /dev/null; then apk add --no-cache libstdc++; fi
diff --git a/packaging/bundle-validation/README.md
b/packaging/bundle-validation/README.md
index 6fac82d14061..141aa065a556 100644
--- a/packaging/bundle-validation/README.md
+++ b/packaging/bundle-validation/README.md
@@ -109,3 +109,13 @@ flag and adding the release version:
HUDI_VERSION: 1.0.0 ## Change to the release version
MAVEN_BASE_URL: 'https://repo1.maven.org/maven2' ## Maven Central URL,
no need to change unless necessary
```
+
+## Spark native bundle
+
+`hudi-spark-native-bundle` is validated on the Java 17 pass in two steps. The
Comet query in
+`spark_native/` asserts Comet operators in the plan, not just row counts,
because Comet silently falls
+back to Spark for anything it cannot accelerate. It needs a glibc JVM for
`libcomet.so`, and the Alpine
+based image's Java 17 is musl, so `validate.sh` skips this step there with a
warning; run it on a glibc
+image with the same `SPARK_HOME` and `/usr/lib/jvm/java-17-openjdk` layout.
The second step runs the
+utilities slim bundle paired with the native bundle instead of the Spark
bundle, with no Comet config,
+so it runs on the Alpine image too.
diff --git a/packaging/bundle-validation/base/build_flink1200hive313spark359.sh
b/packaging/bundle-validation/base/build_flink1200hive313spark359.sh
new file mode 100755
index 000000000000..5d67f39559a8
--- /dev/null
+++ b/packaging/bundle-validation/base/build_flink1200hive313spark359.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+docker build \
+ --build-arg HIVE_VERSION=3.1.3 \
+ --build-arg FLINK_VERSION=1.20.1 \
+ --build-arg SPARK_VERSION=3.5.9 \
+ --build-arg SPARK_HADOOP_VERSION=3 \
+ --build-arg HADOOP_VERSION=3.3.5 \
+ -t hudi-ci-bundle-validation-base:flink1200hive313spark359 .
+docker image tag hudi-ci-bundle-validation-base:flink1200hive313spark359
apachehudi/hudi-ci-bundle-validation-base:flink1200hive313spark359
diff --git a/packaging/bundle-validation/ci_run.sh
b/packaging/bundle-validation/ci_run.sh
index 922ba0af2030..757219d4361f 100755
--- a/packaging/bundle-validation/ci_run.sh
+++ b/packaging/bundle-validation/ci_run.sh
@@ -94,6 +94,17 @@ elif [[ ${SPARK_RUNTIME} == 'spark3.5.0' && ${SCALA_PROFILE}
== 'scala-2.13' ]];
CONFLUENT_VERSION=5.5.12
KAFKA_CONNECT_HDFS_VERSION=10.1.13
IMAGE_TAG=flink1200hive313spark350scala213
+elif [[ ${SPARK_RUNTIME} == 'spark3.5.9' && ${SCALA_PROFILE} == 'scala-2.12'
]]; then
+ # Apache DataFusion Comet 1.1.0 is built against Spark 3.5.9 and needs at
least 3.5.2 at runtime.
+ HADOOP_VERSION=3.3.5
+ HIVE_VERSION=3.1.3
+ DERBY_VERSION=10.14.1.0
+ FLINK_VERSION=1.20.1
+ SPARK_VERSION=3.5.9
+ SPARK_HADOOP_VERSION=3
+ CONFLUENT_VERSION=5.5.12
+ KAFKA_CONNECT_HDFS_VERSION=10.1.13
+ IMAGE_TAG=flink1200hive313spark359
elif [[ ${SPARK_RUNTIME} == 'spark3.5.1' && ${SCALA_PROFILE} == 'scala-2.12'
]]; then
HADOOP_VERSION=3.3.5
HIVE_VERSION=3.1.3
@@ -177,6 +188,13 @@ if [ -z "$STAGING_REPO_NUM" ] && [ -z "$MAVEN_BASE_URL" ];
then
cp
${GITHUB_WORKSPACE}/packaging/hudi-cli-bundle/target/hudi-*-$HUDI_VERSION.jar
$TMP_JARS_DIR/
cp
${GITHUB_WORKSPACE}/packaging/hudi-hadoop-mr-bundle/target/hudi-*-$HUDI_VERSION.jar
$TMP_JARS_DIR/
cp
${GITHUB_WORKSPACE}/packaging/hudi-spark-bundle/target/hudi-*-$HUDI_VERSION.jar
$TMP_JARS_DIR/
+ # Only built for the Spark versions Apache DataFusion Comet releases for.
Stage it only if
+ # it really carries Comet, so a build where the bundle was skipped is never
validated.
+ for nativeBundle in
${GITHUB_WORKSPACE}/packaging/hudi-spark-native-bundle/target/hudi-spark*-native-bundle_*-$HUDI_VERSION.jar;
do
+ if [ -f "$nativeBundle" ] && unzip -l "$nativeBundle" | grep -q
'libcomet'; then
+ cp "$nativeBundle" $TMP_JARS_DIR/
+ fi
+ done
cp
${GITHUB_WORKSPACE}/packaging/hudi-utilities-bundle/target/hudi-*-$HUDI_VERSION.jar
$TMP_JARS_DIR/
cp
${GITHUB_WORKSPACE}/packaging/hudi-utilities-slim-bundle/target/hudi-*-$HUDI_VERSION.jar
$TMP_JARS_DIR/
echo 'Validating jars below:'
@@ -184,6 +202,8 @@ else
echo 'Adding environment variables for bundles in the release candidate or
artifact'
HUDI_HADOOP_MR_BUNDLE_NAME=hudi-hadoop-mr-bundle
+ # Empty for Spark versions Apache DataFusion Comet has no release for.
+ HUDI_SPARK_NATIVE_BUNDLE_NAME=
HUDI_KAFKA_CONNECT_BUNDLE_NAME=hudi-kafka-connect-bundle
HUDI_METASERVER_SERVER_BUNDLE_NAME=hudi-metaserver-server-bundle
@@ -195,26 +215,31 @@ else
elif [[ ${SPARK_PROFILE} == 'spark3.4' ]]; then
HUDI_CLI_BUNDLE_NAME=hudi-cli-bundle_2.12
HUDI_SPARK_BUNDLE_NAME=hudi-spark3.4-bundle_2.12
+ HUDI_SPARK_NATIVE_BUNDLE_NAME=hudi-spark3.4-native-bundle_2.12
HUDI_UTILITIES_BUNDLE_NAME=hudi-utilities-bundle_2.12
HUDI_UTILITIES_SLIM_BUNDLE_NAME=hudi-utilities-slim-bundle_2.12
elif [[ ${SPARK_PROFILE} == 'spark3.5' && ${SCALA_PROFILE} == 'scala-2.12'
]]; then
HUDI_CLI_BUNDLE_NAME=hudi-cli-bundle_2.12
HUDI_SPARK_BUNDLE_NAME=hudi-spark3.5-bundle_2.12
+ HUDI_SPARK_NATIVE_BUNDLE_NAME=hudi-spark3.5-native-bundle_2.12
HUDI_UTILITIES_BUNDLE_NAME=hudi-utilities-bundle_2.12
HUDI_UTILITIES_SLIM_BUNDLE_NAME=hudi-utilities-slim-bundle_2.12
elif [[ ${SPARK_PROFILE} == 'spark3.5' && ${SCALA_PROFILE} == 'scala-2.13'
]]; then
HUDI_CLI_BUNDLE_NAME=hudi-cli-bundle_2.13
HUDI_SPARK_BUNDLE_NAME=hudi-spark3.5-bundle_2.13
+ HUDI_SPARK_NATIVE_BUNDLE_NAME=hudi-spark3.5-native-bundle_2.13
HUDI_UTILITIES_BUNDLE_NAME=hudi-utilities-bundle_2.13
HUDI_UTILITIES_SLIM_BUNDLE_NAME=hudi-utilities-slim-bundle_2.13
elif [[ ${SPARK_PROFILE} == 'spark4.0' && ${SCALA_PROFILE} == 'scala-2.13'
]]; then
HUDI_CLI_BUNDLE_NAME=hudi-cli-bundle_2.13
HUDI_SPARK_BUNDLE_NAME=hudi-spark4.0-bundle_2.13
+ HUDI_SPARK_NATIVE_BUNDLE_NAME=hudi-spark4.0-native-bundle_2.13
HUDI_UTILITIES_BUNDLE_NAME=hudi-utilities-bundle_2.13
HUDI_UTILITIES_SLIM_BUNDLE_NAME=hudi-utilities-slim-bundle_2.13
elif [[ ${SPARK_PROFILE} == 'spark4.1' && ${SCALA_PROFILE} == 'scala-2.13'
]]; then
HUDI_CLI_BUNDLE_NAME=hudi-cli-bundle_2.13
HUDI_SPARK_BUNDLE_NAME=hudi-spark4.1-bundle_2.13
+ HUDI_SPARK_NATIVE_BUNDLE_NAME=hudi-spark4.1-native-bundle_2.13
HUDI_UTILITIES_BUNDLE_NAME=hudi-utilities-bundle_2.13
HUDI_UTILITIES_SLIM_BUNDLE_NAME=hudi-utilities-slim-bundle_2.13
elif [[ ${SPARK_PROFILE} == 'spark4.2' && ${SCALA_PROFILE} == 'scala-2.13'
]]; then
@@ -249,6 +274,9 @@ else
wget -q
$REPO_BASE_URL/$HUDI_HADOOP_MR_BUNDLE_NAME/$HUDI_VERSION/$HUDI_HADOOP_MR_BUNDLE_NAME-$HUDI_VERSION.jar
-P $TMP_JARS_DIR/
wget -q
$REPO_BASE_URL/$HUDI_KAFKA_CONNECT_BUNDLE_NAME/$HUDI_VERSION/$HUDI_KAFKA_CONNECT_BUNDLE_NAME-$HUDI_VERSION.jar
-P $TMP_JARS_DIR/
wget -q
$REPO_BASE_URL/$HUDI_SPARK_BUNDLE_NAME/$HUDI_VERSION/$HUDI_SPARK_BUNDLE_NAME-$HUDI_VERSION.jar
-P $TMP_JARS_DIR/
+ if [[ -n "$HUDI_SPARK_NATIVE_BUNDLE_NAME" ]]; then
+ wget -q
$REPO_BASE_URL/$HUDI_SPARK_NATIVE_BUNDLE_NAME/$HUDI_VERSION/$HUDI_SPARK_NATIVE_BUNDLE_NAME-$HUDI_VERSION.jar
-P $TMP_JARS_DIR/
+ fi
wget -q
$REPO_BASE_URL/$HUDI_UTILITIES_BUNDLE_NAME/$HUDI_VERSION/$HUDI_UTILITIES_BUNDLE_NAME-$HUDI_VERSION.jar
-P $TMP_JARS_DIR/
wget -q
$REPO_BASE_URL/$HUDI_UTILITIES_SLIM_BUNDLE_NAME/$HUDI_VERSION/$HUDI_UTILITIES_SLIM_BUNDLE_NAME-$HUDI_VERSION.jar
-P $TMP_JARS_DIR/
wget -q
$REPO_BASE_URL/$HUDI_METASERVER_SERVER_BUNDLE_NAME/$HUDI_VERSION/$HUDI_METASERVER_SERVER_BUNDLE_NAME-$HUDI_VERSION.jar
-P $TMP_JARS_DIR/
diff --git a/packaging/bundle-validation/spark_native/validate.scala
b/packaging/bundle-validation/spark_native/validate.scala
new file mode 100644
index 000000000000..76fe17cb7092
--- /dev/null
+++ b/packaging/bundle-validation/spark_native/validate.scala
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.spark.sql.SaveMode._
+import org.apache.hudi.DataSourceWriteOptions._
+import org.apache.hudi.config.HoodieWriteConfig._
+
+val outputDir = "/tmp/spark-native-bundle"
+
+// Force a real join rather than a broadcast, so the plan exercises Comet's
join, shuffle and sort.
+spark.conf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
+
+// Deterministic input, so the query result can be asserted exactly. 300 rows
spread evenly over
+// three partitions, fare equal to the row id plus the offset.
+def rows(from: Int, to: Int, offset: Int = 0) = spark.range(from,
to).selectExpr(
+ "concat('id-', cast(id as string)) as uuid",
+ "cast(id % 3 as string) as partitionpath",
+ s"cast(id + $offset as double) as fare",
+ s"id + $offset as ts")
+
+def write(name: String, tableType: String, mode: org.apache.spark.sql.SaveMode,
+ df: org.apache.spark.sql.DataFrame): String = {
+ val path = "file:///tmp/hudi-bundles/tests/" + name
+ df.write.format("hudi").
+ option(PRECOMBINE_FIELD_OPT_KEY, "ts").
+ option(RECORDKEY_FIELD_OPT_KEY, "uuid").
+ option(PARTITIONPATH_FIELD_OPT_KEY, "partitionpath").
+ option(TABLE_TYPE_OPT_KEY, tableType).
+ option(TABLE_NAME, name).
+ mode(mode).
+ save(path)
+ path
+}
+
+// Each partition holds 100 rows per side, so the join emits 100 * 100 rows
per partition and each
+// left fare is summed 100 times. Aggregate a data column, not just the
partition column: a scan
+// projecting no data columns reads as ReadSchema struct<> and Comet declines
to bridge it.
+def probe(label: String, leftPath: String, rightPath: String): Unit = {
+ spark.read.format("hudi").load(leftPath).createOrReplaceTempView("t1")
+ spark.read.format("hudi").load(rightPath).createOrReplaceTempView("t2")
+ val query = spark.sql(
+ "select t1.partitionpath, count(*) as c, sum(t1.fare) as s from t1 " +
+ "join t2 on t1.partitionpath = t2.partitionpath group by t1.partitionpath
order by t1.partitionpath")
+ val result = query.collect().map(r => s"${r.get(0)},${r.get(1)},${r.get(2)}")
+ result.foreach(r => println(s"::warning::native bundle $label row $r"))
+ sc.parallelize(result, 1).saveAsTextFile(s"$outputDir/${label}_rows")
+
+ // Comet does not recognize Hudi's file format and leaves the scan to Spark,
but with
+ // spark.comet.convert.parquet.enabled it bridges the scan's output into
Arrow and runs everything
+ // above it natively. Copy-on-write keeps the vectorized read and bridges
columnar to columnar;
+ // merge-on-read reads row by row because file group merging is row level.
Asserting on the plan
+ // matters because Comet degrades silently: a mis-relocated Comet or a
libcomet.so that failed to
+ // load still returns correct results.
+ val plan = query.queryExecution.executedPlan.toString
+ println(s"::warning::native bundle $label executed plan\n" + plan)
+ sc.parallelize(Seq(plan), 1).saveAsTextFile(s"$outputDir/${label}_plan")
+}
+
+probe("cow", write("native_cow_1", "COPY_ON_WRITE", Overwrite, rows(0, 300)),
+ write("native_cow_2", "COPY_ON_WRITE", Overwrite, rows(0, 300)))
+
+// Merge-on-read with a second commit that updates the fare of half the rows,
so the snapshot read
+// returns the expected sums only if it merges base files with log files.
+val morLeft = write("native_mor_1", "MERGE_ON_READ", Overwrite, rows(0, 300))
+write("native_mor_1", "MERGE_ON_READ", Append, rows(0, 150, 1000))
+val morRight = write("native_mor_2", "MERGE_ON_READ", Overwrite, rows(0, 300))
+write("native_mor_2", "MERGE_ON_READ", Append, rows(0, 150, 1000))
+probe("mor", morLeft, morRight)
+
+System.exit(0)
diff --git a/packaging/bundle-validation/utilities/hoodieapp.properties
b/packaging/bundle-validation/utilities/hoodieapp.properties
index 835b09dcee41..fcf3915bec66 100644
--- a/packaging/bundle-validation/utilities/hoodieapp.properties
+++ b/packaging/bundle-validation/utilities/hoodieapp.properties
@@ -18,6 +18,8 @@ hoodie.datasource.write.recordkey.field=key
hoodie.datasource.write.partitionpath.field=date
hoodie.table.ordering.fields=ts
hoodie.metadata.enable=true
+hoodie.metrics.on=true
+hoodie.metrics.reporter.type=INMEMORY
hoodie.deltastreamer.source.dfs.root=file:///opt/bundle-validation/data/stocks/data
hoodie.deltastreamer.schemaprovider.target.schema.file=file:///opt/bundle-validation/data/stocks/schema.avsc
hoodie.deltastreamer.schemaprovider.source.schema.file=file:///opt/bundle-validation/data/stocks/schema.avsc
diff --git a/packaging/bundle-validation/validate.sh
b/packaging/bundle-validation/validate.sh
index 5a893cd225c0..b7b0af69c675 100755
--- a/packaging/bundle-validation/validate.sh
+++ b/packaging/bundle-validation/validate.sh
@@ -40,7 +40,14 @@ if [[ "$SCALA_PROFILE" != 'scala-2.13' ]]; then
ln -sf $JARS_DIR/hudi-flink*.jar $JARS_DIR/flink.jar
ln -sf $JARS_DIR/hudi-kafka-connect-bundle*.jar $JARS_DIR/kafka-connect.jar
fi
-ln -sf $JARS_DIR/hudi-spark*.jar $JARS_DIR/spark.jar
+# The spark native bundle shares the hudi-spark prefix, so match the plain
spark bundle only.
+ln -sf $(ls $JARS_DIR/hudi-spark*.jar | grep -v -- '-native-bundle_')
$JARS_DIR/spark.jar
+# Only present for the Spark versions Apache DataFusion Comet releases for.
+for nativeBundle in $JARS_DIR/hudi-spark*-native-bundle_*.jar; do
+ if [ -f "$nativeBundle" ]; then
+ ln -sf "$nativeBundle" $JARS_DIR/spark-native.jar
+ fi
+done
ln -sf $JARS_DIR/hudi-utilities-bundle*.jar $JARS_DIR/utilities.jar
ln -sf $JARS_DIR/hudi-utilities-slim*.jar $JARS_DIR/utilities-slim.jar
ln -sf $JARS_DIR/hudi-metaserver-server-bundle*.jar $JARS_DIR/metaserver.jar
@@ -69,6 +76,97 @@ use_default_java_runtime () {
export JAVA_HOME=${DEFAULT_JAVA_HOME}
}
+##
+# Function to test the spark native bundle, which carries Apache DataFusion
Comet.
+#
+# Comet needs Java 17 and a glibc JVM for libcomet.so, so this runs on the
Java 17 pass and skips on
+# a musl image (the Alpine based image's Java 17 is musl too). Comet's plugin
is not enabled without
+# off-heap memory, and nativeLoadRequired makes a libcomet.so load failure an
error instead of a
+# silent fallback to Spark. The session uses the in-memory catalog because a
Hudi path read looks
+# the table up in the catalog and the Hive metastore only runs inside
test_spark_hadoop_mr_bundles.
+#
+# env vars (defined in container):
+# SPARK_HOME: path to the spark directory
+##
+test_spark_native_bundle () {
+ if ls /lib/ld-musl-*.so.1 > /dev/null 2>&1; then
+ echo "::warning::validate.sh skip validating spark native bundle,
libcomet.so needs glibc and this image is musl"
+ return 0
+ fi
+ local outputDir=/tmp/spark-native-bundle
+ rm -rf $outputDir
+ change_java_runtime_version
+ echo "::warning::validate.sh Writing and querying Hudi tables with Comet
enabled"
+ $SPARK_HOME/bin/spark-shell --jars $JARS_DIR/spark-native.jar \
+ --conf 'spark.plugins=org.apache.spark.CometPlugin' \
+ --conf
'spark.sql.extensions=org.apache.spark.sql.hudi.HoodieSparkSessionExtension,org.apache.comet.CometSparkSessionExtensions'
\
+ --conf
'spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager'
\
+ --conf 'spark.memory.offHeap.enabled=true' \
+ --conf 'spark.memory.offHeap.size=1g' \
+ --conf 'spark.comet.enabled=true' \
+ --conf 'spark.comet.exec.enabled=true' \
+ --conf 'spark.comet.nativeLoadRequired=true' \
+ --conf 'spark.comet.convert.parquet.enabled=true' \
+ --conf 'spark.comet.explain.fallback.enabled=true' \
+ --conf 'spark.comet.metrics.enabled=true' \
+ --conf 'spark.serializer=org.apache.spark.serializer.KryoSerializer' \
+ --conf
'spark.kryo.registrator=org.apache.spark.HoodieSparkKryoRegistrar' \
+ --conf 'spark.sql.catalogImplementation=in-memory' <
$WORKDIR/spark_native/validate.scala
+
+ use_default_java_runtime
+
+ # 300 rows per table over three partitions: 100 * 100 joined rows per
partition, and each cow
+ # fare summed 100 times (100 * 14850, 100 * 14950, 100 * 15050).
+ local expectedRows='0,10000,1485000.0
+1,10000,1495000.0
+2,10000,1505000.0'
+ local actualRows
+ actualRows=$(cat $outputDir/cow_rows/part-*)
+ if [ "$actualRows" != "$expectedRows" ]; then
+ echo "::error::validate.sh spark native bundle copy-on-write query
returned unexpected results"
+ echo "expected:"; echo "$expectedRows"
+ echo "actual:"; echo "$actualRows"
+ return 1
+ fi
+
+ # The second merge-on-read commit adds 1000 to the fare of 50 rows per
partition, and it lands
+ # in log files, so a read that skips the logs returns the copy-on-write
sums instead
+ # (100 * (14850 + 50000), 100 * (14950 + 50000), 100 * (15050 + 50000)).
+ local expectedMorRows='0,10000,6485000.0
+1,10000,6495000.0
+2,10000,6505000.0'
+ local actualMorRows
+ actualMorRows=$(cat $outputDir/mor_rows/part-*)
+ if [ "$actualMorRows" != "$expectedMorRows" ]; then
+ echo "::error::validate.sh spark native bundle merge-on-read query
returned unexpected results"
+ echo "expected:"; echo "$expectedMorRows"
+ echo "actual:"; echo "$actualMorRows"
+ return 1
+ fi
+
+ # Comet declines what it cannot accelerate and hands it back to Spark, so
correct results on
+ # their own would still pass with a mis-relocated Comet or a libcomet.so
that failed to load.
+ # Copy-on-write keeps the vectorized read and bridges columnar to
columnar; merge-on-read reads
+ # row by row because file group merging is row level, and bridges through
a row conversion.
+ if ! grep -q 'CometSortMergeJoin' $outputDir/cow_plan/part-*; then
+ echo "::error::validate.sh join over copy-on-write Hudi tables was not
executed natively by Comet"
+ cat $outputDir/cow_plan/part-*
+ return 1
+ fi
+ if ! grep -q 'CometSparkColumnarToColumnar' $outputDir/cow_plan/part-* \
+ || grep -q 'CometSparkRowToColumnar' $outputDir/cow_plan/part-*; then
+ echo "::error::validate.sh copy-on-write scan was not bridged into
Comet columnar to columnar"
+ cat $outputDir/cow_plan/part-*
+ return 1
+ fi
+ if ! grep -q 'CometSparkRowToColumnar' $outputDir/mor_plan/part-*; then
+ echo "::error::validate.sh merge-on-read scan was not bridged into
Comet"
+ cat $outputDir/mor_plan/part-*
+ return 1
+ fi
+ echo "::warning::validate.sh spark native bundle validation was successful"
+}
+
##
# Function to test the spark & hadoop-mr bundles with hive sync.
#
@@ -171,7 +269,9 @@ test_utilities_bundle () {
fi
echo "::warning::validate.sh validating deltastreamer in spark shell"
- SHELL_COMMAND="$SPARK_HOME/bin/spark-shell $OPT_JARS $MAIN_JAR -i
$WORKDIR/utilities/validate.scala"
+ # The bundles go in --jars so spark.sql.extensions can load
HoodieSparkSessionExtension from them; Spark 3.5.9+
+ # fails the session when the extension class is missing, and a positional
jar is not on the driver classpath.
+ SHELL_COMMAND="$SPARK_HOME/bin/spark-shell --jars
$MAIN_JAR${EXTRA_JARS:+,$EXTRA_JARS} -i $WORKDIR/utilities/validate.scala"
echo "::debug::this is the shell command: $SHELL_COMMAND"
LOGFILE="$WORKDIR/${FUNCNAME[0]}.log"
$SHELL_COMMAND >> $LOGFILE
@@ -377,6 +477,22 @@ if [ "$?" -ne 0 ]; then
fi
echo "::warning::validate.sh done validating spark & hadoop-mr bundle"
+if [ -e $JARS_DIR/spark-native.jar ] && [[ ${JAVA_RUNTIME_VERSION} ==
'openjdk17' ]]; then
+ echo "::warning::validate.sh validating spark native bundle"
+ test_spark_native_bundle
+ if [ "$?" -ne 0 ]; then
+ exit 1
+ fi
+ # The slim utilities bundle takes Hudi's relocated third-party classes from
the Spark bundle it is
+ # paired with, so it has to work with this one too. No Comet config is set
here, so this runs on
+ # musl images as well.
+ echo "::warning::validate.sh validating utilities slim bundle with spark
native bundle"
+ test_utilities_bundle $JARS_DIR/utilities-slim.jar $JARS_DIR/spark-native.jar
+ echo "::warning::validate.sh done validating spark native bundle"
+else
+ echo "::warning::validate.sh skip validating spark native bundle, needs
openjdk17 and a Spark version Comet releases for"
+fi
+
if [[ $SPARK_HOME == *"spark-3.5"* || $SPARK_HOME == *"spark-4.0"* ||
$SPARK_HOME == *"spark-4.1"* ]]
then
echo "::warning::validate.sh validating cli bundle"
diff --git a/packaging/hudi-spark-native-bundle/README.md
b/packaging/hudi-spark-native-bundle/README.md
new file mode 100644
index 000000000000..4df90a18d070
--- /dev/null
+++ b/packaging/hudi-spark-native-bundle/README.md
@@ -0,0 +1,123 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# hudi-spark-native-bundle
+
+`hudi-spark-bundle` plus [Apache DataFusion
Comet](https://datafusion.apache.org/comet/) for native
+vectorized execution. Use it instead of `hudi-spark-bundle`, and remove any
other Comet jar from the
+classpath: Comet cannot be relocated, so two copies resolve class by class
while `libcomet.so` is
+picked up as a resource, and the mix ends in `UnsatisfiedLinkError` or
`NoSuchMethodError`.
+
+## Supported Spark versions
+
+Comet publishes one artifact per Spark minor version, so the bundle exists
only where Comet does:
+
+| Spark | Scala | Bundle |
+|-------|------------|---------------------------------------|
+| 3.4 | 2.12 | `hudi-spark3.4-native-bundle_2.12` |
+| 3.5 | 2.12, 2.13 | `hudi-spark3.5-native-bundle_<scala>` |
+| 4.0 | 2.13 | `hudi-spark4.0-native-bundle_2.13` |
+| 4.1 | 2.13 | `hudi-spark4.1-native-bundle_2.13` |
+
+`-Dspark3.3` and `-Dspark4.2` build fine and produce no bundle (Comet dropped
3.3 after 0.7.0 and
+supports 4.2 only experimentally, with no published artifact). Comet 1.1.0
deprecates Spark 3.4, so that bundle goes away once Comet drops it.
+
+Run the bundle on the Spark patch release Comet is built against (3.5.9, 4.0.4
and 4.1.3 for Comet 1.1.0) or a
+close one: Spark 3.5.1 lacks `InMemoryTableScanLike`, which Comet's planner
rule references, so Comet fails to
+initialize there and takes the session down with it.
+
+## Runtime requirements
+
+- **Java 17 or later**, also on Spark 3.4 and 3.5: Comet ships Java 17
bytecode, and Java 11 fails
+ with `UnsupportedClassVersionError ... class file version 61.0`.
+- **Linux with glibc**, `amd64` or `aarch64`: those are the only `libcomet.so`
builds Comet ships.
+ A musl JVM (Alpine) fails to load it, and macOS or Windows fail with
`Unsupported OS/arch`. On a Mac,
+ run in a Linux container (`--platform linux/arm64` on Apple Silicon needs no
emulation).
+
+## Usage
+
+The bundle on the classpath enables nothing by itself:
+
+```
+spark-shell --jars hudi-spark3.5-native-bundle_2.12-<version>.jar \
+ --conf spark.plugins=org.apache.spark.CometPlugin \
+ --conf
spark.sql.extensions=org.apache.spark.sql.hudi.HoodieSparkSessionExtension,org.apache.comet.CometSparkSessionExtensions
\
+ --conf
spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager
\
+ --conf spark.memory.offHeap.enabled=true \
+ --conf spark.memory.offHeap.size=1g \
+ --conf spark.comet.enabled=true \
+ --conf spark.comet.exec.enabled=true \
+ --conf spark.comet.nativeLoadRequired=true \
+ --conf spark.comet.convert.parquet.enabled=true
+```
+
+- Comet's plugin is not enabled without off-heap memory, so size
`spark.memory.offHeap.size` for the job.
+- `spark.comet.nativeLoadRequired=true` fails the session when `libcomet.so`
cannot load, instead of
+ silently running everything on Spark.
+- `spark.comet.convert.parquet.enabled=true` is the switch that lets Comet
accelerate anything over a
+ Hudi table (see below); `spark.comet.sparkToColumnar.enabled` does not do it.
+
+Comet hands back whatever it cannot accelerate, so correct results do not mean
native execution.
+Look for `Comet` operators in `df.queryExecution.executedPlan`, and set
+`spark.comet.explain.fallback.enabled=true` to log why each declined operator
fell back.
+
+## What Comet accelerates
+
+Not the Hudi scan: Comet's scan rule recognizes Spark's `ParquetFileFormat`,
not Hudi's
+`HoodieFileGroupReaderBasedFileFormat`, so the scan stays on Spark. With
+`spark.comet.convert.parquet.enabled=true` Comet bridges the scan output into
Arrow and runs the rest
+of the plan natively, for example a join of two copy-on-write tables:
+
+```
+CometHashAggregate [Final]
++- CometHashAggregate [Partial]
+ +- CometProject
+ +- CometSortMergeJoin [partitionpath], Inner
+ :- CometSort
+ : +- CometExchange hashpartitioning(...), CometNativeShuffle
+ : +- CometSparkColumnarToColumnar
+ : +- FileScan HudiFileGroup [...] Format:
HoodieFileGroupReaderBasedFileFormat
+```
+
+The bridge differs by table type, and the difference is not free:
+
+| Table | Scan | Bridge
|
+|---------------|---------------------------------------------|--------------------------------|
+| Copy-on-write | `Batched: true`, vectorized Parquet reader |
`CometSparkColumnarToColumnar` |
+| Merge-on-read | `Batched: false`, row by row (log merging) |
`CometSparkRowToColumnar` |
+
+Merge-on-read rows are assembled on heap and converted to Arrow cell by cell,
so benchmark before
+assuming such a workload comes out ahead. A scan that projects no data column
(`ReadSchema: struct<>`)
+is not bridged and stays on Spark.
+
+## Shading
+
+The bundle is the shaded `hudi-spark-bundle` jar plus the Comet jar, with no
relocation of its own.
+Hudi's relocated third-party classes (dropwizard metrics, jetty, commons-io,
...) arrive already
+relocated, which keeps the bundle binary compatible with
`hudi-utilities-slim-bundle`, and Comet keeps
+the names Spark resolves it by.
+
+Comet must not be relocated: `libcomet.so` binds through JNI symbols named
+`Java_org_apache_comet_Native_*` and is loaded from the
`org/apache/comet/<os>/<arch>/` resource path,
+so a relocated `org.apache.comet` builds and then fails on the first native
call. The same holds for
+`org.apache.arrow.c` (the Arrow C data interface JNI bindings) and for
`org.apache.spark.CometSource`,
+whose `metricRegistry()` must keep returning the
`com.codahale.metrics.MetricRegistry` that Spark's
+`Source` declares.
+
+Anything else shaded in next to Comet is therefore unrelocated too, and must
not call a Hudi API
+whose signature exposes a relocated type (`Metrics.getRegistry()`, for
example): it compiles and fails
+at runtime with `NoSuchMethodError`.
diff --git a/packaging/hudi-spark-native-bundle/pom.xml
b/packaging/hudi-spark-native-bundle/pom.xml
new file mode 100644
index 000000000000..d2fa08191519
--- /dev/null
+++ b/packaging/hudi-spark-native-bundle/pom.xml
@@ -0,0 +1,242 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <parent>
+ <artifactId>hudi</artifactId>
+ <groupId>org.apache.hudi</groupId>
+ <version>1.3.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
<artifactId>hudi-spark${sparkbundle.version}-native-bundle_${scala.binary.version}</artifactId>
+ <packaging>jar</packaging>
+
+ <!-- The shaded hudi-spark-bundle jar plus Apache DataFusion Comet, with no
relocation of its own:
+ Hudi keeps its relocated third-party copies (and stays binary
compatible with the slim
+ utilities bundle), and Comet keeps the names Spark resolves it by. -->
+
+ <properties>
+ <checkstyle.skip>true</checkstyle.skip>
+ <main.basedir>${project.parent.basedir}</main.basedir>
+ <skipTests>true</skipTests>
+ <!-- Comet releases one artifact per Spark minor version, so by default
this module produces
+ no bundle (a sources jar only, nothing installed or deployed) rather
than one named "native"
+ with no native code in it. The comet-spark<version> profiles below
opt in. -->
+ <hudi.spark.native.bundle.skip>true</hudi.spark.native.bundle.skip>
+
<hudi.spark.native.bundle.jar.phase>none</hudi.spark.native.bundle.jar.phase>
+ <maven.install.skip>${hudi.spark.native.bundle.skip}</maven.install.skip>
+ <maven.deploy.skip>${hudi.spark.native.bundle.skip}</maven.deploy.skip>
+ <!-- The javadoc jar puts the module's own jar on its classpath (see the
root pom), and a skipped bundle has none. -->
+ <maven.javadoc.skip>${hudi.spark.native.bundle.skip}</maven.javadoc.skip>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>default-jar</id>
+ <!-- none when this module produces no bundle. Without it
maven-jar-plugin still emits
+ a jar named as if it were a bundle, holding nothing but the
generated META-INF.
+ skipIfEmpty does not help: remote-resources populates
target/classes regardless. -->
+ <phase>${hudi.spark.native.bundle.jar.phase}</phase>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${maven-shade-plugin.version}</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <skip>${hudi.spark.native.bundle.skip}</skip>
+ <createSourcesJar>${shadeSources}</createSourcesJar>
+
<dependencyReducedPomLocation>${project.build.directory}/dependency-reduced-pom.xml
+ </dependencyReducedPomLocation>
+ <transformers>
+ <transformer
implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"
/>
+ <transformer
implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
+ <addHeader>true</addHeader>
+ </transformer>
+ <transformer
implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <resource>META-INF/LICENSE</resource>
+ <file>target/classes/META-INF/LICENSE</file>
+ </transformer>
+ <transformer
implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+
<resource>META-INF/services/org.apache.spark.sql.sources.DataSourceRegister</resource>
+ </transformer>
+ <transformer
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ </transformers>
+ <!-- Neither the common includes nor the relocations the parent
pom gives every
+ bundle apply here: the spark bundle jar already carries all
of them. Both are
+ overridden as a whole, so a profile appending to the
includes cannot pull the
+ parent's list back in. -->
+ <artifactSet combine.self="override">
+ <includes>
+
<include>org.apache.hudi:hudi-spark${sparkbundle.version}-bundle_${scala.binary.version}</include>
+
+ <!-- NOTE: Comet must never be relocated: libcomet.so binds
through JNI symbols
+ named Java_org_apache_comet_Native_* and is
loaded from the
+ org/apache/comet/<os>/<arch>/ resource path, so a
relocated
+ org.apache.comet builds and then fails on the
first native call. The same
+ holds for org.apache.arrow.c (the Arrow C data
interface JNI bindings) and
+ for org.apache.spark.CometSource, whose
metricRegistry() must keep returning
+ the com.codahale.metrics.MetricRegistry that
Spark's Source declares. -->
+ <include>org.apache.datafusion:*</include>
+ </includes>
+ </artifactSet>
+ <relocations combine.self="override" />
+ <filters>
+ <filter>
+ <!-- Comet ships Arrow and Guava relocated under
org.apache.comet.shaded but keeps
+ their original META-INF/maven descriptors, which
misreports the bundled
+ versions to anything that probes them. -->
+ <artifact>org.apache.datafusion:*</artifact>
+ <excludes>
+ <exclude>META-INF/maven/org.apache.arrow/**</exclude>
+ <exclude>META-INF/maven/com.google.guava/**</exclude>
+ </excludes>
+ </filter>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ <exclude>META-INF/services/javax.*</exclude>
+ <exclude>**/*.proto</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hudi</groupId>
+
<artifactId>hudi-spark${sparkbundle.version}-bundle_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+ <profiles>
+ <!-- One profile per Spark version Comet publishes
comet-spark-spark<X.Y>_<scala> for. Spark 3.3
+ (dropped after Comet 0.7.0) and 4.2 (experimental in Comet, no
published artifact) fall through to the skip above; Spark 3.4
+ is deprecated as of Comet 1.1.0 and goes the same way once Comet
drops it. -->
+ <profile>
+ <id>comet-spark3.4</id>
+ <activation>
+ <property>
+ <name>spark3.4</name>
+ </property>
+ </activation>
+ <properties>
+ <hudi.spark.native.bundle.skip>false</hudi.spark.native.bundle.skip>
+
<hudi.spark.native.bundle.jar.phase>package</hudi.spark.native.bundle.jar.phase>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.datafusion</groupId>
+ <artifactId>comet-spark-spark3.4_${scala.binary.version}</artifactId>
+ <version>${datafusion.comet.version}</version>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <profile>
+ <id>comet-spark3.5</id>
+ <activation>
+ <property>
+ <name>spark3.5</name>
+ </property>
+ </activation>
+ <properties>
+ <hudi.spark.native.bundle.skip>false</hudi.spark.native.bundle.skip>
+
<hudi.spark.native.bundle.jar.phase>package</hudi.spark.native.bundle.jar.phase>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.datafusion</groupId>
+ <artifactId>comet-spark-spark3.5_${scala.binary.version}</artifactId>
+ <version>${datafusion.comet.version}</version>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <profile>
+ <id>comet-spark4.0</id>
+ <activation>
+ <property>
+ <name>spark4.0</name>
+ </property>
+ </activation>
+ <properties>
+ <hudi.spark.native.bundle.skip>false</hudi.spark.native.bundle.skip>
+
<hudi.spark.native.bundle.jar.phase>package</hudi.spark.native.bundle.jar.phase>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.datafusion</groupId>
+ <artifactId>comet-spark-spark4.0_${scala.binary.version}</artifactId>
+ <version>${datafusion.comet.version}</version>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <profile>
+ <id>comet-spark4.1</id>
+ <activation>
+ <property>
+ <name>spark4.1</name>
+ </property>
+ </activation>
+ <properties>
+ <hudi.spark.native.bundle.skip>false</hudi.spark.native.bundle.skip>
+
<hudi.spark.native.bundle.jar.phase>package</hudi.spark.native.bundle.jar.phase>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.datafusion</groupId>
+ <artifactId>comet-spark-spark4.1_${scala.binary.version}</artifactId>
+ <version>${datafusion.comet.version}</version>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+</project>
diff --git
a/packaging/hudi-spark-native-bundle/src/main/java/org/apache/hudi/spark/nativebundle/Main.java
b/packaging/hudi-spark-native-bundle/src/main/java/org/apache/hudi/spark/nativebundle/Main.java
new file mode 100644
index 000000000000..65635afc39ee
--- /dev/null
+++
b/packaging/hudi-spark-native-bundle/src/main/java/org/apache/hudi/spark/nativebundle/Main.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.spark.nativebundle;
+
+import org.apache.hudi.common.util.ReflectionUtils;
+
+/**
+ * A simple main class to dump all classes loaded in current classpath
+ *
+ * This is a workaround for generating sources and javadoc jars for packaging
modules. The maven plugins for generating
+ * javadoc and sources plugins do not generate corresponding jars if there are
no source files.
+ *
+ * This class does not have anything to do with Hudi but is there to keep mvn
javadocs/source plugin happy. It
+ * lives in its own package because the shaded hudi-spark-bundle jar already
carries
+ * org.apache.hudi.spark.bundle.Main.
+ */
+public class Main {
+
+ public static void main(String[] args) {
+
ReflectionUtils.getTopLevelClassesInClasspath(Main.class).forEach(System.out::println);
+ }
+}
diff --git a/pom.xml b/pom.xml
index ad4068c129d4..91fba5ec295f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -55,6 +55,7 @@
<module>packaging/hudi-gcp-bundle</module>
<module>packaging/hudi-azure-bundle</module>
<module>packaging/hudi-spark-bundle</module>
+ <module>packaging/hudi-spark-native-bundle</module>
<module>packaging/hudi-presto-bundle</module>
<module>packaging/hudi-utilities-bundle</module>
<module>packaging/hudi-utilities-slim-bundle</module>
@@ -195,6 +196,7 @@
<spark40.version>4.0.2</spark40.version>
<spark41.version>4.1.1</spark41.version>
<spark42.version>4.2.0</spark42.version>
+ <datafusion.comet.version>1.1.0</datafusion.comet.version>
<hudi.spark.module>hudi-spark3.5.x</hudi.spark.module>
<hudi.spark.common.module>hudi-spark3-common</hudi.spark.common.module>
<avro.version>1.11.4</avro.version>
diff --git a/scripts/release/deploy_staging_jars.sh
b/scripts/release/deploy_staging_jars.sh
index a5947e9a4757..550eaad29a9d 100755
--- a/scripts/release/deploy_staging_jars.sh
+++ b/scripts/release/deploy_staging_jars.sh
@@ -54,19 +54,22 @@ declare -a ALL_VERSION_OPTS=(
# hudi-spark3.5.x_2.13
# hudi-utilities_2.13
# hudi-spark3.5-bundle_2.13
+# hudi-spark3.5-native-bundle_2.13
# hudi-utilities-bundle_2.13
# hudi-utilities-slim-bundle_2.13
# hudi-cli-bundle_2.13
-"-T 1C -Dscala-2.13 -Dspark3.5 -pl
hudi-spark-datasource/hudi-spark-common,hudi-spark-datasource/hudi-spark3.5.x,hudi-spark-datasource/hudi-spark,hudi-utilities,packaging/hudi-spark-bundle,packaging/hudi-utilities-bundle,packaging/hudi-utilities-slim-bundle,packaging/hudi-cli-bundle
-am"
+"-T 1C -Dscala-2.13 -Dspark3.5 -pl
hudi-spark-datasource/hudi-spark-common,hudi-spark-datasource/hudi-spark3.5.x,hudi-spark-datasource/hudi-spark,hudi-utilities,packaging/hudi-spark-bundle,packaging/hudi-spark-native-bundle,packaging/hudi-utilities-bundle,packaging/hudi-utilities-slim-bundle,packaging/hudi-cli-bundle
-am"
# For Spark 3.3, Scala 2.12:
+# (no hudi-spark3.3-native-bundle: Apache DataFusion Comet dropped Spark 3.3
after 0.7.0)
# hudi-spark3.3.x_2.12
# hudi-spark3.3-bundle_2.12
"-T 1C -Dscala-2.12 -Dspark3.3 -pl
hudi-spark-datasource/hudi-spark3.3.x,packaging/hudi-spark-bundle -am"
# For Spark 3.4, Scala 2.12:
# hudi-spark3.4.x_2.12
# hudi-spark3.4-bundle_2.12
-"-T 1C -Dscala-2.12 -Dspark3.4 -pl
hudi-spark-datasource/hudi-spark3.4.x,packaging/hudi-spark-bundle -am"
-# For all modules spark3.5
+# hudi-spark3.4-native-bundle_2.12
+"-T 1C -Dscala-2.12 -Dspark3.4 -pl
hudi-spark-datasource/hudi-spark3.4.x,packaging/hudi-spark-bundle,packaging/hudi-spark-native-bundle
-am"
+# For all modules spark3.5 (this pass also produces
hudi-spark3.5-native-bundle_2.12)
"-Dscala-2.12 -Dspark3.5 -Dflink1.20 -Ddocker.compose.skip=true"
# Upload legacy Spark bundles (not overwriting previous uploads as these jar
names are unique)
diff --git a/scripts/release/deploy_staging_jars_java17.sh
b/scripts/release/deploy_staging_jars_java17.sh
index 7ceb0a1e0d3c..cb764077a083 100755
--- a/scripts/release/deploy_staging_jars_java17.sh
+++ b/scripts/release/deploy_staging_jars_java17.sh
@@ -51,12 +51,15 @@ declare -a ALL_VERSION_OPTS=(
# For Spark 4.0, Scala 2.13:
# hudi-spark4.0.x_2.13
# hudi-spark4.0-bundle_2.13
-"-T 1C -Djava17 -Djava.version=17 -Dscala-2.13 -Dspark4.0 -pl
hudi-spark-datasource/hudi-spark4.0.x,packaging/hudi-spark-bundle -am"
+# hudi-spark4.0-native-bundle_2.13
+"-T 1C -Djava17 -Djava.version=17 -Dscala-2.13 -Dspark4.0 -pl
hudi-spark-datasource/hudi-spark4.0.x,packaging/hudi-spark-bundle,packaging/hudi-spark-native-bundle
-am"
# For Spark 4.1, Scala 2.13:
# hudi-spark4.1.x_2.13
# hudi-spark4.1-bundle_2.13
-"-T 1C -Djava17 -Djava.version=17 -Dscala-2.13 -Dspark4.1 -pl
hudi-spark-datasource/hudi-spark4.1.x,packaging/hudi-spark-bundle -am"
+# hudi-spark4.1-native-bundle_2.13
+"-T 1C -Djava17 -Djava.version=17 -Dscala-2.13 -Dspark4.1 -pl
hudi-spark-datasource/hudi-spark4.1.x,packaging/hudi-spark-bundle,packaging/hudi-spark-native-bundle
-am"
# For Spark 4.2, Scala 2.13:
+# (no hudi-spark4.2-native-bundle: Comet supports Spark 4.2 only
experimentally and publishes no artifact for it)
# hudi-spark4-common
# hudi-spark4.2.x_2.13
# hudi-spark4.2-bundle_2.13
diff --git a/scripts/release/validate_staged_bundles.sh
b/scripts/release/validate_staged_bundles.sh
index f3f4e3aaaf11..b4114de3ee5b 100755
--- a/scripts/release/validate_staged_bundles.sh
+++ b/scripts/release/validate_staged_bundles.sh
@@ -89,7 +89,9 @@ declare -a bundles=("hudi-aws-bundle" "hudi-azure-bundle"
"hudi-cli-bundle_2.12"
"hudi-flink2.0-bundle" "hudi-flink2.1-bundle" "hudi-flink2.2-bundle"
"hudi-gcp-bundle" "hudi-hadoop-mr-bundle" "hudi-hive-sync-bundle"
"hudi-integ-test-bundle"
"hudi-kafka-connect-bundle" "hudi-metaserver-server-bundle"
"hudi-presto-bundle"
"hudi-spark3.3-bundle_2.12" "hudi-spark3.4-bundle_2.12"
"hudi-spark3.5-bundle_2.12"
-"hudi-spark3.5-bundle_2.13" "hudi-spark4.0-bundle_2.13"
"hudi-spark4.1-bundle_2.13" "hudi-spark4.2-bundle_2.13"
"hudi-timeline-server-bundle"
+"hudi-spark3.5-bundle_2.13" "hudi-spark4.0-bundle_2.13"
"hudi-spark4.1-bundle_2.13" "hudi-spark4.2-bundle_2.13"
+"hudi-spark3.4-native-bundle_2.12" "hudi-spark3.5-native-bundle_2.12"
"hudi-spark3.5-native-bundle_2.13"
+"hudi-spark4.0-native-bundle_2.13" "hudi-spark4.1-native-bundle_2.13"
"hudi-timeline-server-bundle"
"hudi-trino"
"hudi-utilities-bundle_2.12" "hudi-utilities-bundle_2.13"
"hudi-utilities-slim-bundle_2.12" "hudi-utilities-slim-bundle_2.13")
