This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 58c9e64e58a0 chore(spark): bump Spark 3.5 to 3.5.9 and fast-path the
CI download (#19884)
58c9e64e58a0 is described below
commit 58c9e64e58a040d5c608b0893c2709e8c231534b
Author: voonhous <[email protected]>
AuthorDate: Thu Sep 10 12:30:30 2026 +0800
chore(spark): bump Spark 3.5 to 3.5.9 and fast-path the CI download (#19884)
spark35.version moves 3.5.5 -> 3.5.9 and the integration-tests Spark
archive pin moves 3.5.3 -> 3.5.9 to match, along with the five
spark-avro spark-submit examples. The old split compiled against 3.5.5
and ran the ITs on 3.5.3, the direction that can hit a missing API.
The stale pin is also why the download was slow: 3.5.3 has rolled
off the ASF CDN, so the job pulled the 382MB tarball from
archive.apache.org, a single un-CDN'd origin measured at 42s to 1h08m
for the same file (#19883). 3.5.9 is the current 3.5 release and is
served by dlcdn.apache.org, so the job now tries the CDN first and
falls back to the archive for pins that have rolled off it. Plain
--retry rather than --retry-all-errors, so a CDN 404 falls through
at once instead of sleeping through five retries.
curl gains -f so an error page is a failure rather than a corrupt
tarball, and --speed-limit 1000 --speed-time 120 to abort a dead
connection. The floor sits far below the archive's real throughput
on purpose: --retry truncates the output back to byte 0, so a floor
near it would re-download 382MB per abort. -C - is not used; the
file never exists when curl starts, so it could never resume. tar
drops -v, which printed 4000 filenames into the job log.
The three Spark sources Hudi copies from are byte-identical between
v3.5.5 and v3.5.9, and every in-tree Spark version gate is
minor-level, so the patch bump crosses none of them.
---
.github/workflows/bot.yml | 30 +++++++++++++++++++---
.../testsuite/HoodieContinuousTestSuiteWriter.java | 2 +-
.../testsuite/HoodieMultiWriterTestSuiteJob.java | 2 +-
.../SparkDataSourceContinuousIngestTool.java | 2 +-
.../hudi/utilities/HoodieDropPartitionsTool.java | 4 +--
pom.xml | 2 +-
6 files changed, 32 insertions(+), 10 deletions(-)
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index c0275fe18590..a8cc61a87c63 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -1468,7 +1468,7 @@ jobs:
include:
- sparkProfile: 'spark3.5'
flinkProfile: 'flink2.2'
- sparkArchive: 'spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz'
+ sparkArchive: 'spark-3.5.9/spark-3.5.9-bin-hadoop3.tgz'
steps:
- if: needs.changes.outputs.relevant == 'true'
uses: actions/checkout@v5
@@ -1516,9 +1516,31 @@ jobs:
SPARK_ARCHIVE: ${{ matrix.sparkArchive }}
SCALA_PROFILE: '-Dscala-2.12 -Dscala.binary.version=2.12'
run: |
- echo "Downloading $SPARK_ARCHIVE"
- curl --retry 5 https://archive.apache.org/dist/spark/$SPARK_ARCHIVE
--create-dirs -o $GITHUB_WORKSPACE/$SPARK_ARCHIVE
- tar -xvf $GITHUB_WORKSPACE/$SPARK_ARCHIVE -C $GITHUB_WORKSPACE/
+ # dlcdn only carries the current release of each line; fall back to
+ # the archive for older pins (#19883). Plain --retry, not
+ # --retry-all-errors, so a 404 on the CDN falls through immediately.
+ # --speed-limit is a dead-connection detector, not a slowness one:
+ # --retry truncates the output back to byte 0, so a floor set near
+ # the archive's real throughput re-downloads 382MB per abort.
+ DEST="$GITHUB_WORKSPACE/$SPARK_ARCHIVE"
+ downloaded=false
+ for base in https://dlcdn.apache.org/spark
https://archive.apache.org/dist/spark; do
+ echo "Downloading $SPARK_ARCHIVE from $base"
+ if curl -fL --create-dirs -o "$DEST" \
+ --retry 5 --retry-delay 10 \
+ --connect-timeout 30 --speed-limit 1000 --speed-time 120 \
+ "$base/$SPARK_ARCHIVE"; then
+ downloaded=true
+ break
+ fi
+ echo "$base did not serve $SPARK_ARCHIVE"
+ rm -f "$DEST"
+ done
+ if [ "$downloaded" != true ]; then
+ echo "ERROR: could not download $SPARK_ARCHIVE from any source"
+ exit 1
+ fi
+ tar -xf "$DEST" -C $GITHUB_WORKSPACE/
mkdir /tmp/spark-events/
SPARK_ARCHIVE_BASENAME=$(basename $SPARK_ARCHIVE)
export SPARK_HOME=$GITHUB_WORKSPACE/${SPARK_ARCHIVE_BASENAME%.*}
diff --git
a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieContinuousTestSuiteWriter.java
b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieContinuousTestSuiteWriter.java
index 535e4049f11e..62f970fefc70 100644
---
a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieContinuousTestSuiteWriter.java
+++
b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieContinuousTestSuiteWriter.java
@@ -41,7 +41,7 @@ import java.util.Properties;
* Test suite Writer that assists in testing async table operations with
Deltastreamer continuous mode.
*
* Sample command
- * ./bin/spark-submit --packages org.apache.spark:spark-avro_2.12:3.5.5 \
+ * ./bin/spark-submit --packages org.apache.spark:spark-avro_2.12:3.5.9 \
* --conf spark.task.cpus=1 --conf spark.executor.cores=1 \
* --conf spark.task.maxFailures=100 \
* --conf spark.memory.fraction=0.4 \
diff --git
a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieMultiWriterTestSuiteJob.java
b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieMultiWriterTestSuiteJob.java
index 0a5485345f55..a6f37013d9fe 100644
---
a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieMultiWriterTestSuiteJob.java
+++
b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieMultiWriterTestSuiteJob.java
@@ -53,7 +53,7 @@ import java.util.concurrent.atomic.AtomicInteger;
*
* Example command
* spark-submit
- * --packages org.apache.spark:spark-avro_2.12:3.5.5
+ * --packages org.apache.spark:spark-avro_2.12:3.5.9
* --conf spark.task.cpus=3
* --conf spark.executor.cores=3
* --conf spark.task.maxFailures=100
diff --git
a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/SparkDataSourceContinuousIngestTool.java
b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/SparkDataSourceContinuousIngestTool.java
index 6d91fa88ca9c..9d5c2331256b 100644
---
a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/SparkDataSourceContinuousIngestTool.java
+++
b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/SparkDataSourceContinuousIngestTool.java
@@ -43,7 +43,7 @@ import java.util.Map;
/**
* Sample command
*
- * ./bin/spark-submit --packages org.apache.spark:spark-avro_2.12:3.5.5
--driver-memory 4g --executor-memory 4g \
+ * ./bin/spark-submit --packages org.apache.spark:spark-avro_2.12:3.5.9
--driver-memory 4g --executor-memory 4g \
* --conf spark.serializer=org.apache.spark.serializer.KryoSerializer --conf
spark.sql.catalogImplementation=hive \
* --class org.apache.hudi.integ.testsuite.SparkDSContinuousIngestTool \
*
${HUDI_ROOT_DIR}/packaging/hudi-integ-test-bundle/target/hudi-integ-test-bundle-0.11.0-SNAPSHOT.jar
\
diff --git
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDropPartitionsTool.java
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDropPartitionsTool.java
index e1e28c4cf7c8..9b6a89c8dae9 100644
---
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDropPartitionsTool.java
+++
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieDropPartitionsTool.java
@@ -65,7 +65,7 @@ import scala.Tuple2;
* ```
* spark-submit \
* --class org.apache.hudi.utilities.HoodieDropPartitionsTool \
- * --packages org.apache.spark:spark-avro_2.12:3.5.5 \
+ * --packages org.apache.spark:spark-avro_2.12:3.5.9 \
* --master local[*]
* --driver-memory 1g \
* --executor-memory 1g \
@@ -87,7 +87,7 @@ import scala.Tuple2;
* ```
* spark-submit \
* --class org.apache.hudi.utilities.HoodieDropPartitionsTool \
- * --packages org.apache.spark:spark-avro_2.12:3.5.5 \
+ * --packages org.apache.spark:spark-avro_2.12:3.5.9 \
* --master local[*]
* --driver-memory 1g \
* --executor-memory 1g \
diff --git a/pom.xml b/pom.xml
index 671d625e6ff5..11dc55c3355f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -188,7 +188,7 @@
<rocksdbjni.version>7.5.3</rocksdbjni.version>
<spark33.version>3.3.4</spark33.version>
<spark34.version>3.4.3</spark34.version>
- <spark35.version>3.5.5</spark35.version>
+ <spark35.version>3.5.9</spark35.version>
<spark40.version>4.0.2</spark40.version>
<spark41.version>4.1.1</spark41.version>
<spark42.version>4.2.0</spark42.version>
