This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 971634bb3e4b chore: Enhance validate_staged_bundles.sh to validate
bundles before closing the staging repo (#18749)
971634bb3e4b is described below
commit 971634bb3e4b3033cf8ed56ca6886bf6c3c3bd05
Author: Y Ethan Guo <[email protected]>
AuthorDate: Fri May 15 12:34:47 2026 -0700
chore: Enhance validate_staged_bundles.sh to validate bundles before
closing the staging repo (#18749)
* chore: Enhance the script of validating staged bundles
* Update release guide to run validate_staged_bundles.sh --auth before
closing the staging repo
---
release/release_guide.md | 15 ++++-
scripts/release/validate_staged_bundles.sh | 103 ++++++++++++++++++++++++-----
2 files changed, 98 insertions(+), 20 deletions(-)
diff --git a/release/release_guide.md b/release/release_guide.md
index 4978f116e685..49c5c2eefe50 100644
--- a/release/release_guide.md
+++ b/release/release_guide.md
@@ -433,16 +433,25 @@ Set up a few environment variables to simplify Maven
commands that follow. This
hudi-spark3-bundle-2.12/2.13, hudi-spark-2.12/2.13,
hudi-spark3-2.12/2.13, hudi-utilities-bundle_2.12/2.13 and
hudi-utilities_2.12/2.13.
> With 0.10.1, we had 4 bundles. spark2 with scala11, spark2 with
scala12, spark3.0.x bundles and spark3.1.x bundles. Ensure each spark bundle
reflects the version correctly. hudi-spark3.1.2-bundle_2.12-0.10.1.jar and
hudi-spark3.0.3-bundle_2.12-0.10.1.jar are the respective bundle names for
spark3 bundles.
- 8. Once you have ensured everything is good and validation of step 7
succeeds, you can close the staging repo. Until
+ 8. Before closing the staging repo, run the validate script against the
private staging URL to programmatically
+ confirm all expected bundle artifacts are present. The `--auth` flag
reads Nexus credentials from
+ `~/.m2/settings.xml` (server id `apache.releases.https`), which is
required because the public URL is not
+ populated until the repo is closed.
+ ```shell
+ ./scripts/release/validate_staged_bundles.sh --auth
orgapachehudi-<stage_repo_number> ${RELEASE_VERSION}-rc${RC_NUM} 2>&1 | tee -a
/tmp/validate_staged_bundles_output.txt
+ ```
+ If the script reports any missing artifacts, re-run the relevant
`deploy_staging_jars*.sh` step rather than
+ closing the repo, since once closed it cannot be re-deployed to.
+ 9. Once you have ensured everything is good and validation of step 8
succeeds, you can close the staging repo. Until
you close, you can re-run deploying to staging multiple times. But once
closed, it will create a new staging repo.
So ensure you close this, so that the next RC (if need be) is on a new
repo. So, once everything is good, close
the staging repository on Apache Nexus. When prompted for a description,
enter
> Apache Hudi, version `${RELEASE_VERSION}`, release candidate
`${RC_NUM}`.
- 9. After closing, run the script to validate the staged bundles again:
+ 10. After closing, run the script to validate the staged bundles again
against the public URL:
```shell
./scripts/release/validate_staged_bundles.sh
orgapachehudi-<stage_repo_number> ${RELEASE_VERSION}-rc${RC_NUM} 2>&1 | tee -a
/tmp/validate_staged_bundles_output.txt
```
- 10. Run the release candidate bundle validation in GitHub Action by
following the instruction in
+ 11. Run the release candidate bundle validation in GitHub Action by
following the instruction in
["Running Bundle Validation on a Release
Candidate"](../packaging/bundle-validation/README.md#running-bundle-validation-on-a-release-candidate).
## Checklist to proceed to the next step
diff --git a/scripts/release/validate_staged_bundles.sh
b/scripts/release/validate_staged_bundles.sh
index 06763dc27e37..36e25ca6aea5 100755
--- a/scripts/release/validate_staged_bundles.sh
+++ b/scripts/release/validate_staged_bundles.sh
@@ -23,41 +23,92 @@
set -o errexit
set -o nounset
+USE_AUTH=false
+if [[ "${1:-}" == "--auth" ]]; then
+ USE_AUTH=true
+ shift
+fi
+
+if [[ $# -ne 2 ]]; then
+ echo "Usage: $(basename "$0") [--auth] <repo-id> <version>"
+ echo ""
+ echo " Default: uses public URL (repo must be closed)"
+ echo " --auth: uses private staging URL with ~/.m2/settings.xml
credentials"
+ echo ""
+ echo "Examples:"
+ echo " $(basename "$0") orgapachehudi-1176 1.2.0-rc1"
+ echo " $(basename "$0") --auth orgapachehudi-1177 1.2.0-rc1"
+ exit 1
+fi
+
REPO=$1
VERSION=$2
-STAGING_REPO="https://repository.apache.org/content/repositories/${REPO}/org/apache/hudi";
+if [[ "$USE_AUTH" == true ]]; then
+
STAGING_REPO="https://repository.apache.org/service/local/repositories/${REPO}/content/org/apache/hudi";
+
+ SETTINGS_XML="$HOME/.m2/settings.xml"
+ if [[ ! -f "$SETTINGS_XML" ]]; then
+ echo "ERROR: $SETTINGS_XML not found"
+ exit 1
+ fi
+
+ if command -v xmllint &>/dev/null; then
+ NEXUS_USER=$(xmllint --xpath \
+ "string(//server[id='apache.releases.https']/username)" "$SETTINGS_XML")
+ NEXUS_PASS=$(xmllint --xpath \
+ "string(//server[id='apache.releases.https']/password)" "$SETTINGS_XML")
+ else
+ NEXUS_USER=$(sed -n '/<server>/,/<\/server>/{
/<id>apache.releases.https<\/id>/,/<\/server>/{
s/.*<username>\(.*\)<\/username>.*/\1/p; }; }' "$SETTINGS_XML" | head -1 |
xargs)
+ NEXUS_PASS=$(sed -n '/<server>/,/<\/server>/{
/<id>apache.releases.https<\/id>/,/<\/server>/{
s/.*<password>\(.*\)<\/password>.*/\1/p; }; }' "$SETTINGS_XML" | head -1 |
xargs)
+ fi
+
+ if [[ -z "$NEXUS_USER" || -z "$NEXUS_PASS" ]]; then
+ echo "ERROR: Could not extract credentials for 'apache.releases.https'
from $SETTINGS_XML"
+ exit 1
+ fi
+
+ export NEXUS_USER NEXUS_PASS
+ CURL_AUTH="-u ${NEXUS_USER}:${NEXUS_PASS}"
+ echo "==> Using private staging URL with authentication (user: $NEXUS_USER)"
+else
+
STAGING_REPO="https://repository.apache.org/content/repositories/${REPO}/org/apache/hudi";
+ CURL_AUTH=""
+ echo "==> Using public URL (no authentication)"
+fi
+
+export CURL_AUTH
declare -a extensions=("-javadoc.jar" "-javadoc.jar.asc" "-javadoc.jar.md5"
"-javadoc.jar.sha1" "-sources.jar"
"-sources.jar.asc" "-sources.jar.md5" "-sources.jar.sha1" ".jar" ".jar.asc"
".jar.md5" ".jar.sha1" ".pom" ".pom.asc"
".pom.md5" ".pom.sha1")
-declare -a bundles=("hudi-aws-bundle" "hudi-cli-bundle_2.12"
"hudi-cli-bundle_2.13" "hudi-datahub-sync-bundle"
+declare -a bundles=("hudi-aws-bundle" "hudi-azure-bundle"
"hudi-cli-bundle_2.12" "hudi-cli-bundle_2.13" "hudi-datahub-sync-bundle"
"hudi-flink1.17-bundle" "hudi-flink1.18-bundle" "hudi-flink1.19-bundle"
"hudi-flink1.20-bundle"
-"hudi-flink2.0-bundle" "hudi-gcp-bundle" "hudi-hadoop-mr-bundle"
"hudi-hive-sync-bundle" "hudi-integ-test-bundle"
+"hudi-flink2.0-bundle" "hudi-flink2.1-bundle" "hudi-gcp-bundle"
"hudi-hadoop-mr-bundle" "hudi-hive-sync-bundle" "hudi-integ-test-bundle"
"hudi-kafka-connect-bundle" "hudi-metaserver-server-bundle"
"hudi-presto-bundle"
"hudi-spark3.3-bundle_2.12" "hudi-spark3.4-bundle_2.12"
"hudi-spark3.5-bundle_2.12"
"hudi-spark3.5-bundle_2.13" "hudi-spark4.0-bundle_2.13"
"hudi-spark4.1-bundle_2.13" "hudi-timeline-server-bundle" "hudi-trino-bundle"
"hudi-utilities-bundle_2.12" "hudi-utilities-bundle_2.13"
"hudi-utilities-slim-bundle_2.12" "hudi-utilities-slim-bundle_2.13")
+MISSING_FILE=$(mktemp)
+export MISSING_FILE
+
curl_with_url() {
local url="$1"
- if curl -s -o /dev/null --head --fail "$url"; then
- echo "Artifact exists: $url"
+ if curl -s -o /dev/null --head --fail $CURL_AUTH "$url"; then
+ echo " OK: $url"
else
- echo "Artifact missing: $url"
- exit 1
+ echo " MISSING: $url"
+ echo "$url" >> "$MISSING_FILE"
fi
}
export -f curl_with_url
-NOW=$(date +%s)
-TMP_DIR_FOR_BUNDLES=/tmp/${NOW}
-mkdir "$TMP_DIR_FOR_BUNDLES"
-
ALL_URLS=""
+TOTAL=0
for bundle in "${bundles[@]}"
do
@@ -65,15 +116,33 @@ do
do
url=${STAGING_REPO}/$bundle/${VERSION}/$bundle-${VERSION}$extension
ALL_URLS+="$url\n"
+ TOTAL=$((TOTAL + 1))
done
done
-echo "-- All bundles to check:"
-echo -e "$ALL_URLS"
+echo "-- Checking $TOTAL artifacts ..."
+echo ""
+echo -e "$ALL_URLS" | xargs -n 1 -P 16 -I {} bash -c 'curl_with_url "{}"'
-if echo -e "$ALL_URLS" | xargs -n 1 -P 16 -I {} bash -c 'curl_with_url "{}"';
then
- echo "All artifacts exist. Validation succeeds."
-else
- echo "Some artifact(s) missing."
+MISSING_COUNT=0
+if [[ -s "$MISSING_FILE" ]]; then
+ MISSING_COUNT=$(wc -l < "$MISSING_FILE" | xargs)
+fi
+
+echo ""
+echo "==========================================="
+echo "Total: $TOTAL | Present: $((TOTAL - MISSING_COUNT)) | Missing:
$MISSING_COUNT"
+echo "==========================================="
+
+if [[ "$MISSING_COUNT" -gt 0 ]]; then
+ echo ""
+ echo "Missing artifacts:"
+ sort "$MISSING_FILE" | while read -r url; do
+ echo " $url"
+ done
+ rm -f "$MISSING_FILE"
exit 1
+else
+ rm -f "$MISSING_FILE"
+ echo "All artifacts exist. Validation succeeds."
fi
