This is an automated email from the ASF dual-hosted git repository.
yashmayya pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 0e95355afed Fix compatibility regression testing (#18597)
0e95355afed is described below
commit 0e95355afed1166d96a0412497ed3975610fe4ab
Author: Xiang Fu <[email protected]>
AuthorDate: Thu May 28 03:54:55 2026 +0800
Fix compatibility regression testing (#18597)
---
.github/workflows/pinot_compatibility_checks.yml | 16 ++++---
.github/workflows/pinot_compatibility_tests.yml | 7 +++-
...ulti_stage_query_engine_compatibility_tests.yml | 7 +++-
.../scripts/.pinot_compatibility_verifier.sh | 25 +++++++----
compatibility-verifier/checkoutAndBuild.sh | 49 +++++++++++++++++++---
5 files changed, 81 insertions(+), 23 deletions(-)
diff --git a/.github/workflows/pinot_compatibility_checks.yml
b/.github/workflows/pinot_compatibility_checks.yml
index 3b66264d8c9..75c1fa23b16 100644
--- a/.github/workflows/pinot_compatibility_checks.yml
+++ b/.github/workflows/pinot_compatibility_checks.yml
@@ -215,15 +215,17 @@ jobs:
WORKING_DIR: /tmp/compatibility-verifier
TEST_SUITE: ${{ matrix.test_suite }}
MAVEN_OPTS: >
- -Xmx2G -DskipShade -DfailIfNoTests=false
-Dmaven.wagon.httpconnectionManager.ttlSeconds=25
- -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false
-Dmaven.wagon.http.pool=false
- -B -ntp
+ -Xmx2G
-XX:+IgnoreUnrecognizedVMOptions
--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+ PINOT_MAVEN_OPTS: >
+ -DskipShade -DfailIfNoTests=false
-Dmaven.wagon.httpconnectionManager.ttlSeconds=25
+ -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false
-Dmaven.wagon.http.pool=false
+ -B -ntp
timeout-minutes: 120
run: |
.github/workflows/scripts/.pinot_compatibility_verifier.sh
@@ -272,15 +274,17 @@ jobs:
WORKING_DIR: /tmp/multi-stage-compatibility-verifier
TEST_SUITE: ${{ matrix.test_suite }}
MAVEN_OPTS: >
- -Xmx2G -DskipShade -DfailIfNoTests=false
-Dmaven.wagon.httpconnectionManager.ttlSeconds=25
- -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false
-Dmaven.wagon.http.pool=false
- -B -ntp
+ -Xmx2G
-XX:+IgnoreUnrecognizedVMOptions
--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+ PINOT_MAVEN_OPTS: >
+ -DskipShade -DfailIfNoTests=false
-Dmaven.wagon.httpconnectionManager.ttlSeconds=25
+ -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false
-Dmaven.wagon.http.pool=false
+ -B -ntp
timeout-minutes: 120
run: |
.github/workflows/scripts/.pinot_compatibility_verifier.sh
diff --git a/.github/workflows/pinot_compatibility_tests.yml
b/.github/workflows/pinot_compatibility_tests.yml
index f102ef311d4..6d373357c50 100644
--- a/.github/workflows/pinot_compatibility_tests.yml
+++ b/.github/workflows/pinot_compatibility_tests.yml
@@ -61,14 +61,17 @@ jobs:
WORKING_DIR: /tmp/compatibility-verifier
TEST_SUITE: ${{ matrix.test_suite }}
MAVEN_OPTS: >
- -Xmx2G -DskipShade -DfailIfNoTests=false
-Dmaven.wagon.httpconnectionManager.ttlSeconds=25
- -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false
-Dmaven.wagon.http.pool=false
+ -Xmx2G
-XX:+IgnoreUnrecognizedVMOptions
--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+ PINOT_MAVEN_OPTS: >
+ -DskipShade -DfailIfNoTests=false
-Dmaven.wagon.httpconnectionManager.ttlSeconds=25
+ -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false
-Dmaven.wagon.http.pool=false
+ -B -ntp
timeout-minutes: 120
run: |
.github/workflows/scripts/.pinot_compatibility_verifier.sh
diff --git
a/.github/workflows/pinot_multi_stage_query_engine_compatibility_tests.yml
b/.github/workflows/pinot_multi_stage_query_engine_compatibility_tests.yml
index 0b0a6ef3b26..348b540aa2a 100644
--- a/.github/workflows/pinot_multi_stage_query_engine_compatibility_tests.yml
+++ b/.github/workflows/pinot_multi_stage_query_engine_compatibility_tests.yml
@@ -61,14 +61,17 @@ jobs:
WORKING_DIR: /tmp/multi-stage-compatibility-verifier
TEST_SUITE: ${{ matrix.test_suite }}
MAVEN_OPTS: >
- -Xmx2G -DskipShade -DfailIfNoTests=false
-Dmaven.wagon.httpconnectionManager.ttlSeconds=25
- -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false
-Dmaven.wagon.http.pool=false
+ -Xmx2G
-XX:+IgnoreUnrecognizedVMOptions
--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+ PINOT_MAVEN_OPTS: >
+ -DskipShade -DfailIfNoTests=false
-Dmaven.wagon.httpconnectionManager.ttlSeconds=25
+ -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false
-Dmaven.wagon.http.pool=false
+ -B -ntp
timeout-minutes: 120
run: |
.github/workflows/scripts/.pinot_compatibility_verifier.sh
diff --git a/.github/workflows/scripts/.pinot_compatibility_verifier.sh
b/.github/workflows/scripts/.pinot_compatibility_verifier.sh
index 93789ce67ca..7aca36dcd6f 100755
--- a/.github/workflows/scripts/.pinot_compatibility_verifier.sh
+++ b/.github/workflows/scripts/.pinot_compatibility_verifier.sh
@@ -60,18 +60,29 @@ echo " </servers>">> ${SETTINGS_FILE}
echo "</settings>">> ${SETTINGS_FILE}
# PINOT_MAVEN_OPTS is used to provide additional maven options to the
checkoutAndBuild.sh command
-export PINOT_MAVEN_OPTS="-s $(pwd)/${SETTINGS_FILE}"
+export PINOT_MAVEN_OPTS="${PINOT_MAVEN_OPTS:-} -s $(pwd)/${SETTINGS_FILE}"
# Compare commit hash for compatibility verification
-git fetch --all
-NEW_COMMIT_HASH=`git log -1 --pretty=format:'%h' HEAD`
+git fetch --all --tags
+function commitHash() {
+ local commit=$1
+ git log -1 --pretty=format:'%h' "${commit}" 2>/dev/null \
+ || git log -1 --pretty=format:'%h' "origin/${commit}" 2>/dev/null \
+ || git log -1 --pretty=format:'%h' "refs/tags/${commit}" 2>/dev/null
+}
+
+NEW_COMMIT_HASH=$(commitHash HEAD)
if [ ! -z "${NEW_COMMIT}" ]; then
- NEW_COMMIT_HASH=`git log -1 --pretty=format:'%h' ${NEW_COMMIT}`
+ NEW_COMMIT_HASH=$(commitHash "${NEW_COMMIT}")
fi
-OLD_COMMIT_HASH=`git log -1 --pretty=format:'%h' ${OLD_COMMIT}`
-if [ $? -ne 0 ]; then
+OLD_COMMIT_HASH=$(commitHash "${OLD_COMMIT}")
+if [ -z "${OLD_COMMIT_HASH}" ]; then
echo "Failed to get commit hash for commit: \"${OLD_COMMIT}\""
- OLD_COMMIT_HASH=`git log -1 --pretty=format:'%h' origin/${OLD_COMMIT}`
+ exit 1
+fi
+if [ -z "${NEW_COMMIT_HASH}" ]; then
+ echo "Failed to get commit hash for commit: \"${NEW_COMMIT:-HEAD}\""
+ exit 1
fi
if [ "${NEW_COMMIT_HASH}" == "${OLD_COMMIT_HASH}" ]; then
echo "No changes between old commit: \"${OLD_COMMIT}\" and new commit:
\"${NEW_COMMIT}\""
diff --git a/compatibility-verifier/checkoutAndBuild.sh
b/compatibility-verifier/checkoutAndBuild.sh
index 977bfccd6e5..ba22c785f79 100755
--- a/compatibility-verifier/checkoutAndBuild.sh
+++ b/compatibility-verifier/checkoutAndBuild.sh
@@ -70,49 +70,86 @@ function checkOut() {
# dedicated maven cache for these builds, and remove the cache after build is
# completed.
# If buildId is less than 0, then the mvn version is not changed in pom files.
+function installPinotBom() {
+ local outFile=$1
+ local repoDir=$2
+ local repoOption=$3
+ local maxRetry=$4
+
+ if [ ! -f "pinot-bom/pom.xml" ]; then
+ return
+ fi
+
+ for i in $(seq 1 $maxRetry); do
+ mvn -U -f pinot-bom/pom.xml install -DskipTests -q -B ${repoOption}
${PINOT_MAVEN_OPTS} 1>>${outFile} 2>&1
+ if [ $? -eq 0 ]; then break; fi
+ if [ $i -eq $maxRetry ]; then exit 1; fi
+ echo ""
+ echo "Pinot BOM install failed, see last 200 lines of output below."
+ tail -200 ${outFile}
+ echo "Retrying after 30 seconds..."
+ find "${repoDir}" -name "*.lastUpdated" -delete 2>/dev/null || true
+ sleep 30
+ done
+}
+
function build() {
local outFile=$1
local buildTests=$2
local buildId=$3
local buildCompatibilityVerifier=$4
- local repoOption=""
local versionOption="-Djdk.version=21"
local maxRetry=5
+ local repoId=${buildId}
mkdir -p ${MVN_CACHE_DIR}
mkdir -p ${mvnCache}
+ if [ ${repoId} -le 0 ]; then
+ repoId="current"
+ fi
+ local repoDir="${mvnCache}/${repoId}"
+ mkdir -p "${repoDir}"
+ local repoOption="-Dmaven.repo.local=${repoDir}"
+
+ installPinotBom "${outFile}" "${repoDir}" "${repoOption}" "${maxRetry}"
if [ ${buildId} -gt 0 ]; then
# Build it in a different env under different version so that maven cache
does
# not collide
local pomVersion=$(grep -E "<version>(.*)-SNAPSHOT</version>" pom.xml |
cut -d'>' -f2 | cut -d'<' -f1 | cut -d'-' -f1)
- mvn versions:set -DnewVersion="${pomVersion}-compat-${buildId}" -q -B
1>${outFile} 2>&1
- mvn versions:commit -q -B 1>${outFile} 2>&1
- repoOption="-Dmaven.repo.local=${mvnCache}/${buildId}"
+ mvn -U versions:set -DnewVersion="${pomVersion}-compat-${buildId}"
-DgenerateBackupPoms=false \
+ -q -B ${repoOption} ${PINOT_MAVEN_OPTS} 1>${outFile} 2>&1 || exit 1
+ if [ -f "pinot-bom/pom.xml" ]; then
+ mvn -U -f pinot-bom/pom.xml versions:set
-DnewVersion="${pomVersion}-compat-${buildId}" \
+ -DgenerateBackupPoms=false -q -B ${repoOption} ${PINOT_MAVEN_OPTS}
1>${outFile} 2>&1 || exit 1
+ fi
+ installPinotBom "${outFile}" "${repoDir}" "${repoOption}" "${maxRetry}"
fi
buildComponents=":pinot-tools"
if [ $buildCompatibilityVerifier -gt 0 ]; then
buildComponents=":pinot-tools,:pinot-compatibility-verifier"
fi
for i in $(seq 1 $maxRetry); do
- mvn clean package -am -pl ${buildComponents} -DskipTests -T1C
${versionOption} ${repoOption} ${PINOT_MAVEN_OPTS} 1>${outFile} 2>&1
+ mvn -U clean package -am -pl ${buildComponents} -DskipTests -T1C
${versionOption} ${repoOption} ${PINOT_MAVEN_OPTS} 1>${outFile} 2>&1
if [ $? -eq 0 ]; then break; fi
if [ $i -eq $maxRetry ]; then exit 1; fi
echo ""
echo "Build failed, see last 1000 lines of output below."
tail -1000 ${outFile}
echo "Retrying after 30 seconds..."
+ find "${repoDir}" -name "*.lastUpdated" -delete 2>/dev/null || true
sleep 30
done
if [ $buildTests -eq 1 ]; then
for i in $(seq 1 $maxRetry); do
- mvn package -am -pl :pinot-integration-tests -DskipTests -T1C
${versionOption} ${repoOption} ${PINOT_MAVEN_OPTS} 1>>${outFile} 2>&1
+ mvn -U package -am -pl :pinot-integration-tests -DskipTests -T1C
${versionOption} ${repoOption} ${PINOT_MAVEN_OPTS} 1>>${outFile} 2>&1
if [ $? -eq 0 ]; then break; fi
if [ $i -eq $maxRetry ]; then exit 1; fi
echo ""
echo "Build failed, see last 500 lines of output below."
tail -500 ${outFile}
echo "Retrying after 30 seconds..."
+ find "${repoDir}" -name "*.lastUpdated" -delete 2>/dev/null || true
sleep 30
done
fi
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
