This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 24e47ffebe [VL][CI] Add a format checker for scala code to enable
early failure reporting (#10747)
24e47ffebe is described below
commit 24e47ffebe0c07646147ea8faec583d2d510a6e2
Author: PHILO-HE <[email protected]>
AuthorDate: Fri Sep 19 23:24:30 2025 +0800
[VL][CI] Add a format checker for scala code to enable early failure
reporting (#10747)
---
.github/workflows/check_license.yml | 5 +--
.../workflows/{code_style.yml => code_format.yml} | 13 ++++---
.github/workflows/labeler.yml | 3 +-
.../{check_license.yml => scala_code_format.yml} | 44 +++++++++++++++++-----
.github/workflows/util/setup_helper.sh | 7 +++-
.github/workflows/velox_backend_arm.yml | 2 +-
...ced_features.yml => velox_backend_enhanced.yml} | 4 +-
.github/workflows/velox_backend_x86.yml | 7 +---
.github/workflows/velox_weekly.yml | 17 ++-------
dev/{formatcppcode.sh => format-cpp-code.sh} | 0
.../setup_helper.sh => dev/format-scala-code.sh | 34 ++++++++---------
11 files changed, 76 insertions(+), 60 deletions(-)
diff --git a/.github/workflows/check_license.yml
b/.github/workflows/check_license.yml
index 338397dbd6..6664b50cf5 100644
--- a/.github/workflows/check_license.yml
+++ b/.github/workflows/check_license.yml
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-name: license header check
+name: License Header Check
on:
pull_request
concurrency:
@@ -21,8 +21,7 @@ concurrency:
cancel-in-progress: true
jobs:
- license-check:
- name: License Header Check
+ check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
diff --git a/.github/workflows/code_style.yml
b/.github/workflows/code_format.yml
similarity index 92%
rename from .github/workflows/code_style.yml
rename to .github/workflows/code_format.yml
index c6ba7c647c..ddabff6e9f 100644
--- a/.github/workflows/code_style.yml
+++ b/.github/workflows/code_format.yml
@@ -13,20 +13,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-name: Code style checks
+name: Code Format Check
on:
pull_request:
paths:
- - '.github/workflows/code_style.yml'
+ - '.github/workflows/code_format.yml'
+ - '.github/workflows/**/*.py'
+ - 'tools/**/*.py'
+ - 'dev/**/*.py'
- 'cpp/**'
+
concurrency:
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
cancel-in-progress: true
jobs:
- Code-format-check:
- name: Code Format Check(cpp)
+ format-check:
runs-on: ubuntu-latest
container: ubuntu:22.04
steps:
@@ -40,7 +43,7 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- - name: Check CPP format
+ - name: Check
run: |
git config --global --add safe.directory $GITHUB_WORKSPACE
cd $GITHUB_WORKSPACE/
diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml
index d2ddaec8f8..e18670d16e 100644
--- a/.github/workflows/labeler.yml
+++ b/.github/workflows/labeler.yml
@@ -13,12 +13,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-name: "Label pull requests"
+name: Label Pull Requests
on: pull_request_target
jobs:
label:
- name: Label pull requests
runs-on: ubuntu-latest
permissions:
contents: read
diff --git a/.github/workflows/check_license.yml
b/.github/workflows/scala_code_format.yml
similarity index 54%
copy from .github/workflows/check_license.yml
copy to .github/workflows/scala_code_format.yml
index 338397dbd6..1d72be94fa 100644
--- a/.github/workflows/check_license.yml
+++ b/.github/workflows/scala_code_format.yml
@@ -13,22 +13,46 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-name: license header check
+name: Code Format Check
on:
- pull_request
+ pull_request:
+ paths:
+ - '.github/workflows/scala_code_format.yml'
+ - 'backends-velox/**'
+ - 'gluten-uniffle/**'
+ - 'gluten-celeborn/**'
+ - 'gluten-ras/**'
+ - 'gluten-core/**'
+ - 'gluten-substrait/**'
+ - 'gluten-arrow/**'
+ - 'gluten-delta/**'
+ - 'gluten-iceberg/**'
+ - 'gluten-hudi/**'
+ - 'gluten-paimon/**'
+ - 'gluten-ut/**'
+ - 'shims/**'
+ - 'tools/gluten-it/**'
+
concurrency:
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
cancel-in-progress: true
+env:
+ SETUP: 'bash .github/workflows/util/setup_helper.sh'
+
jobs:
- license-check:
- name: License Header Check
+
+ scala-format-check:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v3
- - name: Check License Header
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ - name: Check
+ run: |
+ ./dev/format-scala-code.sh check
+ - name: Fix
+ if: failure()
run: |
- git fetch --recurse-submodules=no origin main
${{github.event.pull_request.base.sha}}
- pip install regex
- cd $GITHUB_WORKSPACE/
- ./.github/workflows/util/check.sh
${{github.event.pull_request.base.sha}}
+ ./dev/format-scala-code.sh
+ git show
diff --git a/.github/workflows/util/setup_helper.sh
b/.github/workflows/util/setup_helper.sh
index 86bb43c3d2..4679d1a8c1 100644
--- a/.github/workflows/util/setup_helper.sh
+++ b/.github/workflows/util/setup_helper.sh
@@ -26,7 +26,12 @@ function install_maven {
wget -nv -O ${local_binary} ${url}
tar -xvf ${local_binary} && mv apache-maven-${maven_version} /usr/lib/maven
)
- echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
+ export PATH=/usr/lib/maven/bin:$PATH
+ if [ -n "$GITHUB_ENV" ]; then
+ echo "PATH=/usr/lib/maven/bin:$PATH" >> $GITHUB_ENV
+ else
+ echo "Warning: GITHUB_ENV is not set. Skipping environment variable
export."
+ fi
}
for cmd in "$@"
diff --git a/.github/workflows/velox_backend_arm.yml
b/.github/workflows/velox_backend_arm.yml
index 9ae8278097..5608097f97 100644
--- a/.github/workflows/velox_backend_arm.yml
+++ b/.github/workflows/velox_backend_arm.yml
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-name: Velox backend Github Runner (ARM)
+name: Velox Backend (ARM)
on:
pull_request:
diff --git a/.github/workflows/velox_backend_enhanced_features.yml
b/.github/workflows/velox_backend_enhanced.yml
similarity index 98%
rename from .github/workflows/velox_backend_enhanced_features.yml
rename to .github/workflows/velox_backend_enhanced.yml
index 5a000db6eb..554d13aadd 100644
--- a/.github/workflows/velox_backend_enhanced_features.yml
+++ b/.github/workflows/velox_backend_enhanced.yml
@@ -13,12 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-name: Velox backend Github Runner (Enhanced Features)
+name: Velox Backend (Enhanced)
on:
pull_request:
paths:
- - '.github/workflows/velox_backend_enhanced_features.yml'
+ - '.github/workflows/velox_backend_enhanced.yml'
- 'pom.xml'
- 'backends-velox/**'
- 'gluten-uniffle/**'
diff --git a/.github/workflows/velox_backend_x86.yml
b/.github/workflows/velox_backend_x86.yml
index 7e5ba40647..18c5a5d90d 100644
--- a/.github/workflows/velox_backend_x86.yml
+++ b/.github/workflows/velox_backend_x86.yml
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-name: Velox backend Github Runner (x86)
+name: Velox Backend (x86)
on:
pull_request:
@@ -43,7 +43,7 @@ env:
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
MVN_CMD: 'mvn -ntp'
WGET_CMD: 'wget -nv'
- SETUP: 'bash .github/workflows/util/setup_helper.sh'
+ SETUP: 'source .github/workflows/util/setup_helper.sh'
CCACHE_DIR: "${{ github.workspace }}/.ccache"
# spark.sql.ansi.enabled defaults to false.
SPARK_ANSI_SQL_MODE: false
@@ -306,10 +306,7 @@ jobs:
# Setup java and maven
yum update -y && yum install -y java-1.8.0-openjdk-devel wget
tzdata python3-pip
-
$SETUP install_maven
- export PATH=${PATH}:/usr/lib/maven/bin
-
# Set environment variables
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
diff --git a/.github/workflows/velox_weekly.yml
b/.github/workflows/velox_weekly.yml
index 5091fb1905..68b8c7eb9b 100644
--- a/.github/workflows/velox_weekly.yml
+++ b/.github/workflows/velox_weekly.yml
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-name: Velox backend weekly job
+name: Velox Backend Weekly Job
on:
pull_request:
@@ -25,7 +25,7 @@ on:
env:
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
MVN_CMD: 'mvn -ntp'
- SETUP: 'bash .github/workflows/util/setup_helper.sh'
+ SETUP: 'source .github/workflows/util/setup_helper.sh'
TPCH_TEST: "env GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare
--local --preset=velox --benchmark-type=h --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1"
INSTALL_PREFIX: /usr/local
@@ -62,17 +62,11 @@ jobs:
yum install -y java-1.8.0-openjdk-devel patch wget git perl
python3-pip
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \
export PATH=$JAVA_HOME/bin:$PATH
- wget --no-check-certificate
https://downloads.apache.org/maven/maven-3/3.8.9/binaries/apache-maven-3.8.9-bin.tar.gz
&& \
- tar -xvf apache-maven*bin.tar.gz && \
- mv apache-maven-3.8.9 /usr/lib/maven && \
- export MAVEN_HOME=/usr/lib/maven && \
- export PATH=${PATH}:${MAVEN_HOME}/bin && \
- ln -s /usr/lib/maven/bin /usr/bin/mvn
- ls -l /usr/bin/mvn
+ # The upstream main branch is cloned. Please temporarily switch to
the dev branch for PR validation.
git clone -b main --depth=1
https://github.com/apache/incubator-gluten.git && cd incubator-gluten/
+ $SETUP install_maven
./dev/package.sh --spark_version=3.5
-
build-on-centos:
strategy:
@@ -112,10 +106,7 @@ jobs:
yum install -y java-1.8.0-openjdk-devel patch wget git perl
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \
export PATH=$JAVA_HOME/bin:$PATH
-
$SETUP install_maven
- export MAVEN_HOME=/usr/lib/maven && \
- export PATH=${PATH}:${MAVEN_HOME}/bin
cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5
diff --git a/dev/formatcppcode.sh b/dev/format-cpp-code.sh
similarity index 100%
rename from dev/formatcppcode.sh
rename to dev/format-cpp-code.sh
diff --git a/.github/workflows/util/setup_helper.sh b/dev/format-scala-code.sh
old mode 100644
new mode 100755
similarity index 58%
copy from .github/workflows/util/setup_helper.sh
copy to dev/format-scala-code.sh
index 86bb43c3d2..9dc7f2f220
--- a/.github/workflows/util/setup_helper.sh
+++ b/dev/format-scala-code.sh
@@ -1,4 +1,5 @@
#!/bin/bash
+
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -14,23 +15,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-set -e
+# If a new profile is introduced for new modules, please add it here to ensure
+# the new modules are covered.
+PROFILES="-Pbackends-velox -Pceleborn,uniffle -Piceberg,delta,hudi,paimon \
+ -Pspark-3.2,spark-3.3,spark-3.4,spark-3.5,spark-4.0 -Pspark-ut"
-function install_maven {
- (
- local maven_version="3.9.2"
- local local_binary="apache-maven-${maven_version}-bin.tar.gz"
- local mirror_host="https://www.apache.org/dyn/closer.lua";
- local
url="${mirror_host}/maven/maven-3/${maven_version}/binaries/${local_binary}?action=download"
- cd /opt/
- wget -nv -O ${local_binary} ${url}
- tar -xvf ${local_binary} && mv apache-maven-${maven_version} /usr/lib/maven
- )
- echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
-}
+COMMAND=$1
-for cmd in "$@"
-do
- echo "Running: $cmd"
- "$cmd"
-done
+if [[ "$COMMAND" == "check" ]]; then
+ echo "Checking Scala code style.."
+ mvn -q spotless:check $PROFILES
+elif [[ "$COMMAND" == "apply" ]] || [[ "$COMMAND" == "" ]]; then
+ echo "Fixing Scala code style.."
+ mvn -q spotless:apply $PROFILES
+else
+ echo "Unrecognized option."
+ exit 1
+fi
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
