This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 714c8d31f7 [VL] Remove unnecessary build options (#9021)
714c8d31f7 is described below
commit 714c8d31f7e7f8ee6ad7e24ad2e3ff2572b5a434
Author: PHILO-HE <[email protected]>
AuthorDate: Mon Mar 17 16:36:17 2025 +0800
[VL] Remove unnecessary build options (#9021)
---
.github/workflows/velox_backend.yml | 32 ++++++++++++++++----------------
dev/buildbundle-veloxbe.sh | 2 +-
docs/developers/MicroBenchmarks.md | 4 ++--
docs/get-started/Velox.md | 8 ++++----
4 files changed, 23 insertions(+), 23 deletions(-)
diff --git a/.github/workflows/velox_backend.yml
b/.github/workflows/velox_backend.yml
index 14c1b2ad67..1e3d2c2710 100644
--- a/.github/workflows/velox_backend.yml
+++ b/.github/workflows/velox_backend.yml
@@ -629,7 +629,7 @@ jobs:
run: |
cd $GITHUB_WORKSPACE/
export SPARK_SCALA_VERSION=2.12
- $MVN_CMD clean test -Pspark-3.2 -Pspark-ut -Pbackends-velox
-Pceleborn -Piceberg \
+ $MVN_CMD clean test -Pspark-3.2 -Pspark-ut -Pbackends-velox
-Piceberg \
-Pdelta -Phudi
-DargLine="-Dspark.test.home=/opt/shims/spark32/spark_home/" \
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
- name: Upload test report
@@ -664,7 +664,7 @@ jobs:
- name: Build and run unit test for Spark 3.2.2 (slow tests)
run: |
cd $GITHUB_WORKSPACE/
- $MVN_CMD clean test -Pspark-3.2 -Pspark-ut -Pbackends-velox
-Pceleborn -Piceberg -Pdelta -Phudi \
+ $MVN_CMD clean test -Pspark-3.2 -Pspark-ut -Pbackends-velox
-Piceberg -Pdelta -Phudi \
-DargLine="-Dspark.test.home=/opt/shims/spark32/spark_home/"
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
- name: Upload test report
if: always()
@@ -700,7 +700,7 @@ jobs:
run: |
cd $GITHUB_WORKSPACE/
export SPARK_SCALA_VERSION=2.12
- $MVN_CMD clean test -Pspark-3.3 -Pbackends-velox -Pceleborn
-Piceberg -Pdelta -Phudi -Pspark-ut \
+ $MVN_CMD clean test -Pspark-3.3 -Pbackends-velox -Piceberg -Pdelta
-Phudi -Pspark-ut \
-DargLine="-Dspark.test.home=/opt/shims/spark33/spark_home/" \
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
- name: Upload test report
@@ -737,7 +737,7 @@ jobs:
- name: Build and Run unit test for Spark 3.3.1 (slow tests)
run: |
cd $GITHUB_WORKSPACE/
- $MVN_CMD clean test -Pspark-3.3 -Pbackends-velox -Pceleborn
-Piceberg -Pdelta -Phudi -Pspark-ut \
+ $MVN_CMD clean test -Pspark-3.3 -Pbackends-velox -Piceberg -Pdelta
-Phudi -Pspark-ut \
-DargLine="-Dspark.test.home=/opt/shims/spark33/spark_home/" \
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
- name: Upload test report
@@ -777,7 +777,7 @@ jobs:
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
export SPARK_HOME=/opt/shims/spark34/spark_home/
ls -l /opt/shims/spark34/spark_home/
- $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox
-Pceleborn -Piceberg -Pdelta -Phudi -Pspark-ut \
+ $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Phudi -Pspark-ut \
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
\
-DargLine="-Dspark.test.home=/opt/shims/spark34/spark_home/
${EXTRA_FLAGS}"
- name: Upload test report
@@ -823,7 +823,7 @@ jobs:
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
export SPARK_HOME=/opt/shims/spark34/spark_home/
ls -l /opt/shims/spark34/spark_home/
- $MVN_CMD clean test -Pspark-3.4 -Pjava-8 -Pbackends-velox -Pceleborn
-Pdelta -Phudi -Pspark-ut \
+ $MVN_CMD clean test -Pspark-3.4 -Pjava-8 -Pbackends-velox -Pdelta
-Phudi -Pspark-ut \
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
\
-DargLine="-Dspark.test.home=/opt/shims/spark34/spark_home/"
- name: Upload test report
@@ -860,7 +860,7 @@ jobs:
cd $GITHUB_WORKSPACE/
export SPARK_HOME=/opt/shims/spark34/spark_home/
ls -l /opt/shims/spark34/spark_home/
- $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox
-Pceleborn -Piceberg -Pdelta -Pspark-ut -Phudi \
+ $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Pspark-ut -Phudi \
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest \
-DargLine="-Dspark.test.home=/opt/shims/spark34/spark_home/
${EXTRA_FLAGS}"
- name: Upload test report
@@ -892,7 +892,7 @@ jobs:
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
export SPARK_HOME=/opt/shims/spark34/spark_home/
ls -l /opt/shims/spark34/spark_home/
- $MVN_CMD clean test -Pspark-3.4 -Pjava-8 -Pbackends-velox -Pceleborn
-Pdelta -Pspark-ut -Phudi \
+ $MVN_CMD clean test -Pspark-3.4 -Pjava-8 -Pbackends-velox -Pdelta
-Pspark-ut -Phudi \
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest \
-DargLine="-Dspark.test.home=/opt/shims/spark34/spark_home/"
- name: Upload test report
@@ -929,7 +929,7 @@ jobs:
run: |
cd $GITHUB_WORKSPACE/
export SPARK_SCALA_VERSION=2.12
- $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn
-Piceberg -Pdelta -Phudi -Pspark-ut \
+ $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Piceberg -Pdelta
-Phudi -Pspark-ut \
-DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
- name: Upload test report
@@ -973,7 +973,7 @@ jobs:
cd $GITHUB_WORKSPACE/
export SPARK_SCALA_VERSION=2.12
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
- $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox
-Pceleborn -Piceberg -Pdelta -Phudi -Pspark-ut \
+ $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg
-Pdelta -Phudi -Pspark-ut \
-DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/
${EXTRA_FLAGS}" \
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
- name: Upload test report
@@ -1016,7 +1016,7 @@ jobs:
run: |
cd $GITHUB_WORKSPACE/
export SPARK_SCALA_VERSION=2.13
- $MVN_CMD clean test -Pspark-3.5 -Pscala-2.13 -Pbackends-velox
-Pceleborn -Piceberg \
+ $MVN_CMD clean test -Pspark-3.5 -Pscala-2.13 -Pbackends-velox
-Piceberg \
-Pdelta -Pspark-ut
-DargLine="-Dspark.test.home=/opt/shims/spark35-scala-2.13/spark_home/" \
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
- name: Upload test report
@@ -1045,7 +1045,7 @@ jobs:
- name: Build and Run unit test for Spark 3.5.2 (slow tests)
run: |
cd $GITHUB_WORKSPACE/
- $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn
-Piceberg -Pdelta -Phudi -Pspark-ut \
+ $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Piceberg -Pdelta
-Phudi -Pspark-ut \
-DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
- name: Upload test report
@@ -1082,7 +1082,7 @@ jobs:
run: |
cd $GITHUB_WORKSPACE/
export SPARK_SCALA_VERSION=2.12
- $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn
-Piceberg -Pdelta -Pspark-ut \
+ $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Piceberg -Pdelta
-Pspark-ut \
-DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/
-Dspark.gluten.ras.enabled=true" \
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
- name: Upload test report
@@ -1110,7 +1110,7 @@ jobs:
- name: Build and Run unit test for Spark 3.5.2 (slow tests)
run: |
cd $GITHUB_WORKSPACE/
- $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn
-Piceberg -Pdelta -Pspark-ut \
+ $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Piceberg -Pdelta
-Pspark-ut \
-DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/
-Dspark.gluten.ras.enabled=true" \
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
- name: Upload test report
@@ -1146,7 +1146,7 @@ jobs:
run: |
cd $GITHUB_WORKSPACE/
export SPARK_SCALA_VERSION=2.12
- $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn
-Piceberg -Pdelta -Pspark-ut \
+ $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Piceberg -Pdelta
-Pspark-ut \
-DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/
-Dspark.gluten.sql.columnar.forceShuffledHashJoin=false" \
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
- name: Upload test report
@@ -1174,7 +1174,7 @@ jobs:
- name: Build and Run unit test for Spark 3.5.2 (slow tests)
run: |
cd $GITHUB_WORKSPACE/
- $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn
-Piceberg -Pdelta -Pspark-ut \
+ $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Piceberg -Pdelta
-Pspark-ut \
-DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/
-Dspark.gluten.sql.columnar.forceShuffledHashJoin=false" \
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
- name: Upload test report
diff --git a/dev/buildbundle-veloxbe.sh b/dev/buildbundle-veloxbe.sh
index 39f72a66a1..8d181c2dfe 100755
--- a/dev/buildbundle-veloxbe.sh
+++ b/dev/buildbundle-veloxbe.sh
@@ -20,7 +20,7 @@ source "$BASEDIR/builddeps-veloxbe.sh"
function build_for_spark {
spark_version=$1
- mvn clean package -Pbackends-velox -Pceleborn -Puniffle
-Pspark-$spark_version -DskipTests
+ mvn clean package -Pbackends-velox -Pspark-$spark_version -DskipTests
}
function check_supported {
diff --git a/docs/developers/MicroBenchmarks.md
b/docs/developers/MicroBenchmarks.md
index e274d5e39f..ccb01d4737 100644
--- a/docs/developers/MicroBenchmarks.md
+++ b/docs/developers/MicroBenchmarks.md
@@ -36,8 +36,8 @@ generate example input files:
cd /path/to/gluten/
./dev/buildbundle-veloxbe.sh --build_tests=ON --build_benchmarks=ON
-# Run test to generate input data files. If you are using spark 3.3, replace
-Pspark-3.2 with -Pspark-3.3, If you are using uniffle, replace -Pceleborn with
-Puniffle
-mvn test -Pspark-3.2 -Pbackends-velox -Pcelenborn -pl backends-velox -am \
+# Run test to generate input data files. If you are using spark 3.3, replace
-Pspark-3.2 with -Pspark-3.3.
+mvn test -Pspark-3.2 -Pbackends-velox -pl backends-velox -am \
-DtagsToInclude="org.apache.gluten.tags.GenerateExample" -Dtest=none
-DfailIfNoTests=false -Dexec.skip
```
diff --git a/docs/get-started/Velox.md b/docs/get-started/Velox.md
index 4f922edd75..582a18a0ed 100644
--- a/docs/get-started/Velox.md
+++ b/docs/get-started/Velox.md
@@ -93,13 +93,13 @@ Currently, Gluten is using a [forked
Velox](https://github.com/oap-project/velox
## compile Gluten java module and create package jar
cd /path/to/gluten
# For spark3.2.x
-mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.2 -DskipTests
+mvn clean package -Pbackends-velox -Pspark-3.2 -DskipTests
# For spark3.3.x
-mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.3 -DskipTests
+mvn clean package -Pbackends-velox -Pspark-3.3 -DskipTests
# For spark3.4.x
-mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.4 -DskipTests
+mvn clean package -Pbackends-velox -Pspark-3.4 -DskipTests
# For spark3.5.x
-mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-3.5 -DskipTests
+mvn clean package -Pbackends-velox -Pspark-3.5 -DskipTests
```
Notes： Building Velox may fail caused by OOM. You can prevent this failure by
adjusting `NUM_THREADS` (e.g., `export NUM_THREADS=4`) before building
Gluten/Velox. The recommended minimal memory size is 64G.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
