This is an automated email from the ASF dual-hosted git repository.
zuston pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/auron.git
The following commit(s) were added to refs/heads/master by this push:
new bd36edab [AURON-1139] Setup integration testing with Uniffle (#1222)
bd36edab is described below
commit bd36edab6134d69bdb6df4b0bc2cbf6f03d94dc9
Author: Fei Wang <[email protected]>
AuthorDate: Sun Aug 31 18:32:30 2025 -0700
[AURON-1139] Setup integration testing with Uniffle (#1222)
---
.github/workflows/tpcds-reusable.yml | 77 ++++++++++++++++++++++++++++++++++--
.github/workflows/uniffle.yml | 59 +++++++++++++++++++++++++++
2 files changed, 133 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/tpcds-reusable.yml
b/.github/workflows/tpcds-reusable.yml
index e9d64329..d252be8b 100644
--- a/.github/workflows/tpcds-reusable.yml
+++ b/.github/workflows/tpcds-reusable.yml
@@ -42,6 +42,22 @@ on:
required: false
type: string
default: ''
+ unifflever:
+ required: false
+ type: string
+ default: ''
+ uniffleurl:
+ required: false
+ type: string
+ default: ''
+ hadoopver:
+ required: false
+ type: string
+ default: ''
+ hadoopurl:
+ required: false
+ type: string
+ default: ''
extrabuildopt:
required: false
type: string
@@ -216,10 +232,10 @@ jobs:
- name: Start Celeborn-${{ inputs.celebornver }}
if: ${{ inputs.celebornver != '' && inputs.celebornurl != '' }}
run: |
- mkdir -p /tmp/celeborn/data && mkdir -p /tmp/celeborn/logs && \
+ mkdir -p /tmp/rss/data && mkdir -p /tmp/rss/logs && \
cd celeborn-bin-${{ inputs.celebornver }} && \
- bash -c "echo -e
'CELEBORN_MASTER_MEMORY=4g\nCELEBORN_WORKER_MEMORY=4g\nCELEBORN_WORKER_OFFHEAP_MEMORY=8g\nCELEBORN_LOG_DIR=/tmp/celeborn/logs'
> ./conf/celeborn-env.sh" && \
- bash -c "echo -e 'celeborn.worker.storage.dirs
/tmp/celeborn\nceleborn.worker.storage.workingDir
data\nceleborn.worker.commitFiles.threads
128\nceleborn.worker.sortPartition.threads 64' > ./conf/celeborn-defaults.conf"
&& \
+ bash -c "echo -e
'CELEBORN_MASTER_MEMORY=4g\nCELEBORN_WORKER_MEMORY=4g\nCELEBORN_WORKER_OFFHEAP_MEMORY=8g\nCELEBORN_LOG_DIR=/tmp/rss/logs'
> ./conf/celeborn-env.sh" && \
+ bash -c "echo -e 'celeborn.worker.storage.dirs
/tmp/rss\nceleborn.worker.storage.workingDir
data\nceleborn.worker.commitFiles.threads
128\nceleborn.worker.sortPartition.threads 64' > ./conf/celeborn-defaults.conf"
&& \
bash ./sbin/start-master.sh && \
bash ./sbin/start-worker.sh
@@ -229,6 +245,53 @@ jobs:
ls -la celeborn-bin-${{ inputs.celebornver }}/spark
cp celeborn-bin-${{ inputs.celebornver
}}/spark/celeborn-client-spark-*_${{ inputs.scalaver }}-*.jar spark-bin-${{
inputs.sparkver }}_${{ inputs.scalaver }}/jars/
+ - uses: actions/cache@v4
+ if: ${{ inputs.unifflever != '' && inputs.uniffleurl != '' }}
+ id: cache-uniffle-bin
+ with:
+ path: uniffle-bin-${{ inputs.unifflever }}
+ key: uniffle-bin-${{ inputs.unifflever }}
+
+ - name: Setup Uniffle-${{ inputs.unifflever }}
+ id: setup-uniffle-bin
+ if: ${{ inputs.unifflever != '' && inputs.uniffleurl != '' &&
steps.cache-uniffle-bin.outputs.cache-hit != 'true' }}
+ run: |
+ wget -c ${{ inputs.uniffleurl }} && \
+ mkdir -p uniffle-bin-${{ inputs.unifflever }} && \
+ tar -xf ./apache-uniffle-${{ inputs.unifflever
}}-incubating-bin.tar.gz -C uniffle-bin-${{ inputs.unifflever }}
--strip-component=1
+
+ - uses: actions/cache@v4
+ if: ${{ inputs.hadoopver != '' && inputs.hadoopurl != '' }}
+ id: cache-hadoop-bin
+ with:
+ path: hadoop-bin-${{ inputs.hadoopver }}
+ key: hadoop-bin-${{ inputs.hadoopver }}
+
+ - name: Setup hadoop-${{ inputs.hadoopver }}
+ id: setup-hadoop-bin
+ if: ${{ inputs.hadoopver != '' && inputs.hadoopurl != '' &&
steps.cache-hadoop-bin.outputs.cache-hit != 'true' }}
+ run: |
+ wget -c ${{ inputs.hadoopurl }} && \
+ mkdir -p hadoop-bin-${{ inputs.hadoopver }} && \
+ tar -xf ./hadoop-${{ inputs.hadoopver }}.tar.gz -C hadoop-bin-${{
inputs.hadoopver }} --strip-component=1 && \
+ ls -la hadoop-bin-${{ inputs.hadoopver }}
+
+ - name: Start Uniffle-${{ inputs.unifflever }}
+ if: ${{ inputs.unifflever != '' && inputs.uniffleurl != '' }}
+ run: |
+ mkdir -p /tmp/rss/data && mkdir -p /tmp/rss/logs && \
+ cd uniffle-bin-${{ inputs.unifflever }} && \
+ bash -c "echo -e 'XMX_SIZE=16g\nHADOOP_HOME=../hadoop-bin-${{
inputs.hadoopver }}\nRSS_LOG_DIR=/tmp/rss/logs' > ./bin/rss-env.sh" && \
+ bash -c "echo -e 'rss.coordinator.shuffle.nodes.max
1\nrss.rpc.server.port 19999' > ./conf/coordinator.conf" && \
+ bash -c "echo -e 'rss.server.app.expired.withoutHeartbeat
7200000\nrss.server.heartbeat.delay 3000\nrss.rpc.server.port
19997\nrss.rpc.server.type GRPC_NETTY\nrss.jetty.http.port
19996\nrss.server.netty.port 19995\nrss.storage.basePath
/tmp/rss/data\nrss.storage.type MEMORY_LOCALFILE\nrss.coordinator.quorum
localhost:19999\nrss.server.flush.thread.alive
10\nrss.server.single.buffer.flush.threshold 64m' > ./conf/server.conf" && \
+ bash ./bin/start-coordinator.sh && bash ./bin/start-shuffle-server.sh
+
+ - name: Install Uniffle JAR
+ if: ${{ inputs.unifflever != '' && inputs.uniffleurl != '' }}
+ run: |
+ ls -la uniffle-bin-${{ inputs.unifflever }}/jars/client/spark3/
+ cp uniffle-bin-${{ inputs.unifflever }}/jars/client/spark3/*.jar
spark-bin-${{ inputs.sparkver }}_${{ inputs.scalaver }}/jars/
+
- name: Run
run: |
ls -la
@@ -239,3 +302,11 @@ jobs:
${{ inputs.extrasparkconf }} \
--data-location dev/tpcds_1g \
--query-filter ${{ matrix.query }}
+
+ - name: Upload RSS log
+ if: ${{ failure() && (inputs.celebornver != '' || inputs.unifflever !=
'') }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: rss-log-${{ inputs.extraidentifier }}
+ path: |
+ /tmp/rss/logs/*
diff --git a/.github/workflows/uniffle.yml b/.github/workflows/uniffle.yml
new file mode 100644
index 00000000..910ea080
--- /dev/null
+++ b/.github/workflows/uniffle.yml
@@ -0,0 +1,59 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Uniffle
+
+on:
+ workflow_dispatch:
+ pull_request:
+ branches:
+ - master
+ - branch-*
+
+concurrency:
+ group: uniffle-${{ github.workflow }}-${{ github.event.pull_request.number
|| github.ref }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+ test-uniffle:
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - unifflever: "0.9.2"
+ hadoopver: "2.8.5"
+ uniffleprofile: "uniffle,uniffle-0.9"
+ uses: ./.github/workflows/tpcds-reusable.yml
+ name: Test Uniffle ${{ matrix.unifflever }}
+ with:
+ unifflever: ${{ matrix.unifflever }}
+ uniffleurl: https://archive.apache.org/dist/uniffle/${{
matrix.unifflever }}/apache-uniffle-${{ matrix.unifflever
}}-incubating-bin.tar.gz
+ hadoopver: ${{ matrix.hadoopver }}
+ hadoopurl: https://archive.apache.org/dist/hadoop/common/hadoop-${{
matrix.hadoopver }}/hadoop-${{ matrix.hadoopver }}.tar.gz
+ extrabuildopt: -P${{ matrix.uniffleprofile }} -DuniffleVersion=${{
matrix.unifflever }}
+ extraidentifier: uniffle-${{ matrix.unifflever }}
+ sparkver: "spark-3.5"
+ sparkurl:
"https://archive.apache.org/dist/spark/spark-3.5.6/spark-3.5.6-bin-hadoop3.tgz";
+ scalaver: "2.12"
+ extrasparkconf: >-
+ --conf
spark.shuffle.manager=org.apache.spark.sql.execution.auron.shuffle.uniffle.AuronUniffleShuffleManager
+ --conf spark.serializer=org.apache.spark.serializer.KryoSerializer
+ --conf spark.rss.coordinator.quorum=localhost:19999
+ --conf spark.rss.enabled=true
+ --conf spark.rss.storage.type=MEMORY_LOCALFILE
+ --conf spark.rss.client.type=GRPC_NETTY
+ queries: '["q1,q2,q3,q4,q5,q6,q7,q8,q9"]'
