summaryzb commented on code in PR #3767:
URL: https://github.com/apache/incubator-gluten/pull/3767#discussion_r1532020728
##########
.github/workflows/velox_be.yml:
##########
@@ -441,7 +442,37 @@ jobs:
run: |
$PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
cd /opt/gluten && \
- mvn clean install -Pspark-3.2 -Pbackends-velox -Prss -Piceberg
-Pdelta -DskipTests'
+ mvn clean install -Pspark-3.2 -Pbackends-velox -Prss -Prss-uniffle
-Piceberg -Pdelta -DskipTests'
+ - name: TPC-H SF1.0 && TPC-DS SF10.0 Parquet local spark3.2 with uniffle
+ run: |
+ $PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
+ echo "java path:`command -v java`" && \
+ echo "cur dir `pwd`" && \
+ wget -nv
https://archive.apache.org/dist/incubator/uniffle/0.8.0/apache-uniffle-0.8.0-incubating-bin.tar.gz
&& \
+ tar xzf apache-uniffle-0.8.0-incubating-bin.tar.gz -C /opt/ && mv
/opt/rss-0.8.0-hadoop2.8 /opt/uniffle && \
+ wget -nv
https://archive.apache.org/dist/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz
&& \
+ tar xzf hadoop-2.8.5.tar.gz -C /opt/ && \
+ cd /opt/uniffle && mkdir shuffle_data && \
+ echo -e
"XMX_SIZE=16g\nHADOOP_HOME=/opt/hadoop-2.8.5\nJAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64"
> ./bin/rss-env.sh && \
+ echo -e "rss.coordinator.server.periodic.output.interval.times
1\nrss.coordinator.shuffle.nodes.max 1" > ./conf/coordinator.conf && \
+ echo -e "rss.server.heartbeat.delay 3000\nrss.rpc.server.port
19997\nrss.jetty.http.port 19996\nrss.server.netty.port
19995\nrss.storage.basePath /opt/uniffle/shuffle_data\nrss.storage.type
MEMORY_LOCALFILE\nrss.coordinator.quorum
localhost:19999\nrss.server.flush.thread.alive
5\nrss.server.single.buffer.flush.threshold 64m" > ./conf/server.conf && \
+ bash ./bin/start-coordinator.sh && bash
./bin/start-shuffle-server.sh && \
+ sleep 50 && \
+ echo "=====show uniffle startup====" && \
+ cat ./logs/coordinator.log && \
+ cat ./logs/shuffle_server.log && \
+ cd /opt/gluten/tools/gluten-it && mvn clean install
-Pspark-3.2,rss-uniffle \
+ && tail -n 200 /opt/uniffle/logs/coordinator.log \
+ && tail -n 200 /opt/uniffle/logs/shuffle_server.log \
+ && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
+ --local --preset=velox-with-uniffle --benchmark-type=h
--error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
+# && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
+# --local --preset=velox-with-uniffle --benchmark-type=ds
--error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
Review Comment:
thanks for reminding
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@gluten.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@gluten.apache.org
For additional commands, e-mail: commits-h...@gluten.apache.org
