This is an automated email from the ASF dual-hosted git repository.
xianjin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new be6afb92d [#1959] improvement(script): Decouple scripts and configs
(#1957)
be6afb92d is described below
commit be6afb92d4d6a72ba3f2b08b402d341f52104128
Author: maobaolong <baoloong...@tencent.com>
AuthorDate: Tue Jul 30 10:37:12 2024 +0800
[#1959] improvement(script): Decouple scripts and configs (#1957)
### What changes were proposed in this pull request?
There are many hard code defined ENV variables in start-xxx.sh, we need to
reactor and abstract variables to let user be able to do customize config.
### Why are the changes needed?
Scripts in the `bin` directory should be immutable and driven by
configurations.
Fix: #1959
### Does this PR introduce _any_ user-facing change?
User can define more environment variables in `conf/rss-env.sh` or by line
env-var.
### How was this patch tested?
```
COORDINATOR_JAVA_OPTS=" -Daa=bb"
COORDINATOR_CONF_FILE=`pwd`/conf/coordinator.conf COORDINATOR_XMX_SIZE=3g
bin/start-coordinator.sh
SHUFFLE_SERVER_XMX_SIZE=4g bin/start-shuffle-server.sh
bin/start-dashboard.sh
```
---
README.md | 8 +++--
bin/start-coordinator.sh | 26 +++++++-------
bin/start-dashboard.sh | 26 +++++++-------
bin/start-shuffle-server.sh | 42 +++++++++++-----------
conf/rss-env.sh | 26 +++++++++++---
deploy/kubernetes/docker/rss-env.sh | 21 +++++++++++
.../operator/examples/configuration.yaml | 10 +++++-
docs/coordinator_guide.md | 6 ++--
docs/dashboard_guide.md | 4 +--
docs/server_guide.md | 6 ++--
10 files changed, 115 insertions(+), 60 deletions(-)
diff --git a/README.md b/README.md
index f0a9e2ed2..3fc1b5692 100644
--- a/README.md
+++ b/README.md
@@ -155,7 +155,9 @@ If you have packaged tgz with hadoop jars, the env of
`HADOOP_HOME` is needn't s
```
JAVA_HOME=<java_home>
HADOOP_HOME=<hadoop home>
- XMX_SIZE="16g"
+ COORDINATOR_XMX_SIZE="16g"
+ # You can set coordinator memory size by `XMX_SIZE` too, but it affects
all components.
+ # XMX_SIZE="16g"
```
3. update RSS_HOME/conf/coordinator.conf, e.g.,
```
@@ -206,7 +208,9 @@ Deploy Steps:
```
JAVA_HOME=<java_home>
HADOOP_HOME=<hadoop home>
- XMX_SIZE="80g"
+ SHUFFLE_SERVER_XMX_SIZE="80g"
+ # You can set shuffle server memory size by `XMX_SIZE` too, but it
affects all components.
+ # XMX_SIZE="80g"
```
3. update RSS_HOME/conf/server.conf, e.g.,
```
diff --git a/bin/start-coordinator.sh b/bin/start-coordinator.sh
index 688675cd8..05bc4c104 100755
--- a/bin/start-coordinator.sh
+++ b/bin/start-coordinator.sh
@@ -60,15 +60,17 @@ set -u
echo "class path is $CLASSPATH"
-JVM_ARGS=" -server \
- -Xmx${XMX_SIZE:-8g} \
- -Xms${XMX_SIZE:-8g} \
- -XX:+UseG1GC \
+COORDINATOR_XMX_SIZE=${COORDINATOR_XMX_SIZE:-${XMX_SIZE:-"8g"}}
+COORDINATOR_BASE_JVM_ARGS=${COORDINATOR_BASE_JVM_ARGS:-" -server \
+ -Xmx${COORDINATOR_XMX_SIZE} \
+ -Xms${COORDINATOR_XMX_SIZE} \
+ -XX:+PrintCommandLineFlags"}
+
+DEFAULT_GC_ARGS=" -XX:+UseG1GC \
-XX:MaxGCPauseMillis=200 \
-XX:ParallelGCThreads=20 \
-XX:ConcGCThreads=5 \
- -XX:InitiatingHeapOccupancyPercent=45 \
- -XX:+PrintCommandLineFlags"
+ -XX:InitiatingHeapOccupancyPercent=45"
GC_LOG_ARGS_LEGACY=" -XX:+PrintGC \
-XX:+PrintAdaptiveSizePolicy \
@@ -91,10 +93,10 @@ GC_LOG_ARGS_NEW=" -XX:+IgnoreUnrecognizedVMOptions \
-Xlog:gc+start=debug \
-Xlog:gc*:file=${RSS_LOG_DIR}/gc-%t.log:tags,uptime,time,level"
-ARGS=""
+JVM_LOG_ARGS=""
if [ -f ${LOG_CONF_FILE} ]; then
- ARGS="$ARGS -Dlog4j2.configurationFile=file:${LOG_CONF_FILE}
-Dlog.path=${LOG_PATH}"
+ JVM_LOG_ARGS=" -Dlog4j2.configurationFile=file:${LOG_CONF_FILE}
-Dlog.path=${LOG_PATH}"
else
echo "Exit with error: ${LOG_CONF_FILE} file doesn't exist."
exit 1
@@ -102,13 +104,13 @@ fi
version=$($RUNNER -version 2>&1 | awk -F[\".] '/version/ {print $2}')
if [[ "$version" -lt "9" ]]; then
- GC_ARGS=$GC_LOG_ARGS_LEGACY
+ COORDINATOR_JVM_GC_ARGS="${COORDINATOR_JVM_GC_ARGS:-${DEFAULT_GC_ARGS}
${GC_LOG_ARGS_LEGACY}}"
else
- GC_ARGS=$GC_LOG_ARGS_NEW
+ COORDINATOR_JVM_GC_ARGS="${COORDINATOR_JVM_GC_ARGS:-${DEFAULT_GC_ARGS}
${GC_LOG_ARGS_NEW}}"
fi
-UNIFFLE_COORDINATOR_JAVA_OPTS=${UNIFFLE_COORDINATOR_JAVA_OPTS:-""}
-$RUNNER ${UNIFFLE_COORDINATOR_JAVA_OPTS} $ARGS $JVM_ARGS $GC_ARGS -cp
$CLASSPATH $MAIN_CLASS --conf "$COORDINATOR_CONF_FILE" $@ &
+COORDINATOR_JAVA_OPTS=${COORDINATOR_JAVA_OPTS:-""}
+$RUNNER ${COORDINATOR_BASE_JVM_ARGS} ${COORDINATOR_JVM_GC_ARGS}
${JVM_LOG_ARGS} ${COORDINATOR_JAVA_OPTS} -cp ${CLASSPATH} ${MAIN_CLASS} --conf
"${COORDINATOR_CONF_FILE}" $@ &
get_pid_file_name coordinator
echo $! >${RSS_PID_DIR}/${pid_file}
diff --git a/bin/start-dashboard.sh b/bin/start-dashboard.sh
index 757059d86..2b35cfa76 100755
--- a/bin/start-dashboard.sh
+++ b/bin/start-dashboard.sh
@@ -47,15 +47,17 @@ mkdir -p "${RSS_PID_DIR}"
echo "class path is $CLASSPATH"
-JVM_ARGS=" -server \
- -Xmx${XMX_SIZE:-8g} \
- -Xms${XMX_SIZE:-8g} \
- -XX:+UseG1GC \
+DASHBOARD_XMX_SIZE=${DASHBOARD_XMX_SIZE:-${XMX_SIZE:-"8g"}}
+DASHBOARD_BASE_JVM_ARGS=${DASHBOARD_BASE_JVM_ARGS:-" -server \
+ -Xmx${DASHBOARD_XMX_SIZE} \
+ -Xms${DASHBOARD_XMX_SIZE} \
+ -XX:+PrintCommandLineFlags"}
+
+DEFAULT_GC_ARGS=" -XX:+UseG1GC \
-XX:MaxGCPauseMillis=200 \
-XX:ParallelGCThreads=20 \
-XX:ConcGCThreads=5 \
- -XX:InitiatingHeapOccupancyPercent=45 \
- -XX:+PrintCommandLineFlags"
+ -XX:InitiatingHeapOccupancyPercent=45"
GC_LOG_ARGS_LEGACY=" -XX:+PrintGC \
-XX:+PrintAdaptiveSizePolicy \
@@ -72,10 +74,10 @@ GC_LOG_ARGS_NEW=" -XX:+IgnoreUnrecognizedVMOptions \
-Xlog:gc*:file=${RSS_LOG_DIR}/dashboard-gc-%t.log:tags,uptime,time,level:filecount=5,filesize=102400"
-ARGS=""
+JVM_LOG_ARGS=""
if [ -f ${LOG_CONF_FILE} ]; then
- ARGS="$ARGS -Dlog4j2.configurationFile=file:${LOG_CONF_FILE}
-Dlog.path=${LOG_PATH}"
+ JVM_LOG_ARGS=" -Dlog4j2.configurationFile=file:${LOG_CONF_FILE}
-Dlog.path=${LOG_PATH}"
else
echo "Exit with error: ${LOG_CONF_FILE} file doesn't exist."
exit 1
@@ -83,13 +85,13 @@ fi
version=$($RUNNER -version 2>&1 | awk -F[\".] '/version/ {print $2}')
if [[ "$version" -lt "9" ]]; then
- GC_ARGS=$GC_LOG_ARGS_LEGACY
+ DASHBOARD_JVM_GC_ARGS="${DASHBOARD_JVM_GC_ARGS:-${DEFAULT_GC_ARGS}
${GC_LOG_ARGS_LEGACY}}"
else
- GC_ARGS=$GC_LOG_ARGS_NEW
+ DASHBOARD_JVM_GC_ARGS="${DASHBOARD_JVM_GC_ARGS:-${DEFAULT_GC_ARGS}
${GC_LOG_ARGS_NEW}}"
fi
-UNIFFLE_DASHBOARD_JAVA_OPTS=${UNIFFLE_DASHBOARD_JAVA_OPTS:-""}
-$RUNNER ${UNIFFLE_DASHBOARD_JAVA_OPTS} $ARGS $JVM_ARGS $GC_ARGS -cp $CLASSPATH
$MAIN_CLASS --conf "$DASHBOARD_CONF_FILE" $@ &
+DASHBOARD_JAVA_OPTS=${DASHBOARD_JAVA_OPTS:-""}
+$RUNNER ${DASHBOARD_BASE_JVM_ARGS} ${DASHBOARD_JVM_GC_ARGS} ${JVM_LOG_ARGS}
${DASHBOARD_JAVA_OPTS} -cp ${CLASSPATH} ${MAIN_CLASS} --conf
"${DASHBOARD_CONF_FILE}" $@ &
get_pid_file_name dashboard
echo $! >${RSS_PID_DIR}/${pid_file}
diff --git a/bin/start-shuffle-server.sh b/bin/start-shuffle-server.sh
index 1f89d7d09..bc52c65ee 100755
--- a/bin/start-shuffle-server.sh
+++ b/bin/start-shuffle-server.sh
@@ -30,14 +30,15 @@ SHUFFLE_SERVER_CONF_FILE="${RSS_CONF_DIR}/server.conf"
JAR_DIR="${RSS_HOME}/jars"
LOG_CONF_FILE="${RSS_CONF_DIR}/log4j2.xml"
LOG_PATH="${RSS_LOG_DIR}/shuffle_server.log"
-SHUFFLE_SERVER_STORAGE_AUDIT_LOG_PATH="${RSS_LOG_DIR}/shuffle_server_storage_audit.log"
-SHUFFLE_SERVER_RPC_AUDIT_LOG_PATH="${RSS_LOG_DIR}/shuffle_server_rpc_audit.log"
+SHUFFLE_SERVER_STORAGE_AUDIT_LOG_PATH=${SHUFFLE_SERVER_STORAGE_AUDIT_LOG_PATH:-"${RSS_LOG_DIR}/shuffle_server_storage_audit.log"}
+SHUFFLE_SERVER_RPC_AUDIT_LOG_PATH=${SHUFFLE_SERVER_RPC_AUDIT_LOG_PATH:-"${RSS_LOG_DIR}/shuffle_server_rpc_audit.log"}
-if [ -z "${XMX_SIZE:-}" ]; then
- echo "No env XMX_SIZE."
+SHUFFLE_SERVER_XMX_SIZE=${SHUFFLE_SERVER_XMX_SIZE:-${XMX_SIZE}}
+if [ -z "${SHUFFLE_SERVER_XMX_SIZE:-}" ]; then
+ echo "No env XMX_SIZE or SHUFFLE_SERVER_XMX_SIZE."
exit 1
fi
-echo "Shuffle Server JVM XMX size: ${XMX_SIZE}"
+echo "Shuffle Server JVM XMX size: ${SHUFFLE_SERVER_XMX_SIZE}"
if [ -n "${RSS_IP:-}" ]; then
echo "Shuffle Server RSS_IP: ${RSS_IP}"
fi
@@ -92,22 +93,23 @@ if [ -n "${OOM_DUMP_PATH:-}" ]; then
JAVA_OPTS="-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$OOM_DUMP_PATH"
fi
-JVM_ARGS=" -server \
- -Xmx${XMX_SIZE} \
- -Xms${XMX_SIZE} \
+SHUFFLE_SERVER_BASE_JVM_ARGS=${SHUFFLE_SERVER_BASE_JVM_ARGS:-" -server \
+ -Xmx${SHUFFLE_SERVER_XMX_SIZE} \
+ -Xms${SHUFFLE_SERVER_XMX_SIZE} \
${MAX_DIRECT_MEMORY_OPTS} \
${JAVA_OPTS} \
- -XX:+UseG1GC \
+ -XX:+CrashOnOutOfMemoryError \
+ -XX:+ExitOnOutOfMemoryError \
+ -XX:+UnlockExperimentalVMOptions \
+ -XX:+PrintCommandLineFlags"}
+
+DEFAULT_GC_ARGS=" -XX:+UseG1GC \
-XX:MaxGCPauseMillis=200 \
-XX:ParallelGCThreads=20 \
-XX:ConcGCThreads=5 \
-XX:InitiatingHeapOccupancyPercent=60 \
-XX:G1HeapRegionSize=32m \
- -XX:+UnlockExperimentalVMOptions \
- -XX:G1NewSizePercent=10 \
- -XX:+CrashOnOutOfMemoryError \
- -XX:+ExitOnOutOfMemoryError \
- -XX:+PrintCommandLineFlags"
+ -XX:G1NewSizePercent=10"
GC_LOG_ARGS_LEGACY=" -XX:+PrintGC \
-XX:+PrintAdaptiveSizePolicy \
@@ -130,10 +132,10 @@ GC_LOG_ARGS_NEW=" -XX:+IgnoreUnrecognizedVMOptions \
-Xlog:gc+start=debug \
-Xlog:gc*:file=${RSS_LOG_DIR}/gc-%t.log:tags,uptime,time,level"
-ARGS=""
+JVM_LOG_ARGS=""
if [ -f ${LOG_CONF_FILE} ]; then
- ARGS="$ARGS -Dlog4j2.configurationFile=file:${LOG_CONF_FILE}
-Dlog.path=${LOG_PATH}
-Dshuffle.server.storage.audit.log.path=${SHUFFLE_SERVER_STORAGE_AUDIT_LOG_PATH}
-Dshuffle.server.rpc.audit.log.path=${SHUFFLE_SERVER_RPC_AUDIT_LOG_PATH}"
+ JVM_LOG_ARGS=" -Dlog4j2.configurationFile=file:${LOG_CONF_FILE}
-Dlog.path=${LOG_PATH}
-Dshuffle.server.storage.audit.log.path=${SHUFFLE_SERVER_STORAGE_AUDIT_LOG_PATH}
-Dshuffle.server.rpc.audit.log.path=${SHUFFLE_SERVER_RPC_AUDIT_LOG_PATH}"
else
echo "Exit with error: ${LOG_CONF_FILE} file doesn't exist."
exit 1
@@ -141,13 +143,13 @@ fi
version=$($RUNNER -version 2>&1 | awk -F[\".] '/version/ {print $2}')
if [[ "$version" -lt "9" ]]; then
- GC_ARGS=$GC_LOG_ARGS_LEGACY
+ SHUFFLE_SERVER_JVM_GC_ARGS="${SHUFFLE_SERVER_JVM_GC_ARGS:-${DEFAULT_GC_ARGS}
${GC_LOG_ARGS_LEGACY}}"
else
- GC_ARGS=$GC_LOG_ARGS_NEW
+ SHUFFLE_SERVER_JVM_GC_ARGS="${SHUFFLE_SERVER_JVM_GC_ARGS:-${DEFAULT_GC_ARGS}
${GC_LOG_ARGS_NEW}}"
fi
-UNIFFLE_SHUFFLE_SERVER_JAVA_OPTS=${UNIFFLE_SHUFFLE_SERVER_JAVA_OPTS:-""}
-$RUNNER ${UNIFFLE_SHUFFLE_SERVER_JAVA_OPTS} $ARGS $JVM_ARGS $GC_ARGS
$JAVA_LIB_PATH -cp $CLASSPATH $MAIN_CLASS --conf "$SHUFFLE_SERVER_CONF_FILE" $@
&
+SHUFFLE_SERVER_JAVA_OPTS=${SHUFFLE_SERVER_JAVA_OPTS:-""}
+$RUNNER ${SHUFFLE_SERVER_BASE_JVM_ARGS} ${SHUFFLE_SERVER_JVM_GC_ARGS}
${JVM_LOG_ARGS} ${JAVA_LIB_PATH} ${SHUFFLE_SERVER_JAVA_OPTS} -cp ${CLASSPATH}
${MAIN_CLASS} --conf "${SHUFFLE_SERVER_CONF_FILE}" $@ &
get_pid_file_name shuffle-server
echo $! >${RSS_PID_DIR}/${pid_file}
diff --git a/conf/rss-env.sh b/conf/rss-env.sh
index fb577e5d1..c505aadad 100644
--- a/conf/rss-env.sh
+++ b/conf/rss-env.sh
@@ -23,8 +23,10 @@ set -o errexit # exit the script if any statement returns
a non-true return va
# JAVA_HOME=<java_home_dir>
# HADOOP_HOME=<hadoop_home_dir>
-XMX_SIZE="80g" # Shuffle Server JVM XMX size
+XMX_SIZE="8g" # Coordinator/Dashboard JVM XMX size
+SHUFFLE_SERVER_XMX_SIZE="80g" # ShuffleServer JVM XMX size
+# Variables for the all components
# RSS_HOME, RSS home directory (Default: parent directory of the script).
# If you want to set it to another place, please make sure the RSS_HOME
# specified externally is respected, by defining it as follows:
@@ -40,10 +42,24 @@ XMX_SIZE="80g" # Shuffle Server JVM XMX size
# HADOOP_CONF_DIR, Hadoop configuration directory (Default:
${HADOOP_HOME}/etc/hadoop)
# RSS_PID_DIR, Where the pid file is stored (Default: ${RSS_HOME})
# RSS_LOG_DIR, Where log files are stored (Default: ${RSS_HOME}/logs)
+
+# Variables for the dashboard
+# DASHBOARD_BASE_JVM_ARGS, Base JVM options, which by default is calculated
from
+# `DASHBOARD_XMX_SIZE` or `XMX_SIZE and has sane defaults. Normally no need
for customization.
+# DASHBOARD_JAVA_OPTS, Extra JVM options for the dashboard, which will append
after base arguments.
+# This will override some default JVM options if they are both specified in
default and this one.
+
+# Variables for the coordinator
+# COORDINATOR_BASE_JVM_ARGS, Base JVM options, which by default is calculated
from
+# `COORDINATOR_XMX_SIZE` or `XMX_SIZE and has sane defaults. Normally no need
for customization.
+# COORDINATOR_JAVA_OPTS, Extra JVM options for the coordinator, which will
append after base arguments.
+
+# Variables for the shuffle server
# RSS_IP, IP address Shuffle Server binds to on this node (Default: first
non-loopback ipv4)
# MAX_DIRECT_MEMORY_SIZE Shuffle Server JVM off heap memory size (Default: not
set)
# MALLOC_ARENA_MAX, Set the number of memory arenas for Shuffle Server
(Default: 4)
-
-# UNIFFLE_DASHBOARD_JAVA_OPTS, JVM options for the dashboard
-# UNIFFLE_SHUFFLE_SERVER_JAVA_OPTS, JVM options for the shuffle server
-# UNIFFLE_COORDINATOR_JAVA_OPTS, JVM options for the coordinator
+# SHUFFLE_SERVER_JVM_ARGS, Base JVM options, which by default is calculated
from
+# `SHUFFLE_SERVER_XMX_SIZE` or `XMX_SIZE and has sane defaults. Normally no
need for customization.
+# SHUFFLE_SERVER_JAVA_OPTS, Extra JVM options for the shuffle server, which
will append after base arguments.
+# SHUFFLE_SERVER_STORAGE_AUDIT_LOG_PATH, Shuffle Server storage audit log path
(Default: ${RSS_LOG_DIR}/shuffle_server_storage_audit.log)
+# SHUFFLE_SERVER_RPC_AUDIT_LOG_PATH, Shuffle Server RPC audit log path
(Default: ${RSS_LOG_DIR}/shuffle_server_rpc_audit.log)
diff --git a/deploy/kubernetes/docker/rss-env.sh
b/deploy/kubernetes/docker/rss-env.sh
index 7c8715331..d6b0dfc48 100644
--- a/deploy/kubernetes/docker/rss-env.sh
+++ b/deploy/kubernetes/docker/rss-env.sh
@@ -25,6 +25,7 @@ HADOOP_HOME="/data/rssadmin/hadoop"
RUNNER="${JAVA_HOME}/bin/java"
JPS="${JAVA_HOME}/bin/jps"
+# Variables for the all components
# RSS_HOME, RSS home directory (Default: parent directory of the script).
# If you want to set it to another place, please make sure the RSS_HOME
# specified externally is respected, by defining it as follows:
@@ -40,4 +41,24 @@ JPS="${JAVA_HOME}/bin/jps"
# HADOOP_CONF_DIR, Hadoop configuration directory (Default:
${HADOOP_HOME}/etc/hadoop)
# RSS_PID_DIR, Where the pid file is stored (Default: ${RSS_HOME})
# RSS_LOG_DIR, Where log files are stored (Default: ${RSS_HOME}/logs)
+
+# Variables for the dashboard
+# DASHBOARD_BASE_JVM_ARGS, Base JVM options, which by default is calculated
from
+# `DASHBOARD_XMX_SIZE` or `XMX_SIZE and has sane defaults. Normally no need
for customization.
+# DASHBOARD_JAVA_OPTS, Extra JVM options for the dashboard, which will append
after base arguments.
+# This will override some default JVM options if they are both specified in
default and this one.
+
+# Variables for the coordinator
+# COORDINATOR_BASE_JVM_ARGS, Base JVM options, which by default is calculated
from
+# `COORDINATOR_XMX_SIZE` or `XMX_SIZE and has sane defaults. Normally no need
for customization.
+# COORDINATOR_JAVA_OPTS, Extra JVM options for the coordinator, which will
append after base arguments.
+
+# Variables for the shuffle server
# RSS_IP, IP address Shuffle Server binds to on this node (Default: first
non-loopback ipv4)
+# MAX_DIRECT_MEMORY_SIZE Shuffle Server JVM off heap memory size (Default: not
set)
+# MALLOC_ARENA_MAX, Set the number of memory arenas for Shuffle Server
(Default: 4)
+# SHUFFLE_SERVER_JVM_ARGS, Base JVM options, which by default is calculated
from
+# `SHUFFLE_SERVER_XMX_SIZE` or `XMX_SIZE and has sane defaults. Normally no
need for customization.
+# SHUFFLE_SERVER_JAVA_OPTS, Extra JVM options for the shuffle server, which
will append after base arguments.
+# SHUFFLE_SERVER_STORAGE_AUDIT_LOG_PATH, Shuffle Server storage audit log path
(Default: ${RSS_LOG_DIR}/shuffle_server_storage_audit.log)
+# SHUFFLE_SERVER_RPC_AUDIT_LOG_PATH, Shuffle Server RPC audit log path
(Default: ${RSS_LOG_DIR}/shuffle_server_rpc_audit.log)
diff --git a/deploy/kubernetes/operator/examples/configuration.yaml
b/deploy/kubernetes/operator/examples/configuration.yaml
index 6b7183b5f..e18fa6264 100644
--- a/deploy/kubernetes/operator/examples/configuration.yaml
+++ b/deploy/kubernetes/operator/examples/configuration.yaml
@@ -93,4 +93,12 @@ data:
rss.server.read.buffer.capacity 5g
rss.server.shuffle.expired.timeout.ms 120000
rss.server.write.retry.max 2
- rss.storage.type MEMORY_LOCALFILE
\ No newline at end of file
+ rss.storage.type MEMORY_LOCALFILE
+ rss-env.sh: |-
+ #!/usr/bin/env bash
+ set -o pipefail
+ set -o nounset # exit the script if you try to use an uninitialised
variable
+ set -o errexit # exit the script if any statement returns a non-true
return value
+ HADOOP_HOME="/data/rssadmin/hadoop"
+ RUNNER="${JAVA_HOME}/bin/java"
+ JPS="${JAVA_HOME}/bin/jps"
diff --git a/docs/coordinator_guide.md b/docs/coordinator_guide.md
index 978e0a107..e443494c1 100644
--- a/docs/coordinator_guide.md
+++ b/docs/coordinator_guide.md
@@ -34,12 +34,12 @@ This document will introduce how to deploy Uniffle
coordinators.
```
JAVA_HOME=<java_home>
HADOOP_HOME=<hadoop home>
- XMX_SIZE="16g"
+ COORDINATOR_XMX_SIZE="16g"
```
- You can add extra JVM arguments for the Uniffle coordinator by specifying
`UNIFFLE_COORDINATOR_JAVA_OPTS` in `rss-env.sh`.
+ You can add extra JVM arguments for the Uniffle coordinator by specifying
`COORDINATOR_JAVA_OPTS` in `rss-env.sh`.
Example:
```
- UNIFFLE_COORDINATOR_JAVA_OPTS="-Xdebug
-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5006
-Drss.jetty.http.port=19998"
+ COORDINATOR_JAVA_OPTS="-Xdebug
-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5006
-Drss.jetty.http.port=19998"
```
3. update RSS_HOME/conf/coordinator.conf, eg,
```
diff --git a/docs/dashboard_guide.md b/docs/dashboard_guide.md
index 48a2a8e53..4f23c4037 100644
--- a/docs/dashboard_guide.md
+++ b/docs/dashboard_guide.md
@@ -33,10 +33,10 @@ coordinator.web.address http://coordinator.hostname:19998/
```
### Configure related JVM parameters
-You can add extra JVM arguments for the Uniffle dashboard by specifying
`UNIFFLE_DASHBOARD_JAVA_OPTS` in `RSS_HOME/bin/rss-env.sh`
+You can add extra JVM arguments for the Uniffle dashboard by specifying
`DASHBOARD_JAVA_OPTS` in `RSS_HOME/bin/rss-env.sh`
Example:
```
-UNIFFLE_DASHBOARD_JAVA_OPTS="-Xdebug
-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5004
-Drss.jetty.http.port=19997"
+DASHBOARD_JAVA_OPTS="-Xdebug
-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5004
-Drss.jetty.http.port=19997"
```
### Start the dashboard process
diff --git a/docs/server_guide.md b/docs/server_guide.md
index 16e2146ff..3cc061c1e 100644
--- a/docs/server_guide.md
+++ b/docs/server_guide.md
@@ -30,7 +30,7 @@ This document will introduce how to deploy Uniffle shuffle
servers.
```
JAVA_HOME=<java_home>
HADOOP_HOME=<hadoop home>
- XMX_SIZE="80g"
+ SHUFFLE_SERVER_XMX_SIZE="80g"
```
For the following cases, you don't need to specify `HADOOP_HOME` that will
simplify the server deployment.
@@ -176,12 +176,12 @@ XMX_SIZE=40g
MAX_DIRECT_MEMORY_SIZE=360g
```
-You can add extra JVM arguments for the Uniffle server by specifying
`UNIFFLE_SHUFFLE_SERVER_JAVA_OPTS` in `rss-env.sh`.
+You can add extra JVM arguments for the Uniffle server by specifying
`SHUFFLE_SERVER_JAVA_OPTS` in `rss-env.sh`.
For example:
```
-UNIFFLE_SHUFFLE_SERVER_JAVA_OPTS="-Xdebug
-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5005
-Drss.jetty.http.port=19978"
+SHUFFLE_SERVER_JAVA_OPTS="-Xdebug
-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5005
-Drss.jetty.http.port=19978"
```
##### server.conf
