This is an automated email from the ASF dual-hosted git repository. yikun pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark-docker.git
The following commit(s) were added to refs/heads/master by this push: new 6022289 [SPARK-44177] Add 'set -eo pipefail' to entrypoint and quote variables 6022289 is described below commit 60222892836549f05c56edd49ac81c688c8e7356 Author: Yikun Jiang <yikunk...@gmail.com> AuthorDate: Tue Jun 27 08:59:03 2023 +0800 [SPARK-44177] Add 'set -eo pipefail' to entrypoint and quote variables ### What changes were proposed in this pull request? Add 'set -eo pipefail' to entrypoint and quote variables ### Why are the changes needed? Address DOI comments: 1. Have you considered a set -eo pipefail on the entrypoint script to help prevent any errors from being silently ignored? 2. You probably want to quote this (and many of the other variables in this execution); ala --driver-url "$SPARK_DRIVER_URL" [1] https://github.com/docker-library/official-images/pull/13089#issuecomment-1601334895 [2] https://github.com/docker-library/official-images/pull/13089#issuecomment-1601813499 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? CI passed Closes #49 from Yikun/quote. Authored-by: Yikun Jiang <yikunk...@gmail.com> Signed-off-by: Yikun Jiang <yikunk...@gmail.com> --- 3.4.0/scala2.12-java11-ubuntu/entrypoint.sh | 31 ++++++++++++++++------------- 3.4.1/scala2.12-java11-ubuntu/entrypoint.sh | 31 ++++++++++++++++------------- entrypoint.sh.template | 31 ++++++++++++++++------------- 3 files changed, 51 insertions(+), 42 deletions(-) diff --git a/3.4.0/scala2.12-java11-ubuntu/entrypoint.sh b/3.4.0/scala2.12-java11-ubuntu/entrypoint.sh index 08fc925..2e3d2a8 100755 --- a/3.4.0/scala2.12-java11-ubuntu/entrypoint.sh +++ b/3.4.0/scala2.12-java11-ubuntu/entrypoint.sh @@ -15,6 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# Prevent any errors from being silently ignored +set -eo pipefail + attempt_setup_fake_passwd_entry() { # Check whether there is a passwd entry for the container UID local myuid; myuid="$(id -u)" @@ -51,10 +54,10 @@ if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH" fi -if ! [ -z ${PYSPARK_PYTHON+x} ]; then +if ! [ -z "${PYSPARK_PYTHON+x}" ]; then export PYSPARK_PYTHON fi -if ! [ -z ${PYSPARK_DRIVER_PYTHON+x} ]; then +if ! [ -z "${PYSPARK_DRIVER_PYTHON+x}" ]; then export PYSPARK_DRIVER_PYTHON fi @@ -64,13 +67,13 @@ if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)" fi -if ! [ -z ${HADOOP_CONF_DIR+x} ]; then +if ! [ -z "${HADOOP_CONF_DIR+x}" ]; then SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH"; fi -if ! [ -z ${SPARK_CONF_DIR+x} ]; then +if ! [ -z "${SPARK_CONF_DIR+x}" ]; then SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH"; -elif ! [ -z ${SPARK_HOME+x} ]; then +elif ! [ -z "${SPARK_HOME+x}" ]; then SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH"; fi @@ -99,17 +102,17 @@ case "$1" in CMD=( ${JAVA_HOME}/bin/java "${SPARK_EXECUTOR_JAVA_OPTS[@]}" - -Xms$SPARK_EXECUTOR_MEMORY - -Xmx$SPARK_EXECUTOR_MEMORY + -Xms"$SPARK_EXECUTOR_MEMORY" + -Xmx"$SPARK_EXECUTOR_MEMORY" -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH" org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend - --driver-url $SPARK_DRIVER_URL - --executor-id $SPARK_EXECUTOR_ID - --cores $SPARK_EXECUTOR_CORES - --app-id $SPARK_APPLICATION_ID - --hostname $SPARK_EXECUTOR_POD_IP - --resourceProfileId $SPARK_RESOURCE_PROFILE_ID - --podName $SPARK_EXECUTOR_POD_NAME + --driver-url "$SPARK_DRIVER_URL" + --executor-id "$SPARK_EXECUTOR_ID" + --cores "$SPARK_EXECUTOR_CORES" + --app-id "$SPARK_APPLICATION_ID" + --hostname "$SPARK_EXECUTOR_POD_IP" + --resourceProfileId "$SPARK_RESOURCE_PROFILE_ID" + --podName "$SPARK_EXECUTOR_POD_NAME" ) attempt_setup_fake_passwd_entry # Execute the container CMD under tini for better hygiene diff --git a/3.4.1/scala2.12-java11-ubuntu/entrypoint.sh b/3.4.1/scala2.12-java11-ubuntu/entrypoint.sh index 08fc925..2e3d2a8 100755 --- a/3.4.1/scala2.12-java11-ubuntu/entrypoint.sh +++ b/3.4.1/scala2.12-java11-ubuntu/entrypoint.sh @@ -15,6 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# Prevent any errors from being silently ignored +set -eo pipefail + attempt_setup_fake_passwd_entry() { # Check whether there is a passwd entry for the container UID local myuid; myuid="$(id -u)" @@ -51,10 +54,10 @@ if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH" fi -if ! [ -z ${PYSPARK_PYTHON+x} ]; then +if ! [ -z "${PYSPARK_PYTHON+x}" ]; then export PYSPARK_PYTHON fi -if ! [ -z ${PYSPARK_DRIVER_PYTHON+x} ]; then +if ! [ -z "${PYSPARK_DRIVER_PYTHON+x}" ]; then export PYSPARK_DRIVER_PYTHON fi @@ -64,13 +67,13 @@ if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)" fi -if ! [ -z ${HADOOP_CONF_DIR+x} ]; then +if ! [ -z "${HADOOP_CONF_DIR+x}" ]; then SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH"; fi -if ! [ -z ${SPARK_CONF_DIR+x} ]; then +if ! [ -z "${SPARK_CONF_DIR+x}" ]; then SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH"; -elif ! [ -z ${SPARK_HOME+x} ]; then +elif ! [ -z "${SPARK_HOME+x}" ]; then SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH"; fi @@ -99,17 +102,17 @@ case "$1" in CMD=( ${JAVA_HOME}/bin/java "${SPARK_EXECUTOR_JAVA_OPTS[@]}" - -Xms$SPARK_EXECUTOR_MEMORY - -Xmx$SPARK_EXECUTOR_MEMORY + -Xms"$SPARK_EXECUTOR_MEMORY" + -Xmx"$SPARK_EXECUTOR_MEMORY" -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH" org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend - --driver-url $SPARK_DRIVER_URL - --executor-id $SPARK_EXECUTOR_ID - --cores $SPARK_EXECUTOR_CORES - --app-id $SPARK_APPLICATION_ID - --hostname $SPARK_EXECUTOR_POD_IP - --resourceProfileId $SPARK_RESOURCE_PROFILE_ID - --podName $SPARK_EXECUTOR_POD_NAME + --driver-url "$SPARK_DRIVER_URL" + --executor-id "$SPARK_EXECUTOR_ID" + --cores "$SPARK_EXECUTOR_CORES" + --app-id "$SPARK_APPLICATION_ID" + --hostname "$SPARK_EXECUTOR_POD_IP" + --resourceProfileId "$SPARK_RESOURCE_PROFILE_ID" + --podName "$SPARK_EXECUTOR_POD_NAME" ) attempt_setup_fake_passwd_entry # Execute the container CMD under tini for better hygiene diff --git a/entrypoint.sh.template b/entrypoint.sh.template index 08fc925..2e3d2a8 100644 --- a/entrypoint.sh.template +++ b/entrypoint.sh.template @@ -15,6 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# Prevent any errors from being silently ignored +set -eo pipefail + attempt_setup_fake_passwd_entry() { # Check whether there is a passwd entry for the container UID local myuid; myuid="$(id -u)" @@ -51,10 +54,10 @@ if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH" fi -if ! [ -z ${PYSPARK_PYTHON+x} ]; then +if ! [ -z "${PYSPARK_PYTHON+x}" ]; then export PYSPARK_PYTHON fi -if ! [ -z ${PYSPARK_DRIVER_PYTHON+x} ]; then +if ! [ -z "${PYSPARK_DRIVER_PYTHON+x}" ]; then export PYSPARK_DRIVER_PYTHON fi @@ -64,13 +67,13 @@ if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)" fi -if ! [ -z ${HADOOP_CONF_DIR+x} ]; then +if ! [ -z "${HADOOP_CONF_DIR+x}" ]; then SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH"; fi -if ! [ -z ${SPARK_CONF_DIR+x} ]; then +if ! [ -z "${SPARK_CONF_DIR+x}" ]; then SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH"; -elif ! [ -z ${SPARK_HOME+x} ]; then +elif ! [ -z "${SPARK_HOME+x}" ]; then SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH"; fi @@ -99,17 +102,17 @@ case "$1" in CMD=( ${JAVA_HOME}/bin/java "${SPARK_EXECUTOR_JAVA_OPTS[@]}" - -Xms$SPARK_EXECUTOR_MEMORY - -Xmx$SPARK_EXECUTOR_MEMORY + -Xms"$SPARK_EXECUTOR_MEMORY" + -Xmx"$SPARK_EXECUTOR_MEMORY" -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH" org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend - --driver-url $SPARK_DRIVER_URL - --executor-id $SPARK_EXECUTOR_ID - --cores $SPARK_EXECUTOR_CORES - --app-id $SPARK_APPLICATION_ID - --hostname $SPARK_EXECUTOR_POD_IP - --resourceProfileId $SPARK_RESOURCE_PROFILE_ID - --podName $SPARK_EXECUTOR_POD_NAME + --driver-url "$SPARK_DRIVER_URL" + --executor-id "$SPARK_EXECUTOR_ID" + --cores "$SPARK_EXECUTOR_CORES" + --app-id "$SPARK_APPLICATION_ID" + --hostname "$SPARK_EXECUTOR_POD_IP" + --resourceProfileId "$SPARK_RESOURCE_PROFILE_ID" + --podName "$SPARK_EXECUTOR_POD_NAME" ) attempt_setup_fake_passwd_entry # Execute the container CMD under tini for better hygiene --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org