This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new b870ee6 [ZEPPELIN-5629] spark.driver.extraJavaOptions setting in
%spark.conf does not work well
b870ee6 is described below
commit b870ee69a0751d2eeefe43cfb22fa1acc5ef8a12
Author: huage1994 <[email protected]>
AuthorDate: Mon Jan 17 18:08:24 2022 +0800
[ZEPPELIN-5629] spark.driver.extraJavaOptions setting in %spark.conf does
not work well
### What is this PR for?
The value of `spark.driver.extraJavaOptions` setting in `%spark.conf` will
override the its initial value provided by zeppelin. It should append instead
of override.
This PR fix this bug.
### What type of PR is it?
[Bug Fix]
### Todos
* [ ] - Task
### What is the Jira issue?
* Open an issue on Jira https://issues.apache.org/jira/browse/ZEPPELIN-5629
### How should this be tested?
* CI passed
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: huage1994 <[email protected]>
Closes #4289 from huage1994/ZEPPELIN-5629 and squashes the following
commits:
24805331fb [huage1994] [ZEPPELIN-5629] spark.driver.extraJavaOptions
setting in %spark.conf does not work well
---
bin/interpreter.sh | 4 ++--
.../interpreter/launcher/SparkInterpreterLauncher.java | 16 ++++++++++------
2 files changed, 12 insertions(+), 8 deletions(-)
diff --git a/bin/interpreter.sh b/bin/interpreter.sh
index 6d9c048..baeaa5b 100755
--- a/bin/interpreter.sh
+++ b/bin/interpreter.sh
@@ -300,9 +300,9 @@ if [[ -n "${SPARK_SUBMIT}" ]]; then
IFS=' ' read -r -a SPARK_SUBMIT_OPTIONS_ARRAY <<< "${SPARK_SUBMIT_OPTIONS}"
IFS='|' read -r -a ZEPPELIN_SPARK_CONF_ARRAY <<< "${ZEPPELIN_SPARK_CONF}"
if [[ "${ZEPPELIN_SPARK_YARN_CLUSTER}" == "true" ]]; then
- INTERPRETER_RUN_COMMAND+=("${SPARK_SUBMIT}" "--class" "${ZEPPELIN_SERVER}"
"--driver-java-options" "${JAVA_INTP_OPTS}" "${SPARK_SUBMIT_OPTIONS_ARRAY[@]}"
"${ZEPPELIN_SPARK_CONF_ARRAY[@]}" "${SPARK_APP_JAR}" "${CALLBACK_HOST}"
"${PORT}" "${INTP_GROUP_ID}" "${INTP_PORT}")
+ INTERPRETER_RUN_COMMAND+=("${SPARK_SUBMIT}" "--class" "${ZEPPELIN_SERVER}"
"--driver-java-options" "${SPARK_DRIVER_EXTRAJAVAOPTIONS_CONF}
${JAVA_INTP_OPTS}" "${SPARK_SUBMIT_OPTIONS_ARRAY[@]}"
"${ZEPPELIN_SPARK_CONF_ARRAY[@]}" "${SPARK_APP_JAR}" "${CALLBACK_HOST}"
"${PORT}" "${INTP_GROUP_ID}" "${INTP_PORT}")
else
- INTERPRETER_RUN_COMMAND+=("${SPARK_SUBMIT}" "--class" "${ZEPPELIN_SERVER}"
"--driver-class-path"
"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}"
"--driver-java-options" "${JAVA_INTP_OPTS}" "${SPARK_SUBMIT_OPTIONS_ARRAY[@]}"
"${ZEPPELIN_SPARK_CONF_ARRAY[@]}" "${SPARK_APP_JAR}" "${CALLBACK_HOST}"
"${PORT}" "${INTP_GROUP_ID}" "${INTP_PORT}")
+ INTERPRETER_RUN_COMMAND+=("${SPARK_SUBMIT}" "--class" "${ZEPPELIN_SERVER}"
"--driver-class-path"
"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}"
"--driver-java-options" "${SPARK_DRIVER_EXTRAJAVAOPTIONS_CONF}
${JAVA_INTP_OPTS}" "${SPARK_SUBMIT_OPTIONS_ARRAY[@]}"
"${ZEPPELIN_SPARK_CONF_ARRAY[@]}" "${SPARK_APP_JAR}" "${CALLBACK_HOST}"
"${PORT}" "${INTP_GROUP_ID}" "${INTP_PORT}")
fi
elif [[ -n "${ZEPPELIN_FLINK_APPLICATION_MODE}" ]]; then
IFS='|' read -r -a ZEPPELIN_FLINK_APPLICATION_MODE_CONF_ARRAY <<<
"${ZEPPELIN_FLINK_APPLICATION_MODE_CONF}"
diff --git
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
index 239976d..02f481d 100644
---
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
+++
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
@@ -27,10 +27,7 @@ import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-import java.util.StringJoiner;
+import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -46,8 +43,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.util.Map;
-import java.util.Properties;
/**
* Spark specific launcher.
@@ -77,6 +72,15 @@ public class SparkInterpreterLauncher extends
StandardInterpreterLauncher {
env.put(key, propValue);
}
if (isSparkConf(key, propValue)) {
+ // There is already initial value following --driver-java-options when
SparkInterpreter launches.
+ // Values in sparkProperties would be added by --conf,
+ // and --conf spark.driver.extraJavaOptions would conflict with
--driver-java-options.
+ // Therefore we add values of spark.driver.extraJavaOptions following
--driver-java-options
+ // instead of into sparkProperties.
+ if (Objects.equals("spark.driver.extraJavaOptions", key)) {
+ env.put("SPARK_DRIVER_EXTRAJAVAOPTIONS_CONF", (String)
properties.remove(key));
+ continue;
+ }
sparkProperties.setProperty(key, propValue);
}
}
