Repository: zeppelin Updated Branches: refs/heads/master adf83a3f5 -> 31696b788
ZEPPELIN-3556. set spark.yarn.submit.waitAppCompletion to false in yarn-cluster mode ### What is this PR for? Set `spark.yarn.submit.waitAppCompletion` to false in yarn-cluster mode, so that in the interpreter log we won't see many logs of monitor yarn app status which is useless. ### What type of PR is it? [Improvement] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3556 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang <zjf...@apache.org> Closes #3038 from zjffdu/ZEPPELIN-3556 and squashes the following commits: 7474987a6 [Jeff Zhang] ZEPPELIN-3556. set spark.yarn.submit.waitAppCompletion to false in yarn-cluster mode Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/31696b78 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/31696b78 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/31696b78 Branch: refs/heads/master Commit: 31696b7889799d47eb36160fa3288ed05cb690a8 Parents: adf83a3 Author: Jeff Zhang <zjf...@apache.org> Authored: Wed Jun 20 14:56:11 2018 +0800 Committer: Jeff Zhang <zjf...@apache.org> Committed: Tue Jun 26 08:25:36 2018 +0800 ---------------------------------------------------------------------- .../zeppelin/interpreter/launcher/SparkInterpreterLauncher.java | 1 + .../interpreter/launcher/SparkInterpreterLauncherTest.java | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/31696b78/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java index 688d95f..ff65e0d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java @@ -58,6 +58,7 @@ public class SparkInterpreterLauncher extends ShellScriptLauncher { setupPropertiesForSparkR(sparkProperties); if (isYarnMode() && getDeployMode().equals("cluster")) { env.put("ZEPPELIN_SPARK_YARN_CLUSTER", "true"); + sparkProperties.setProperty("spark.yarn.submit.waitAppCompletion", "false"); } StringBuilder sparkConfBuilder = new StringBuilder(); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/31696b78/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java index 7d8b002..c2abd60 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java @@ -159,7 +159,7 @@ public class SparkInterpreterLauncherTest { assertTrue(interpreterProcess.getEnv().size() >= 3); assertEquals("/user/spark", interpreterProcess.getEnv().get("SPARK_HOME")); assertEquals("true", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_YARN_CLUSTER")); - assertEquals(" --master yarn-cluster --files .//conf/log4j_yarn_cluster.properties --conf spark.files='file_1' --conf spark.jars='jar_1' --conf spark.yarn.isPython=true", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF")); + assertEquals(" --master yarn-cluster --files .//conf/log4j_yarn_cluster.properties --conf spark.files='file_1' --conf spark.jars='jar_1' --conf spark.yarn.isPython=true --conf spark.yarn.submit.waitAppCompletion=false", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF")); } @Test @@ -187,6 +187,6 @@ public class SparkInterpreterLauncherTest { assertTrue(interpreterProcess.getEnv().size() >= 3); assertEquals("/user/spark", interpreterProcess.getEnv().get("SPARK_HOME")); assertEquals("true", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_YARN_CLUSTER")); - assertEquals(" --master yarn --files .//conf/log4j_yarn_cluster.properties --conf spark.files='file_1' --conf spark.jars='jar_1' --conf spark.submit.deployMode='cluster' --conf spark.yarn.isPython=true --proxy-user user1", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF")); + assertEquals(" --master yarn --files .//conf/log4j_yarn_cluster.properties --conf spark.files='file_1' --conf spark.jars='jar_1' --conf spark.submit.deployMode='cluster' --conf spark.yarn.isPython=true --conf spark.yarn.submit.waitAppCompletion=false --proxy-user user1", interpreterProcess.getEnv().get("ZEPPELIN_SPARK_CONF")); } }
