Repository: zeppelin Updated Branches: refs/heads/master e71fac375 -> 343fd178e
ZEPPELIN-3668. Can't hide Spark Jobs (Spark UI) button ### What is this PR for? This is to fix the bug of unable to hide spark jobs. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3668 ### How should this be tested? * Unit test added * Verify it manually as well. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang <zjf...@apache.org> Closes #3104 from zjffdu/ZEPPELIN-3668 and squashes the following commits: c7ddecc6e [Jeff Zhang] ZEPPELIN-3668. Can't hide Spark Jobs (Spark UI) button Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/343fd178 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/343fd178 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/343fd178 Branch: refs/heads/master Commit: 343fd178edf85bb7880ebd4fcecf0b11a7f38561 Parents: e71fac3 Author: Jeff Zhang <zjf...@apache.org> Authored: Tue Jul 31 09:04:57 2018 +0800 Committer: Jeff Zhang <zjf...@apache.org> Committed: Wed Aug 1 08:33:36 2018 +0800 ---------------------------------------------------------------------- .../zeppelin/spark/NewSparkInterpreterTest.java | 34 +++++++++++++++++--- .../apache/zeppelin/spark/SparkShimsTest.java | 5 ++- .../org/apache/zeppelin/spark/SparkShims.java | 22 +++++-------- .../org/apache/zeppelin/spark/Spark1Shims.java | 4 ++- .../org/apache/zeppelin/spark/Spark2Shims.java | 4 ++- 5 files changed, 45 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/343fd178/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java ---------------------------------------------------------------------- diff --git a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java index 48be45b..59b39dd 100644 --- a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java +++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java @@ -44,7 +44,6 @@ import java.net.URL; import java.nio.channels.Channels; import java.nio.channels.ReadableByteChannel; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -53,6 +52,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -163,7 +163,7 @@ public class NewSparkInterpreterTest { result = interpreter.interpret( "case class Bank(age:Integer, job:String, marital : String, education : String, balance : Integer)\n" + - "val bank = bankText.map(s=>s.split(\";\")).filter(s => s(0)!=\"\\\"age\\\"\").map(\n" + + "val bank = bankText.map(s=>s.split(\";\")).filter(s => s(0)!=\"\\\"age\\\"\").map(\n" + " s => Bank(s(0).toInt, \n" + " s(1).replaceAll(\"\\\"\", \"\"),\n" + " s(2).replaceAll(\"\\\"\", \"\"),\n" + @@ -188,7 +188,7 @@ public class NewSparkInterpreterTest { "df.show()", getInterpreterContext()); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertTrue(output.contains( - "+---+----+\n" + + "+---+----+\n" + "| _1| _2|\n" + "+---+----+\n" + "| 1| a|\n" + @@ -203,7 +203,7 @@ public class NewSparkInterpreterTest { "df.show()", getInterpreterContext()); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertTrue(output.contains( - "+---+----+\n" + + "+---+----+\n" + "| _1| _2|\n" + "+---+----+\n" + "| 1| a|\n" + @@ -318,7 +318,7 @@ public class NewSparkInterpreterTest { interpretThread.start(); boolean nonZeroProgress = false; int progress = 0; - while(interpretThread.isAlive()) { + while (interpretThread.isAlive()) { progress = interpreter.getProgress(context2); assertTrue(progress >= 0); if (progress != 0 && progress != 100) { @@ -463,6 +463,30 @@ public class NewSparkInterpreterTest { assertEquals(null, interpreter.getSparkContext().getLocalProperty("spark.scheduler.pool")); } + @Test + public void testDisableSparkUI() throws InterpreterException { + Properties properties = new Properties(); + properties.setProperty("spark.master", "local"); + properties.setProperty("spark.app.name", "test"); + properties.setProperty("zeppelin.spark.maxResult", "100"); + properties.setProperty("zeppelin.spark.test", "true"); + properties.setProperty("zeppelin.spark.useNew", "true"); + properties.setProperty("spark.ui.enabled", "false"); + + interpreter = new SparkInterpreter(properties); + assertTrue(interpreter.getDelegation() instanceof NewSparkInterpreter); + interpreter.setInterpreterGroup(mock(InterpreterGroup.class)); + InterpreterContext.set(getInterpreterContext()); + interpreter.open(); + + InterpreterContext context = getInterpreterContext(); + InterpreterResult result = interpreter.interpret("sc.range(1, 10).sum", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + + // spark job url is not sent + verify(mockRemoteEventClient, never()).onParaInfosReceived(any(Map.class)); + } + @After public void tearDown() throws InterpreterException { if (this.interpreter != null) { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/343fd178/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkShimsTest.java ---------------------------------------------------------------------- diff --git a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkShimsTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkShimsTest.java index ccebac3..fd47ce2 100644 --- a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkShimsTest.java +++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkShimsTest.java @@ -108,7 +108,6 @@ public class SparkShimsTest { @PrepareForTest({BaseZeppelinContext.class, VersionInfo.class}) @PowerMockIgnore({"javax.net.*", "javax.security.*"}) public static class SingleTests { - @Mock Properties mockProperties; @Captor ArgumentCaptor<Map<String, String>> argumentCaptor; SparkShims sparkShims; @@ -130,7 +129,7 @@ public class SparkShimsTest { @Test public void runUnderLocalTest() { - sparkShims.buildSparkJobUrl("local", "http://sparkurl", 0, mockProperties, mockContext); + sparkShims.buildSparkJobUrl("local", "http://sparkurl", 0, mockContext); Map<String, String> mapValue = argumentCaptor.getValue(); assertTrue(mapValue.keySet().contains("jobUrl")); @@ -140,7 +139,7 @@ public class SparkShimsTest { @Test public void runUnderYarnTest() { - sparkShims.buildSparkJobUrl("yarn", "http://sparkurl", 0, mockProperties, mockContext); + sparkShims.buildSparkJobUrl("yarn", "http://sparkurl", 0, mockContext); Map<String, String> mapValue = argumentCaptor.getValue(); assertTrue(mapValue.keySet().contains("jobUrl")); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/343fd178/spark/spark-shims/src/main/scala/org/apache/zeppelin/spark/SparkShims.java ---------------------------------------------------------------------- diff --git a/spark/spark-shims/src/main/scala/org/apache/zeppelin/spark/SparkShims.java b/spark/spark-shims/src/main/scala/org/apache/zeppelin/spark/SparkShims.java index d308762..1287c57 100644 --- a/spark/spark-shims/src/main/scala/org/apache/zeppelin/spark/SparkShims.java +++ b/spark/spark-shims/src/main/scala/org/apache/zeppelin/spark/SparkShims.java @@ -104,26 +104,20 @@ public abstract class SparkShims { protected void buildSparkJobUrl(String master, String sparkWebUrl, int jobId, - Properties jobProperties, InterpreterContext context) { - String uiEnabled = jobProperties.getProperty("spark.ui.enabled"); String jobUrl = sparkWebUrl + "/jobs/job?id=" + jobId; - // Button visible if Spark UI property not set, set as invalid boolean or true - boolean showSparkUI = - uiEnabled == null || !uiEnabled.trim().toLowerCase().equals("false"); String version = VersionInfo.getVersion(); if (master.toLowerCase().contains("yarn") && !supportYarn6615(version)) { jobUrl = sparkWebUrl + "/jobs"; } - if (showSparkUI && jobUrl != null) { - Map<String, String> infos = new java.util.HashMap<String, String>(); - infos.put("jobUrl", jobUrl); - infos.put("label", "SPARK JOB"); - infos.put("tooltip", "View in Spark web UI"); - infos.put("noteId", context.getNoteId()); - infos.put("paraId", context.getParagraphId()); - context.getIntpEventClient().onParaInfosReceived(infos); - } + + Map<String, String> infos = new java.util.HashMap<String, String>(); + infos.put("jobUrl", jobUrl); + infos.put("label", "SPARK JOB"); + infos.put("tooltip", "View in Spark web UI"); + infos.put("noteId", context.getNoteId()); + infos.put("paraId", context.getParagraphId()); + context.getIntpEventClient().onParaInfosReceived(infos); } /** http://git-wip-us.apache.org/repos/asf/zeppelin/blob/343fd178/spark/spark1-shims/src/main/scala/org/apache/zeppelin/spark/Spark1Shims.java ---------------------------------------------------------------------- diff --git a/spark/spark1-shims/src/main/scala/org/apache/zeppelin/spark/Spark1Shims.java b/spark/spark1-shims/src/main/scala/org/apache/zeppelin/spark/Spark1Shims.java index db0727c..3981204 100644 --- a/spark/spark1-shims/src/main/scala/org/apache/zeppelin/spark/Spark1Shims.java +++ b/spark/spark1-shims/src/main/scala/org/apache/zeppelin/spark/Spark1Shims.java @@ -38,7 +38,9 @@ public class Spark1Shims extends SparkShims { sc.addSparkListener(new JobProgressListener(sc.getConf()) { @Override public void onJobStart(SparkListenerJobStart jobStart) { - buildSparkJobUrl(master, sparkWebUrl, jobStart.jobId(), jobStart.properties(), context); + if (sc.getConf().getBoolean("spark.ui.enabled", true)) { + buildSparkJobUrl(master, sparkWebUrl, jobStart.jobId(), context); + } } }); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/343fd178/spark/spark2-shims/src/main/scala/org/apache/zeppelin/spark/Spark2Shims.java ---------------------------------------------------------------------- diff --git a/spark/spark2-shims/src/main/scala/org/apache/zeppelin/spark/Spark2Shims.java b/spark/spark2-shims/src/main/scala/org/apache/zeppelin/spark/Spark2Shims.java index 177b0ac..5f0cf87 100644 --- a/spark/spark2-shims/src/main/scala/org/apache/zeppelin/spark/Spark2Shims.java +++ b/spark/spark2-shims/src/main/scala/org/apache/zeppelin/spark/Spark2Shims.java @@ -39,7 +39,9 @@ public class Spark2Shims extends SparkShims { sc.addSparkListener(new SparkListener() { @Override public void onJobStart(SparkListenerJobStart jobStart) { - buildSparkJobUrl(master, sparkWebUrl, jobStart.jobId(), jobStart.properties(), context); + if (sc.getConf().getBoolean("spark.ui.enabled", true)) { + buildSparkJobUrl(master, sparkWebUrl, jobStart.jobId(), context); + } } }); }