Repository: zeppelin Updated Branches: refs/heads/master 43926485c -> 380cfbfe3
[ZEPPELIN-2589]Let zeppelin show more than one line of error log returned by livy. ### What is this PR for? Let zeppelin show more than one line of error log returned by livy. ### What type of PR is it? Improvement ### What is the Jira issue? [ZEPPELIN-2589 ](https://issues.apache.org/jira/browse/ZEPPELIN-2589) ### How should this be tested? 1. build: mvn clean package -DskipTests; 2. /bin/zeppelin-daemon.sh restart 3. Start a notebook 4. Use Livy interpreter and input some erroneous spark code, check the shown result is more than one line ### Screenshots (if appropriate) When using Livy 0.4 to run spark streaming job. Original Zeppelin:  After add traceback to interpreter result:  ### Questions: * Does the licenses files need update?No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: æ±ªèµ«æ¬ <qwemich...@gmail.com> Closes #2390 from qwemicheal/master and squashes the following commits: ee3b42c [汪赫æ¬] ci fail randomly 6e4d2aa [汪赫æ¬] trigger ci c7e9b0e [汪赫æ¬] minor change to trigger ci 0b657a8 [汪赫æ¬] add check for traceback and modify way of checking msg number 4ea3edb [汪赫æ¬] merger evalue and traceback to one message and use join to concatenate traceback b1caa2c [汪赫æ¬] minor change e729db2 [汪赫æ¬] merge unit test to pyspark test case 774b370 [汪赫æ¬] use stringBuilder to concat traceback lines f8de47a [汪赫æ¬] code style fixed f96abbb [汪赫æ¬] only use pyspark de2e06b [汪赫æ¬] close the pyspark interpreter after test 0212e35 [汪赫æ¬] change to pyspark for livy 0.2 when testing traceback msg 799f1c2 [汪赫æ¬] take livy livy version into consideration e3f9791 [汪赫æ¬] add unit test bf26dbe [汪赫æ¬] whitespace fitting f6fc200 [汪赫æ¬] fix traceback type mismatch 3310055 [汪赫æ¬] add try catch to find which string failed ci e306d4f [汪赫æ¬] add white space after comma 3307c6f [汪赫æ¬] add more debug msg 6a2983f [汪赫æ¬] fix typo 04061d3 [汪赫æ¬] add debug print 5a03e05 [汪赫æ¬] add check for livy 0.2 when traceback field is missing 45c5d9f [汪赫æ¬] add traceback msg from livy return json to InterpreterResult f6135cc [汪赫æ¬] add traceback msg from livy return json to InterpreterResult 2eb6011 [汪赫æ¬] add traceback msg from livy return json to InterpreterResult 905c54d [汪赫æ¬] add traceback msg from livy return json to InterpreterResult Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/380cfbfe Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/380cfbfe Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/380cfbfe Branch: refs/heads/master Commit: 380cfbfe3cdef7e9281ad39b0b7e2cd3b161f72b Parents: 4392648 Author: æ±ªèµ«æ¬ <qwemich...@gmail.com> Authored: Wed Jun 7 21:31:40 2017 +0800 Committer: Jeff Zhang <zjf...@apache.org> Committed: Sat Jun 10 09:46:54 2017 +0800 ---------------------------------------------------------------------- .../zeppelin/livy/BaseLivyInterpreter.java | 27 +++++++++++++++++--- .../apache/zeppelin/livy/LivyInterpreterIT.java | 21 ++++++++++++++- 2 files changed, 44 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/380cfbfe/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java ---------------------------------------------------------------------- diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java index fcb7e9d..523dfad 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java @@ -341,7 +341,17 @@ public abstract class BaseLivyInterpreter extends Interpreter { private InterpreterResult getResultFromStatementInfo(StatementInfo stmtInfo, boolean displayAppInfo) { if (stmtInfo.output != null && stmtInfo.output.isError()) { - return new InterpreterResult(InterpreterResult.Code.ERROR, stmtInfo.output.evalue); + InterpreterResult result = new InterpreterResult(InterpreterResult.Code.ERROR); + StringBuilder sb = new StringBuilder(); + sb.append(stmtInfo.output.evalue); + // in case evalue doesn't have newline char + if (!stmtInfo.output.evalue.contains("\n")) + sb.append("\n"); + if (stmtInfo.output.traceback != null) { + sb.append(StringUtils.join(stmtInfo.output.traceback)); + } + result.add(sb.toString()); + return result; } else if (stmtInfo.isCancelled()) { // corner case, output might be null if it is cancelled. return new InterpreterResult(InterpreterResult.Code.ERROR, "Job is cancelled"); @@ -659,7 +669,18 @@ public abstract class BaseLivyInterpreter extends Interpreter { } public static StatementInfo fromJson(String json) { - return gson.fromJson(json, StatementInfo.class); + String right_json = ""; + try { + gson.fromJson(json, StatementInfo.class); + right_json = json; + } catch (Exception e) { + if (json.contains("\"traceback\":{}")) { + LOGGER.debug("traceback type mismatch, replacing the mismatching part "); + right_json = json.replace("\"traceback\":{}", "\"traceback\":[]"); + LOGGER.debug("new json string is {}", right_json); + } + } + return gson.fromJson(right_json, StatementInfo.class); } public boolean isAvailable() { @@ -676,7 +697,7 @@ public abstract class BaseLivyInterpreter extends Interpreter { public Data data; public String ename; public String evalue; - public Object traceback; + public String[] traceback; public TableMagic tableMagic; public boolean isError() { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/380cfbfe/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java ---------------------------------------------------------------------- diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java index 80965ff..824eac3 100644 --- a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java +++ b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java @@ -523,7 +523,7 @@ public class LivyInterpreterIT { } @Test - public void testPySparkInterpreter() { + public void testPySparkInterpreter() throws LivyException { if (!checkPreCondition()) { return; } @@ -536,6 +536,24 @@ public class LivyInterpreterIT { "title", "text", authInfo, null, null, null, null, null, output); pysparkInterpreter.open(); + // test traceback msg + try { + pysparkInterpreter.getLivyVersion(); + // for livy version >=0.3 , input some erroneous spark code, check the shown result is more than one line + InterpreterResult result = pysparkInterpreter.interpret("sc.parallelize(wrongSyntax(1, 2)).count()", context); + assertEquals(InterpreterResult.Code.ERROR, result.code()); + assertTrue(result.message().get(0).getData().split("\n").length>1); + assertTrue(result.message().get(0).getData().contains("Traceback")); + } catch (APINotFoundException e) { + // only livy 0.2 can throw this exception since it doesn't have /version endpoint + // in livy 0.2, most error msg is encapsulated in evalue field, only print(a) in pyspark would return none-empty + // traceback + InterpreterResult result = pysparkInterpreter.interpret("print(a)", context); + assertEquals(InterpreterResult.Code.ERROR, result.code()); + assertTrue(result.message().get(0).getData().split("\n").length>1); + assertTrue(result.message().get(0).getData().contains("Traceback")); + } + try { InterpreterResult result = pysparkInterpreter.interpret("sc.version", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); @@ -763,6 +781,7 @@ public class LivyInterpreterIT { } } + private boolean isSpark2(BaseLivyInterpreter interpreter, InterpreterContext context) { InterpreterResult result = null; if (interpreter instanceof LivySparkRInterpreter) {
