Repository: incubator-zeppelin Updated Branches: refs/heads/master 831f426db -> 9b72c8940
selenium test spark, pyspark and sparkSql ### What is this PR for? Test functionality of spark, pyspark, sparksql ### What type of PR is it? Improvement ### Todos * [x] - Selenium for spark * [x] - Selenium for pyspark * [x] - Selenium for sparksql * [x] - refactor with https://github.com/apache/incubator-zeppelin/pull/619 ### Is there a relevant Jira issue? ZEPPELIN-587 ### How should this be tested? On macOS PATH=~/Applications/Firefox.app/Contents/MacOS/:$PATH CI="" \ mvn -Dtest=org.apache.zeppelin.integration.TestSparkParagraph -Denforcer.skip=true \ test -pl zeppelin-server Author: Prabhjyot Singh <[email protected]> Closes #654 from prabhjyotsingh/ZEPPELIN-587 and squashes the following commits: 8f24695 [Prabhjyot Singh] use handleException in all other test cases remove test for spark 1.1.1 more meaningful log message 28dfc55 [Prabhjyot Singh] thorwong exception similar to https://github.com/apache/incubator-zeppelin/pull/709 21bcc45 [Prabhjyot Singh] Merge remote-tracking branch 'origin/master' into ZEPPELIN-587 b05b81b [Prabhjyot Singh] have SHIFT_ENTER enum 9a206f4 [Prabhjyot Singh] add missing endToEndTestEnabled check for testSparkInterpreterDependencyLoading ab03287 [Prabhjyot Singh] have static import of AbstractZeppelinIT.HelperKeys.*, and rg.openqa.selenium.Keys.* 5187a16 [Prabhjyot Singh] check if spark version is less than 1.3 then don't append ".toDF()" 6927f7e [Prabhjyot Singh] CI FIX 9236e3c [Prabhjyot Singh] implemeting @bzz review comments 2c28758 [Prabhjyot Singh] missed refactor with #619 d49344f [Prabhjyot Singh] selenium test spark, pyspark and sparkSql Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/9b72c894 Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/9b72c894 Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/9b72c894 Branch: refs/heads/master Commit: 9b72c894017880bd4a2a0c018c6f9d9b42a5d51d Parents: 831f426 Author: Prabhjyot Singh <[email protected]> Authored: Wed Feb 17 11:23:52 2016 +0530 Committer: Lee moon soo <[email protected]> Committed: Sun Feb 21 10:50:33 2016 -0800 ---------------------------------------------------------------------- .../org/apache/zeppelin/AbstractZeppelinIT.java | 46 +++- .../integration/ParagraphActionsIT.java | 8 +- .../zeppelin/integration/SparkParagraphIT.java | 228 +++++++++++++++++++ .../apache/zeppelin/integration/ZeppelinIT.java | 8 +- 4 files changed, 276 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/9b72c894/zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java b/zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java index 29569fc..2b68356 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java @@ -19,8 +19,6 @@ package org.apache.zeppelin; import com.google.common.base.Function; -import org.junit.After; -import org.junit.Before; import org.openqa.selenium.*; import org.openqa.selenium.support.ui.ExpectedConditions; import org.openqa.selenium.support.ui.FluentWait; @@ -29,10 +27,14 @@ import org.openqa.selenium.support.ui.WebDriverWait; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.util.LinkedList; import java.util.List; import java.util.concurrent.TimeUnit; +import static org.openqa.selenium.Keys.ENTER; +import static org.openqa.selenium.Keys.SHIFT; + abstract public class AbstractZeppelinIT { protected WebDriver driver; @@ -131,4 +133,44 @@ abstract public class AbstractZeppelinIT { sleep(100, true); } + public enum HelperKeys implements CharSequence { + OPEN_PARENTHESIS(Keys.chord(Keys.SHIFT, "9")), + EXCLAMATION(Keys.chord(Keys.SHIFT, "1")), + PERCENTAGE(Keys.chord(Keys.SHIFT, "5")), + SHIFT_ENTER(Keys.chord(SHIFT, ENTER)); + + private final CharSequence keyCode; + + HelperKeys(CharSequence keyCode) { + this.keyCode = keyCode; + } + + public char charAt(int index) { + return index == 0 ? keyCode.charAt(index) : '\ue000'; + } + + public int length() { + return 1; + } + + public CharSequence subSequence(int start, int end) { + if (start == 0 && end == 1) { + return String.valueOf(this.keyCode); + } else { + throw new IndexOutOfBoundsException(); + } + } + + public String toString() { + return String.valueOf(this.keyCode); + } + + } + + protected void handleException(String message, Exception e) throws Exception { + LOG.error(message, e); + File scrFile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE); + throw e; + } + } http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/9b72c894/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java index 0af660c..ee27db1 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java @@ -227,9 +227,7 @@ public class ParagraphActionsIT extends AbstractZeppelinIT { deleteTestNotebook(driver); } catch (Exception e) { - LOG.error("Exception in ParagraphActionsIT while testMoveUpAndDown ", e); - File scrFile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE); - throw e; + handleException("Exception in ParagraphActionsIT while testMoveUpAndDown ", e); } } @@ -267,9 +265,7 @@ public class ParagraphActionsIT extends AbstractZeppelinIT { deleteTestNotebook(driver); } catch (Exception e) { - LOG.error("Exception in ParagraphActionsIT while testDisableParagraphRunButton ", e); - File scrFile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE); - throw e; + handleException("Exception in ParagraphActionsIT while testDisableParagraphRunButton ", e); } } http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/9b72c894/zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java new file mode 100644 index 0000000..e9f2bf8 --- /dev/null +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.integration; + + +import org.apache.zeppelin.AbstractZeppelinIT; +import org.apache.zeppelin.WebDriverManager; +import org.hamcrest.CoreMatchers; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ErrorCollector; +import org.openqa.selenium.By; +import org.openqa.selenium.TimeoutException; +import org.openqa.selenium.WebElement; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.zeppelin.AbstractZeppelinIT.HelperKeys.*; +import static org.openqa.selenium.Keys.*; + +public class SparkParagraphIT extends AbstractZeppelinIT { + private static final Logger LOG = LoggerFactory.getLogger(SparkParagraphIT.class); + + + @Rule + public ErrorCollector collector = new ErrorCollector(); + + @Before + public void startUp() { + if (!endToEndTestEnabled()) { + return; + } + driver = WebDriverManager.getWebDriver(); + createNewNote(); + waitForParagraph(1, "READY"); + } + + @After + public void tearDown() { + if (!endToEndTestEnabled()) { + return; + } + deleteTestNotebook(driver); + driver.quit(); + } + + @Test + public void testSpark() throws Exception { + if (!endToEndTestEnabled()) { + return; + } + try { + WebElement paragraph1Editor = driver.findElement(By.xpath(getParagraphXPath(1) + "//textarea")); + paragraph1Editor.sendKeys("sc.version"); + paragraph1Editor.sendKeys(SHIFT_ENTER); + + waitForParagraph(1, "FINISHED"); + WebElement paragraph1Result = driver.findElement(By.xpath( + getParagraphXPath(1) + "//div[@class=\"tableDisplay\"]")); + Float sparkVersion = Float.parseFloat(paragraph1Result.getText().split("= ")[1].substring(0, 3)); + + WebElement paragraph2Editor = driver.findElement(By.xpath(getParagraphXPath(2) + "//textarea")); + + + /* + equivalent of + import org.apache.commons.io.IOUtils + import java.net.URL + import java.nio.charset.Charset + val bankText = sc.parallelize(IOUtils.toString(new URL("https://s3.amazonaws.com/apache-zeppelin/tutorial/bank/bank.csv";),Charset.forName("utf8")).split("\n")) + case class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer) + + val bank = bankText.map(s => s.split(";")).filter(s => s(0) != "\"age\"").map(s => Bank(s(0).toInt,s(1).replaceAll("\"", ""),s(2).replaceAll("\"", ""),s(3).replaceAll("\"", ""),s(5).replaceAll("\"", "").toInt)).toDF() + bank.registerTempTable("bank") + */ + paragraph2Editor.sendKeys("import org.apache.commons.io.IOUtils" + + ENTER + + + "import java.net.URL" + + ENTER + + + "import java.nio.charset.Charset" + + ENTER + + + "val bankText = sc.parallelize" + OPEN_PARENTHESIS + + "IOUtils.toString" + OPEN_PARENTHESIS + "new URL" + OPEN_PARENTHESIS + + "\"https://s3.amazonaws.com/apache"; + SUBTRACT + "zeppelin/tutorial/bank/bank." + + "csv\"),Charset.forName" + OPEN_PARENTHESIS + "\"utf8\"))" + + ".split" + OPEN_PARENTHESIS + "\"\\n\"))" + + ENTER + + + "case class Bank" + OPEN_PARENTHESIS + + "age: Integer, job: String, marital: String, education: String, balance: Integer)" + + ENTER + + ENTER + + + "val bank = bankText.map" + OPEN_PARENTHESIS + "s => s.split" + + OPEN_PARENTHESIS + "\";\")).filter" + OPEN_PARENTHESIS + + "s => s" + OPEN_PARENTHESIS + "0) " + EXCLAMATION + + "= \"\\\"age\\\"\").map" + OPEN_PARENTHESIS + + "s => Bank" + OPEN_PARENTHESIS + "s" + OPEN_PARENTHESIS + + "0).toInt,s" + OPEN_PARENTHESIS + "1).replaceAll" + + OPEN_PARENTHESIS + "\"\\\"\", \"\")," + + "s" + OPEN_PARENTHESIS + "2).replaceAll" + + OPEN_PARENTHESIS + "\"\\\"\", \"\")," + + "s" + OPEN_PARENTHESIS + "3).replaceAll" + + OPEN_PARENTHESIS + "\"\\\"\", \"\")," + + "s" + OPEN_PARENTHESIS + "5).replaceAll" + + OPEN_PARENTHESIS + "\"\\\"\", \"\").toInt" + ")" + + ")" + (sparkVersion < 1.3f ? "" : ".toDF" + OPEN_PARENTHESIS + ")") + + ENTER + + + "bank.registerTempTable" + OPEN_PARENTHESIS + "\"bank\")" + ); + paragraph2Editor.sendKeys("" + END + BACK_SPACE + BACK_SPACE + + BACK_SPACE + BACK_SPACE + BACK_SPACE + BACK_SPACE + + BACK_SPACE + BACK_SPACE + BACK_SPACE + BACK_SPACE + BACK_SPACE); + + paragraph2Editor.sendKeys(SHIFT_ENTER); + + try { + waitForParagraph(2, "FINISHED"); + } catch (TimeoutException e) { + waitForParagraph(2, "ERROR"); + collector.checkThat("2nd Paragraph from SparkParagraphIT of testSpark status:", + "ERROR", CoreMatchers.equalTo("FINISHED") + ); + } + + WebElement paragraph2Result = driver.findElement(By.xpath( + getParagraphXPath(2) + "//div[@class=\"tableDisplay\"]")); + + collector.checkThat("2nd Paragraph from SparkParagraphIT of testSpark result: ", + paragraph2Result.getText().toString(), CoreMatchers.containsString( + "import org.apache.commons.io.IOUtils" + ) + ); + + } catch (Exception e) { + handleException("Exception in SparkParagraphIT while testSpark", e); + } + } + + @Test + public void testPySpark() throws Exception { + if (!endToEndTestEnabled()) { + return; + } + try { + WebElement paragraph1Editor = driver.findElement(By.xpath(getParagraphXPath(1) + "//textarea")); + + paragraph1Editor.sendKeys(PERCENTAGE + "pyspark" + ENTER + + "for x in range" + OPEN_PARENTHESIS + "0, 3):" + ENTER + + " print \"test loop " + PERCENTAGE + "d\" " + + PERCENTAGE + " " + OPEN_PARENTHESIS + "x)" + ENTER); + + paragraph1Editor.sendKeys(SHIFT_ENTER); + + try { + waitForParagraph(1, "FINISHED"); + } catch (TimeoutException e) { + waitForParagraph(1, "ERROR"); + collector.checkThat("Paragraph from SparkParagraphIT of testPySpark status: ", + "ERROR", CoreMatchers.equalTo("FINISHED") + ); + } + + WebElement paragraph1Result = driver.findElement(By.xpath( + getParagraphXPath(1) + "//div[@class=\"tableDisplay\"]")); + collector.checkThat("Paragraph from SparkParagraphIT of testPySpark result: ", + paragraph1Result.getText().toString(), CoreMatchers.equalTo("test loop 0\ntest loop 1\ntest loop 2") + ); + + } catch (Exception e) { + handleException("Exception in SparkParagraphIT while testPySpark", e); + } + } + + @Test + public void testSqlSpark() throws Exception { + if (!endToEndTestEnabled()) { + return; + } + try { + WebElement paragraph1Editor = driver.findElement(By.xpath(getParagraphXPath(1) + "//textarea")); + + paragraph1Editor.sendKeys(PERCENTAGE + "sql" + ENTER + + "select * from bank limit 1"); + + paragraph1Editor.sendKeys(SHIFT_ENTER); + + try { + waitForParagraph(1, "FINISHED"); + } catch (TimeoutException e) { + waitForParagraph(1, "ERROR"); + collector.checkThat("Paragraph from SparkParagraphIT of testSqlSpark status: ", + "ERROR", CoreMatchers.equalTo("FINISHED") + ); + } + + WebElement paragraph1Result = driver.findElement(By.xpath( + getParagraphXPath(1) + "//div[@class=\"tableDisplay\"]")); + collector.checkThat("Paragraph from SparkParagraphIT of testSqlSpark result: ", + paragraph1Result.getText().toString(), CoreMatchers.equalTo("age job marital education balance\n" + + "30 unemployed married primary 1,787") + ); + } catch (Exception e) { + handleException("Exception in SparkParagraphIT while testSqlSpark", e); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/9b72c894/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java index 4dd20e8..f01a5de 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java @@ -196,9 +196,7 @@ public class ZeppelinIT extends AbstractZeppelinIT { System.out.println("testCreateNotebook Test executed"); } catch (Exception e) { - LOG.error("Exception in ZeppelinIT while testAngularDisplay ", e); - File scrFile = ((TakesScreenshot)driver).getScreenshotAs(OutputType.FILE); - throw e; + handleException("Exception in ZeppelinIT while testAngularDisplay ", e); } } @@ -254,9 +252,7 @@ public class ZeppelinIT extends AbstractZeppelinIT { driver.findElement(By.xpath("//button[contains(.,'Save')]")).submit(); driver.switchTo().alert().accept(); } catch (Exception e) { - LOG.error("Exception in ZeppelinIT while testSparkInterpreterDependencyLoading ", e); - File scrFile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE); - throw e; + handleException("Exception in ZeppelinIT while testSparkInterpreterDependencyLoading ", e); } } }
