This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 33a153a6bbcb [SPARK-47030][TESTS] Add `WebBrowserTest` 33a153a6bbcb is described below commit 33a153a6bbcba0d9b2ab20404c7d3b6db86d7b4a Author: Dongjoon Hyun <dh...@apple.com> AuthorDate: Mon Feb 12 17:01:35 2024 -0800 [SPARK-47030][TESTS] Add `WebBrowserTest` ### What changes were proposed in this pull request? This PR aims to add a new test tag, `WebBrowserTest`. ### Why are the changes needed? Currently, several browser-based tests exist in multiple modules like the following. It's difficult to find and run them. ``` common/tags/src/test/java/org/apache/spark/tags/WebBrowserTest.java core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala sql/core/src/test/scala/org/apache/spark/sql/execution/ui/UISeleniumSuite.scala sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala ``` In addition, the previous `ChromeUITest` is designed to disable `ChromeUI*` suite and doesn't cover all `WebBroser` based test suite. ### Does this PR introduce _any_ user-facing change? No, this is a new test tag. ### How was this patch tested? Pass the CIs. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #45089 from dongjoon-hyun/SPARK-47030. Authored-by: Dongjoon Hyun <dh...@apple.com> Signed-off-by: Dongjoon Hyun <dh...@apple.com> --- .../java/org/apache/spark/tags/WebBrowserTest.java | 36 +++++----------------- .../history/ChromeUIHistoryServerSuite.scala | 4 ++- .../spark/deploy/history/HistoryServerSuite.scala | 4 ++- .../apache/spark/ui/ChromeUISeleniumSuite.scala | 3 +- .../org/apache/spark/ui/UISeleniumSuite.scala | 2 ++ .../spark/sql/execution/ui/UISeleniumSuite.scala | 2 ++ .../spark/sql/streaming/ui/UISeleniumSuite.scala | 4 ++- .../sql/hive/thriftserver/UISeleniumSuite.scala | 2 ++ .../apache/spark/streaming/UISeleniumSuite.scala | 2 ++ 9 files changed, 26 insertions(+), 33 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala b/common/tags/src/test/java/org/apache/spark/tags/WebBrowserTest.java similarity index 50% copy from core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala copy to common/tags/src/test/java/org/apache/spark/tags/WebBrowserTest.java index 459af6748e0e..715dcbf3b747 100644 --- a/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala +++ b/common/tags/src/test/java/org/apache/spark/tags/WebBrowserTest.java @@ -15,35 +15,13 @@ * limitations under the License. */ -package org.apache.spark.ui +package org.apache.spark.tags; -import org.openqa.selenium.{JavascriptExecutor, WebDriver} -import org.openqa.selenium.chrome.{ChromeDriver, ChromeOptions} +import java.lang.annotation.*; -import org.apache.spark.tags.ChromeUITest +import org.scalatest.TagAnnotation; -/** - * Selenium tests for the Spark Web UI with Chrome. - */ -@ChromeUITest -class ChromeUISeleniumSuite extends RealBrowserUISeleniumSuite("webdriver.chrome.driver") { - - override var webDriver: WebDriver with JavascriptExecutor = _ - - override def beforeAll(): Unit = { - super.beforeAll() - val chromeOptions = new ChromeOptions - chromeOptions.addArguments("--headless", "--disable-gpu") - webDriver = new ChromeDriver(chromeOptions) - } - - override def afterAll(): Unit = { - try { - if (webDriver != null) { - webDriver.quit() - } - } finally { - super.afterAll() - } - } -} +@TagAnnotation +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD, ElementType.TYPE}) +public @interface WebBrowserTest { } diff --git a/core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala index ec910e9bf343..ec9278f81b6c 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala @@ -21,7 +21,7 @@ import org.openqa.selenium.WebDriver import org.openqa.selenium.chrome.{ChromeDriver, ChromeOptions} import org.apache.spark.internal.config.History.HybridStoreDiskBackend -import org.apache.spark.tags.{ChromeUITest, ExtendedLevelDBTest} +import org.apache.spark.tags.{ChromeUITest, ExtendedLevelDBTest, WebBrowserTest} /** @@ -50,12 +50,14 @@ abstract class ChromeUIHistoryServerSuite } } +@WebBrowserTest @ChromeUITest @ExtendedLevelDBTest class LevelDBBackendChromeUIHistoryServerSuite extends ChromeUIHistoryServerSuite { override protected def diskBackend: HybridStoreDiskBackend.Value = HybridStoreDiskBackend.LEVELDB } +@WebBrowserTest @ChromeUITest class RocksDBBackendChromeUIHistoryServerSuite extends ChromeUIHistoryServerSuite { override protected def diskBackend: HybridStoreDiskBackend.Value = HybridStoreDiskBackend.ROCKSDB diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala index b3d7315e169b..c8ffbf68a0c5 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala @@ -49,7 +49,7 @@ import org.apache.spark.internal.config.Tests.IS_TESTING import org.apache.spark.internal.config.UI._ import org.apache.spark.status.api.v1.ApplicationInfo import org.apache.spark.status.api.v1.JobData -import org.apache.spark.tags.ExtendedLevelDBTest +import org.apache.spark.tags.{ExtendedLevelDBTest, WebBrowserTest} import org.apache.spark.ui.SparkUI import org.apache.spark.util.{ResetSystemProperties, ShutdownHookManager, Utils} import org.apache.spark.util.ArrayImplicits._ @@ -797,12 +797,14 @@ object FakeAuthFilter { val FAKE_HTTP_USER = "HTTP_USER" } +@WebBrowserTest @ExtendedLevelDBTest class LevelDBBackendHistoryServerSuite extends HistoryServerSuite { override protected def diskBackend: History.HybridStoreDiskBackend.Value = HybridStoreDiskBackend.LEVELDB } +@WebBrowserTest class RocksDBBackendHistoryServerSuite extends HistoryServerSuite { override protected def diskBackend: History.HybridStoreDiskBackend.Value = HybridStoreDiskBackend.ROCKSDB diff --git a/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala index 459af6748e0e..12c74e3ae989 100644 --- a/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala @@ -20,11 +20,12 @@ package org.apache.spark.ui import org.openqa.selenium.{JavascriptExecutor, WebDriver} import org.openqa.selenium.chrome.{ChromeDriver, ChromeOptions} -import org.apache.spark.tags.ChromeUITest +import org.apache.spark.tags.{ChromeUITest, WebBrowserTest} /** * Selenium tests for the Spark Web UI with Chrome. */ +@WebBrowserTest @ChromeUITest class ChromeUISeleniumSuite extends RealBrowserUISeleniumSuite("webdriver.chrome.driver") { diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala index dbcc95fc350e..7e49cbce6ec1 100644 --- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala @@ -46,6 +46,7 @@ import org.apache.spark.internal.config.Status._ import org.apache.spark.internal.config.UI._ import org.apache.spark.shuffle.FetchFailedException import org.apache.spark.status.api.v1.{JacksonMessageWriter, RDDDataDistribution, StageStatus} +import org.apache.spark.tags.WebBrowserTest import org.apache.spark.util.Utils private[spark] class SparkUICssErrorHandler extends DefaultCssErrorHandler { @@ -80,6 +81,7 @@ private[spark] class SparkUICssErrorHandler extends DefaultCssErrorHandler { /** * Selenium tests for the Spark Web UI. */ +@WebBrowserTest class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers { implicit var webDriver: WebDriver = _ diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/UISeleniumSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/UISeleniumSuite.scala index c376d1f4614f..dc617046c430 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/UISeleniumSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/UISeleniumSuite.scala @@ -30,8 +30,10 @@ import org.scalatestplus.selenium.WebBrowser import org.apache.spark.{SparkFunSuite, SparkRuntimeException} import org.apache.spark.sql.SparkSession +import org.apache.spark.tags.WebBrowserTest import org.apache.spark.ui.SparkUICssErrorHandler +@WebBrowserTest class UISeleniumSuite extends SparkFunSuite with WebBrowser { private var spark: SparkSession = _ diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala index 6ad940dadc67..cce34e65bdfc 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala @@ -35,10 +35,11 @@ import org.apache.spark.sql.functions.{window => windowFn, _} import org.apache.spark.sql.internal.SQLConf.SHUFFLE_PARTITIONS import org.apache.spark.sql.internal.StaticSQLConf.ENABLED_STREAMING_UI_CUSTOM_METRIC_LIST import org.apache.spark.sql.streaming.{StreamingQueryException, Trigger} -import org.apache.spark.tags.SlowSQLTest +import org.apache.spark.tags.{SlowSQLTest, WebBrowserTest} import org.apache.spark.ui.SparkUICssErrorHandler import org.apache.spark.util.Utils +@WebBrowserTest @SlowSQLTest class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers { @@ -182,6 +183,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers { } } +@WebBrowserTest @SlowSQLTest class UISeleniumWithRocksDBBackendSuite extends UISeleniumSuite { private val storePath = Utils.createTempDir() diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala index 53a69d964b1c..f4b36f234ae6 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala @@ -33,8 +33,10 @@ import org.scalatest.matchers.should.Matchers._ import org.scalatest.time.SpanSugar._ import org.scalatestplus.selenium.WebBrowser +import org.apache.spark.tags.WebBrowserTest import org.apache.spark.ui.SparkUICssErrorHandler +@WebBrowserTest class UISeleniumSuite extends HiveThriftServer2TestBase with WebBrowser with Matchers with BeforeAndAfterAll { diff --git a/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala index eab8012f8189..2e31c0090934 100644 --- a/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala +++ b/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala @@ -29,11 +29,13 @@ import org.scalatestplus.selenium.WebBrowser import org.apache.spark._ import org.apache.spark.internal.config.UI.UI_ENABLED +import org.apache.spark.tags.WebBrowserTest import org.apache.spark.ui.SparkUICssErrorHandler /** * Selenium tests for the Spark Streaming Web UI. */ +@WebBrowserTest class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with TestSuiteBase { implicit var webDriver: WebDriver = _ --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org