This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 33a153a6bbcb [SPARK-47030][TESTS] Add `WebBrowserTest`
33a153a6bbcb is described below
commit 33a153a6bbcba0d9b2ab20404c7d3b6db86d7b4a
Author: Dongjoon Hyun <dh...@apple.com>
AuthorDate: Mon Feb 12 17:01:35 2024 -0800
[SPARK-47030][TESTS] Add `WebBrowserTest`
### What changes were proposed in this pull request?
This PR aims to add a new test tag, `WebBrowserTest`.
### Why are the changes needed?
Currently, several browser-based tests exist in multiple modules like the
following. It's difficult to find and run them.
```
common/tags/src/test/java/org/apache/spark/tags/WebBrowserTest.java
core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala
core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala
core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
sql/core/src/test/scala/org/apache/spark/sql/execution/ui/UISeleniumSuite.scala
sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala
sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala
streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala
```
In addition, the previous `ChromeUITest` is designed to disable `ChromeUI*`
suite and doesn't cover all `WebBroser` based test suite.
### Does this PR introduce _any_ user-facing change?
No, this is a new test tag.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #45089 from dongjoon-hyun/SPARK-47030.
Authored-by: Dongjoon Hyun <dh...@apple.com>
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
.../java/org/apache/spark/tags/WebBrowserTest.java | 36 +++++-----------------
.../history/ChromeUIHistoryServerSuite.scala | 4 ++-
.../spark/deploy/history/HistoryServerSuite.scala | 4 ++-
.../apache/spark/ui/ChromeUISeleniumSuite.scala | 3 +-
.../org/apache/spark/ui/UISeleniumSuite.scala | 2 ++
.../spark/sql/execution/ui/UISeleniumSuite.scala | 2 ++
.../spark/sql/streaming/ui/UISeleniumSuite.scala | 4 ++-
.../sql/hive/thriftserver/UISeleniumSuite.scala | 2 ++
.../apache/spark/streaming/UISeleniumSuite.scala | 2 ++
9 files changed, 26 insertions(+), 33 deletions(-)
diff --git
a/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala
b/common/tags/src/test/java/org/apache/spark/tags/WebBrowserTest.java
similarity index 50%
copy from core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala
copy to common/tags/src/test/java/org/apache/spark/tags/WebBrowserTest.java
index 459af6748e0e..715dcbf3b747 100644
--- a/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala
+++ b/common/tags/src/test/java/org/apache/spark/tags/WebBrowserTest.java
@@ -15,35 +15,13 @@
* limitations under the License.
*/
-package org.apache.spark.ui
+package org.apache.spark.tags;
-import org.openqa.selenium.{JavascriptExecutor, WebDriver}
-import org.openqa.selenium.chrome.{ChromeDriver, ChromeOptions}
+import java.lang.annotation.*;
-import org.apache.spark.tags.ChromeUITest
+import org.scalatest.TagAnnotation;
-/**
- * Selenium tests for the Spark Web UI with Chrome.
- */
-@ChromeUITest
-class ChromeUISeleniumSuite extends
RealBrowserUISeleniumSuite("webdriver.chrome.driver") {
-
- override var webDriver: WebDriver with JavascriptExecutor = _
-
- override def beforeAll(): Unit = {
- super.beforeAll()
- val chromeOptions = new ChromeOptions
- chromeOptions.addArguments("--headless", "--disable-gpu")
- webDriver = new ChromeDriver(chromeOptions)
- }
-
- override def afterAll(): Unit = {
- try {
- if (webDriver != null) {
- webDriver.quit()
- }
- } finally {
- super.afterAll()
- }
- }
-}
+@TagAnnotation
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.METHOD, ElementType.TYPE})
+public @interface WebBrowserTest { }
diff --git
a/core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala
index ec910e9bf343..ec9278f81b6c 100644
---
a/core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala
+++
b/core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala
@@ -21,7 +21,7 @@ import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.{ChromeDriver, ChromeOptions}
import org.apache.spark.internal.config.History.HybridStoreDiskBackend
-import org.apache.spark.tags.{ChromeUITest, ExtendedLevelDBTest}
+import org.apache.spark.tags.{ChromeUITest, ExtendedLevelDBTest,
WebBrowserTest}
/**
@@ -50,12 +50,14 @@ abstract class ChromeUIHistoryServerSuite
}
}
+@WebBrowserTest
@ChromeUITest
@ExtendedLevelDBTest
class LevelDBBackendChromeUIHistoryServerSuite extends
ChromeUIHistoryServerSuite {
override protected def diskBackend: HybridStoreDiskBackend.Value =
HybridStoreDiskBackend.LEVELDB
}
+@WebBrowserTest
@ChromeUITest
class RocksDBBackendChromeUIHistoryServerSuite extends
ChromeUIHistoryServerSuite {
override protected def diskBackend: HybridStoreDiskBackend.Value =
HybridStoreDiskBackend.ROCKSDB
diff --git
a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
index b3d7315e169b..c8ffbf68a0c5 100644
---
a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
+++
b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
@@ -49,7 +49,7 @@ import org.apache.spark.internal.config.Tests.IS_TESTING
import org.apache.spark.internal.config.UI._
import org.apache.spark.status.api.v1.ApplicationInfo
import org.apache.spark.status.api.v1.JobData
-import org.apache.spark.tags.ExtendedLevelDBTest
+import org.apache.spark.tags.{ExtendedLevelDBTest, WebBrowserTest}
import org.apache.spark.ui.SparkUI
import org.apache.spark.util.{ResetSystemProperties, ShutdownHookManager,
Utils}
import org.apache.spark.util.ArrayImplicits._
@@ -797,12 +797,14 @@ object FakeAuthFilter {
val FAKE_HTTP_USER = "HTTP_USER"
}
+@WebBrowserTest
@ExtendedLevelDBTest
class LevelDBBackendHistoryServerSuite extends HistoryServerSuite {
override protected def diskBackend: History.HybridStoreDiskBackend.Value =
HybridStoreDiskBackend.LEVELDB
}
+@WebBrowserTest
class RocksDBBackendHistoryServerSuite extends HistoryServerSuite {
override protected def diskBackend: History.HybridStoreDiskBackend.Value =
HybridStoreDiskBackend.ROCKSDB
diff --git
a/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala
b/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala
index 459af6748e0e..12c74e3ae989 100644
--- a/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala
@@ -20,11 +20,12 @@ package org.apache.spark.ui
import org.openqa.selenium.{JavascriptExecutor, WebDriver}
import org.openqa.selenium.chrome.{ChromeDriver, ChromeOptions}
-import org.apache.spark.tags.ChromeUITest
+import org.apache.spark.tags.{ChromeUITest, WebBrowserTest}
/**
* Selenium tests for the Spark Web UI with Chrome.
*/
+@WebBrowserTest
@ChromeUITest
class ChromeUISeleniumSuite extends
RealBrowserUISeleniumSuite("webdriver.chrome.driver") {
diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
index dbcc95fc350e..7e49cbce6ec1 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
@@ -46,6 +46,7 @@ import org.apache.spark.internal.config.Status._
import org.apache.spark.internal.config.UI._
import org.apache.spark.shuffle.FetchFailedException
import org.apache.spark.status.api.v1.{JacksonMessageWriter,
RDDDataDistribution, StageStatus}
+import org.apache.spark.tags.WebBrowserTest
import org.apache.spark.util.Utils
private[spark] class SparkUICssErrorHandler extends DefaultCssErrorHandler {
@@ -80,6 +81,7 @@ private[spark] class SparkUICssErrorHandler extends
DefaultCssErrorHandler {
/**
* Selenium tests for the Spark Web UI.
*/
+@WebBrowserTest
class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers {
implicit var webDriver: WebDriver = _
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/UISeleniumSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/UISeleniumSuite.scala
index c376d1f4614f..dc617046c430 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/UISeleniumSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/UISeleniumSuite.scala
@@ -30,8 +30,10 @@ import org.scalatestplus.selenium.WebBrowser
import org.apache.spark.{SparkFunSuite, SparkRuntimeException}
import org.apache.spark.sql.SparkSession
+import org.apache.spark.tags.WebBrowserTest
import org.apache.spark.ui.SparkUICssErrorHandler
+@WebBrowserTest
class UISeleniumSuite extends SparkFunSuite with WebBrowser {
private var spark: SparkSession = _
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala
index 6ad940dadc67..cce34e65bdfc 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala
@@ -35,10 +35,11 @@ import org.apache.spark.sql.functions.{window => windowFn,
_}
import org.apache.spark.sql.internal.SQLConf.SHUFFLE_PARTITIONS
import
org.apache.spark.sql.internal.StaticSQLConf.ENABLED_STREAMING_UI_CUSTOM_METRIC_LIST
import org.apache.spark.sql.streaming.{StreamingQueryException, Trigger}
-import org.apache.spark.tags.SlowSQLTest
+import org.apache.spark.tags.{SlowSQLTest, WebBrowserTest}
import org.apache.spark.ui.SparkUICssErrorHandler
import org.apache.spark.util.Utils
+@WebBrowserTest
@SlowSQLTest
class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers {
@@ -182,6 +183,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser
with Matchers {
}
}
+@WebBrowserTest
@SlowSQLTest
class UISeleniumWithRocksDBBackendSuite extends UISeleniumSuite {
private val storePath = Utils.createTempDir()
diff --git
a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala
b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala
index 53a69d964b1c..f4b36f234ae6 100644
---
a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala
+++
b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala
@@ -33,8 +33,10 @@ import org.scalatest.matchers.should.Matchers._
import org.scalatest.time.SpanSugar._
import org.scalatestplus.selenium.WebBrowser
+import org.apache.spark.tags.WebBrowserTest
import org.apache.spark.ui.SparkUICssErrorHandler
+@WebBrowserTest
class UISeleniumSuite
extends HiveThriftServer2TestBase
with WebBrowser with Matchers with BeforeAndAfterAll {
diff --git
a/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala
b/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala
index eab8012f8189..2e31c0090934 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala
@@ -29,11 +29,13 @@ import org.scalatestplus.selenium.WebBrowser
import org.apache.spark._
import org.apache.spark.internal.config.UI.UI_ENABLED
+import org.apache.spark.tags.WebBrowserTest
import org.apache.spark.ui.SparkUICssErrorHandler
/**
* Selenium tests for the Spark Streaming Web UI.
*/
+@WebBrowserTest
class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with
TestSuiteBase {
implicit var webDriver: WebDriver = _
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
