This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new f5b1e37c9e6a [SPARK-44445][BUILD][TESTS] Use `org.seleniumhq.selenium.htmlunit3-driver` instead of `net.sourceforge.htmlunit` f5b1e37c9e6a is described below commit f5b1e37c9e6a4ec2fd897f97cd4526415e6c0e49 Author: Dongjoon Hyun <dh...@apple.com> AuthorDate: Mon Feb 12 09:12:10 2024 -0800 [SPARK-44445][BUILD][TESTS] Use `org.seleniumhq.selenium.htmlunit3-driver` instead of `net.sourceforge.htmlunit` ### What changes were proposed in this pull request? This PR aims to use `org.seleniumhq.selenium.htmlunit3-driver` 4.17.0 instead of old `net.sourceforge.htmlunit` test dependencies and `org.seleniumhq.selenium.htmlunit-driver` test dependency; - Remove `net.sourceforge.htmlunit.htmlunit` `2.70.0`. - Remove `net.sourceforge.htmlunit.htmlunit-core-js` `2.70.0`. - Remove `org.seleniumhq.selenium.htmlunit-driver` `4.12.0`. - Remove `xml-apis:xml-apis` `1.4.01`. ### Why are the changes needed? To help browser-based test suites. ### Does this PR introduce _any_ user-facing change? No. This is a test-only dependency and code change. ### How was this patch tested? Manual tests. ``` build/sbt -Dguava.version=32.1.2-jre -Dspark.test.webdriver.chrome.driver=/opt/homebrew/bin/chromedriver -Dtest.default.exclude.tags="" -Phive -Phive-thriftserver \ "core/testOnly *HistoryServerSuite" ``` ``` build/sbt -Dguava.version=32.1.2-jre -Dspark.test.webdriver.chrome.driver=/opt/homebrew/bin/chromedriver -Dtest.default.exclude.tags="" -Phive -Phive-thriftserver \ "core/testOnly *UISeleniumSuite" ``` ``` build/sbt -Dguava.version=32.1.2-jre -Dspark.test.webdriver.chrome.driver=/opt/homebrew/bin/chromedriver -Dtest.default.exclude.tags="" -Phive -Phive-thriftserver \ "sql/testOnly *UISeleniumSuite" ``` ``` build/sbt -Dguava.version=32.1.2-jre -Dspark.test.webdriver.chrome.driver=/opt/homebrew/bin/chromedriver -Dtest.default.exclude.tags="" -Phive -Phive-thriftserver \ "streaming/testOnly *UISeleniumSuite" ``` ``` build/sbt -Dguava.version=32.1.2-jre -Dspark.test.webdriver.chrome.driver=/opt/homebrew/bin/chromedriver -Dtest.default.exclude.tags="" -Phive -Phive-thriftserver \ "hive-thriftserver/testOnly *UISeleniumSuite" ``` ### Was this patch authored or co-authored using generative AI tooling? No. Closes #45079 from dongjoon-hyun/SPARK-44445. Authored-by: Dongjoon Hyun <dh...@apple.com> Signed-off-by: Dongjoon Hyun <dh...@apple.com> --- core/pom.xml | 19 +--------------- .../spark/deploy/history/HistoryServerSuite.scala | 4 +++- .../history/RealBrowserUIHistoryServerSuite.scala | 6 ++++- .../org/apache/spark/ui/UISeleniumSuite.scala | 4 ++-- pom.xml | 26 +++------------------- sql/core/pom.xml | 2 +- .../spark/sql/execution/ui/UISeleniumSuite.scala | 8 ++++--- sql/hive-thriftserver/pom.xml | 2 +- streaming/pom.xml | 2 +- 9 files changed, 22 insertions(+), 51 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index f780551fb555..9b5297cb8543 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -354,18 +354,7 @@ </dependency> <dependency> <groupId>org.seleniumhq.selenium</groupId> - <artifactId>htmlunit-driver</artifactId> - <scope>test</scope> - </dependency> - <!-- Coerce sbt into honoring these dependency updates: --> - <dependency> - <groupId>net.sourceforge.htmlunit</groupId> - <artifactId>htmlunit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>net.sourceforge.htmlunit</groupId> - <artifactId>htmlunit-core-js</artifactId> + <artifactId>htmlunit3-driver</artifactId> <scope>test</scope> </dependency> <!-- at least just for tests, coerce SBT to use the updated httpcore/client version --> @@ -384,12 +373,6 @@ <artifactId>httpcore</artifactId> <scope>test</scope> </dependency> - <!-- Added for selenium: --> - <dependency> - <groupId>xml-apis</groupId> - <artifactId>xml-apis</artifactId> - <scope>test</scope> - </dependency> <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala index 1ca1e8fefd06..b3d7315e169b 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala @@ -561,7 +561,9 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with // app is no longer incomplete listApplications(false) should not contain(appId) - assert(jobcount === getNumJobs("/jobs")) + eventually(stdTimeout, stdInterval) { + assert(jobcount === getNumJobs("/jobs")) + } // no need to retain the test dir now the tests complete ShutdownHookManager.registerShutdownDeleteDir(logDir) diff --git a/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala index ea3a5ef5ba10..5eeb2c7a8053 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala @@ -22,8 +22,10 @@ import javax.servlet.http.HttpServletRequest import org.eclipse.jetty.proxy.ProxyServlet import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder} import org.openqa.selenium.WebDriver +import org.scalatest.concurrent.Eventually._ import org.scalatest.matchers.must.Matchers import org.scalatest.matchers.should.Matchers._ +import org.scalatest.time.SpanSugar._ import org.scalatestplus.selenium.WebBrowser import org.apache.spark._ @@ -147,7 +149,9 @@ abstract class RealBrowserUIHistoryServerSuite(val driverProp: String) // there are at least some URL links that were generated via javascript, // and they all contain the spark.ui.proxyBase (uiRoot) - links.length should be > 4 + eventually(timeout(10.seconds)) { + links.length should be > 4 + } for (link <- links) { link should startWith(url + uiRoot) } diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala index 8689a9c16d69..dbcc95fc350e 100644 --- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala @@ -24,9 +24,9 @@ import javax.servlet.http.{HttpServletRequest, HttpServletResponse} import scala.io.Source import scala.xml.Node -import com.gargoylesoftware.css.parser.CSSParseException -import com.gargoylesoftware.htmlunit.DefaultCssErrorHandler import org.glassfish.jersey.internal.util.collection.MultivaluedStringMap +import org.htmlunit.DefaultCssErrorHandler +import org.htmlunit.cssparser.parser.CSSParseException import org.json4s._ import org.json4s.jackson.JsonMethods import org.openqa.selenium.{By, WebDriver} diff --git a/pom.xml b/pom.xml index 79d572f1b8bf..ed6f48262570 100644 --- a/pom.xml +++ b/pom.xml @@ -215,8 +215,7 @@ <antlr4.version>4.13.1</antlr4.version> <jpam.version>1.1</jpam.version> <selenium.version>4.12.1</selenium.version> - <htmlunit-driver.version>4.12.0</htmlunit-driver.version> - <htmlunit.version>2.70.0</htmlunit.version> + <htmlunit3-driver.version>4.17.0</htmlunit3-driver.version> <maven-antrun.version>3.1.0</maven-antrun.version> <commons-crypto.version>1.1.0</commons-crypto.version> <commons-cli.version>1.6.0</commons-cli.version> @@ -710,29 +709,10 @@ </dependency> <dependency> <groupId>org.seleniumhq.selenium</groupId> - <artifactId>htmlunit-driver</artifactId> - <version>${htmlunit-driver.version}</version> + <artifactId>htmlunit3-driver</artifactId> + <version>${htmlunit3-driver.version}</version> <scope>test</scope> </dependency> - <!-- Update htmlunit dependency that selenium uses for better JS support --> - <dependency> - <groupId>net.sourceforge.htmlunit</groupId> - <artifactId>htmlunit</artifactId> - <version>${htmlunit.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>net.sourceforge.htmlunit</groupId> - <artifactId>htmlunit-core-js</artifactId> - <version>${htmlunit.version}</version> - <scope>test</scope> - </dependency> - <!-- Added for selenium only, and should match its dependent version: --> - <dependency> - <groupId>xml-apis</groupId> - <artifactId>xml-apis</artifactId> - <version>1.4.01</version> - </dependency> <!-- log4j --> <dependency> diff --git a/sql/core/pom.xml b/sql/core/pom.xml index f2d888ceafe7..1ef41f9b43da 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -225,7 +225,7 @@ </dependency> <dependency> <groupId>org.seleniumhq.selenium</groupId> - <artifactId>htmlunit-driver</artifactId> + <artifactId>htmlunit3-driver</artifactId> <scope>test</scope> </dependency> </dependencies> diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/UISeleniumSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/UISeleniumSuite.scala index fca2c3b4c7c5..c376d1f4614f 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/UISeleniumSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/UISeleniumSuite.scala @@ -22,6 +22,7 @@ import scala.jdk.CollectionConverters._ import org.apache.commons.text.StringEscapeUtils.escapeJava import org.apache.commons.text.translate.EntityArrays._ +import org.openqa.selenium.By import org.openqa.selenium.htmlunit.HtmlUnitDriver import org.scalatest.concurrent.Eventually.eventually import org.scalatest.concurrent.Futures.{interval, timeout} @@ -50,7 +51,8 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser { val webUrl = spark.sparkContext.uiWebUrl assert(webUrl.isDefined, "please turn on spark.ui.enabled") go to s"${webUrl.get}/SQL" - findAll(cssSelector("""#failed-table td .stacktrace-details""")).map(_.text).toList + findAll(cssSelector("""#failed-table td .stacktrace-details""")) + .map(_.underlying.findElement(By.xpath("..")).getText).toList } private def findErrorSummaryOnSQLUI(): String = { @@ -102,13 +104,13 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser { eventually(timeout(10.seconds), interval(100.milliseconds)) { val sd = findErrorMessageOnSQLUI() assert(sd.size === 1, "Analyze fail shall show the query in failed table") - assert(sd.head.startsWith("[TABLE_OR_VIEW_NOT_FOUND]")) + assert(sd.head.startsWith("TABLE_OR_VIEW_NOT_FOUND")) val id = findExecutionIDOnSQLUI() // check query detail page go to s"${spark.sparkContext.uiWebUrl.get}/SQL/execution/?id=$id" val planDot = findAll(cssSelector(""".dot-file""")).map(_.text).toList - assert(planDot.head.startsWith("digraph G {")) + assert(planDot.size === 1) val planDetails = findAll(cssSelector("""#physical-plan-details""")).map(_.text).toList assert(planDetails.head.isEmpty) } diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index 920a9b67376f..04d77209bb39 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -103,7 +103,7 @@ </dependency> <dependency> <groupId>org.seleniumhq.selenium</groupId> - <artifactId>htmlunit-driver</artifactId> + <artifactId>htmlunit3-driver</artifactId> <scope>test</scope> </dependency> <dependency> diff --git a/streaming/pom.xml b/streaming/pom.xml index f409791f4670..85a4d268d2a2 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -113,7 +113,7 @@ </dependency> <dependency> <groupId>org.seleniumhq.selenium</groupId> - <artifactId>htmlunit-driver</artifactId> + <artifactId>htmlunit3-driver</artifactId> <scope>test</scope> </dependency> <dependency> --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org