This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 7346fb184e2 [SPARK-39464][CORE][TESTS] Use `Utils.localCanonicalHostName` instead of `localhost` in tests 7346fb184e2 is described below commit 7346fb184e2d0d27f9720b44cfdd5fb7fe5a10f5 Author: Dongjoon Hyun <dongj...@apache.org> AuthorDate: Tue Jun 14 08:58:23 2022 -0700 [SPARK-39464][CORE][TESTS] Use `Utils.localCanonicalHostName` instead of `localhost` in tests ### What changes were proposed in this pull request? This PR aims to use `Utils.localCanonicalHostName` instead of a constant `localhost` in the following suites. - `MasterSuite` - `MasterWebUISuite` - `RocksDBBackendHistoryServerSuite` ### Why are the changes needed? These test cases fails when we run with `SPARK_LOCAL_IP` on `IPv6`-only environment. ### Does this PR introduce _any_ user-facing change? No. This is a test-only change. ### How was this patch tested? Pass the CIs first and manually test on `IPv6`-only environment. Closes #36866 from dongjoon-hyun/SPARK-39464. Authored-by: Dongjoon Hyun <dongj...@apache.org> Signed-off-by: Dongjoon Hyun <dongj...@apache.org> --- .../spark/deploy/history/HistoryServerSuite.scala | 35 +++++++++++----------- .../apache/spark/deploy/master/MasterSuite.scala | 12 ++++---- .../spark/deploy/master/ui/MasterWebUISuite.scala | 9 +++--- 3 files changed, 29 insertions(+), 27 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala index 02a32a80ddd..1aa846b3ac4 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala @@ -74,6 +74,7 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with private var provider: FsHistoryProvider = null private var server: HistoryServer = null + private val localhost: String = Utils.localCanonicalHostName() private var port: Int = -1 protected def diskBackend: HybridStoreDiskBackend.Value @@ -428,12 +429,12 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with // build a URL for an app or app/attempt plus a page underneath def buildURL(appId: String, suffix: String): URL = { - new URL(s"http://localhost:$port/history/$appId$suffix") + new URL(s"http://$localhost:$port/history/$appId$suffix") } // build a rest URL for the application and suffix. def applications(appId: String, suffix: String): URL = { - new URL(s"http://localhost:$port/api/v1/applications/$appId$suffix") + new URL(s"http://$localhost:$port/api/v1/applications/$appId$suffix") } // start initial job @@ -586,11 +587,11 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with val port = server.boundPort val testUrls = Seq( - s"http://localhost:$port/api/v1/applications/$appId/1/jobs", - s"http://localhost:$port/history/$appId/1/jobs/", - s"http://localhost:$port/api/v1/applications/$appId/logs", - s"http://localhost:$port/api/v1/applications/$appId/1/logs", - s"http://localhost:$port/api/v1/applications/$appId/2/logs") + s"http://$localhost:$port/api/v1/applications/$appId/1/jobs", + s"http://$localhost:$port/history/$appId/1/jobs/", + s"http://$localhost:$port/api/v1/applications/$appId/logs", + s"http://$localhost:$port/api/v1/applications/$appId/1/logs", + s"http://$localhost:$port/api/v1/applications/$appId/2/logs") tests.foreach { case (user, expectedCode) => testUrls.foreach { url => @@ -609,9 +610,9 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with val port = server.boundPort val testUrls = Seq( - s"http://localhost:$port/api/v1/applications/$appId/logs", - s"http://localhost:$port/api/v1/applications/$appId/1/logs", - s"http://localhost:$port/api/v1/applications/$appId/2/logs") + s"http://$localhost:$port/api/v1/applications/$appId/logs", + s"http://$localhost:$port/api/v1/applications/$appId/1/logs", + s"http://$localhost:$port/api/v1/applications/$appId/2/logs") testUrls.foreach { url => TestUtils.httpResponseCode(new URL(url)) @@ -636,9 +637,9 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with def buildPageAttemptUrl(appId: String, attemptId: Option[Int]): URL = { attemptId match { case Some(id) => - new URL(s"http://localhost:$port/history/$appId/$id") + new URL(s"http://$localhost:$port/history/$appId/$id") case None => - new URL(s"http://localhost:$port/history/$appId") + new URL(s"http://$localhost:$port/history/$appId") } } @@ -663,7 +664,7 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with } def getContentAndCode(path: String, port: Int = port): (Int, Option[String], Option[String]) = { - HistoryServerSuite.getContentAndCode(new URL(s"http://localhost:$port/api/v1/$path")) + HistoryServerSuite.getContentAndCode(new URL(s"http://$localhost:$port/api/v1/$path")) } def getUrl(path: String): String = { @@ -671,7 +672,7 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with } def generateURL(path: String): URL = { - new URL(s"http://localhost:$port/api/v1/$path") + new URL(s"http://$localhost:$port/api/v1/$path") } def generateExpectation(name: String, path: String): Unit = { @@ -686,7 +687,7 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with test("SPARK-31697: HistoryServer should set Content-Type") { val port = server.boundPort val nonExistenceAppId = "local-non-existence" - val url = new URL(s"http://localhost:$port/history/$nonExistenceAppId") + val url = new URL(s"http://$localhost:$port/history/$nonExistenceAppId") val conn = url.openConnection().asInstanceOf[HttpURLConnection] conn.setRequestMethod("GET") conn.connect() @@ -697,7 +698,7 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with test("Redirect to the root page when accessed to /history/") { val port = server.boundPort - val url = new URL(s"http://localhost:$port/history/") + val url = new URL(s"http://$localhost:$port/history/") val conn = url.openConnection().asInstanceOf[HttpURLConnection] conn.setRequestMethod("GET") conn.setUseCaches(false) @@ -705,7 +706,7 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with conn.setInstanceFollowRedirects(false) conn.connect() assert(conn.getResponseCode === 302) - assert(conn.getHeaderField("Location") === s"http://localhost:$port/") + assert(conn.getHeaderField("Location") === s"http://$localhost:$port/") } } diff --git a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala index 3832e382a70..1fac3522aed 100644 --- a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala @@ -325,9 +325,9 @@ class MasterSuite extends SparkFunSuite val conf = new SparkConf() val localCluster = LocalSparkCluster(2, 2, 512, conf) localCluster.start() - val masterUrl = s"http://localhost:${localCluster.masterWebUIPort}" + val masterUrl = s"http://${Utils.localCanonicalHostName()}:${localCluster.masterWebUIPort}" try { - eventually(timeout(5.seconds), interval(100.milliseconds)) { + eventually(timeout(50.seconds), interval(100.milliseconds)) { val json = Utils .tryWithResource(Source.fromURL(s"$masterUrl/json"))(_.getLines().mkString("\n")) val JArray(workers) = (parse(json) \ "workers") @@ -362,9 +362,9 @@ class MasterSuite extends SparkFunSuite conf.set(UI_REVERSE_PROXY, true) val localCluster = LocalSparkCluster(2, 2, 512, conf) localCluster.start() - val masterUrl = s"http://localhost:${localCluster.masterWebUIPort}" + val masterUrl = s"http://${Utils.localCanonicalHostName()}:${localCluster.masterWebUIPort}" try { - eventually(timeout(5.seconds), interval(100.milliseconds)) { + eventually(timeout(50.seconds), interval(100.milliseconds)) { val json = Utils .tryWithResource(Source.fromURL(s"$masterUrl/json"))(_.getLines().mkString("\n")) val JArray(workers) = (parse(json) \ "workers") @@ -400,9 +400,9 @@ class MasterSuite extends SparkFunSuite conf.set(UI_REVERSE_PROXY_URL, reverseProxyUrl) val localCluster = LocalSparkCluster(2, 2, 512, conf) localCluster.start() - val masterUrl = s"http://localhost:${localCluster.masterWebUIPort}" + val masterUrl = s"http://${Utils.localCanonicalHostName()}:${localCluster.masterWebUIPort}" try { - eventually(timeout(5.seconds), interval(100.milliseconds)) { + eventually(timeout(50.seconds), interval(100.milliseconds)) { val json = Utils .tryWithResource(Source.fromURL(s"$masterUrl/json"))(_.getLines().mkString("\n")) val JArray(workers) = (parse(json) \ "workers") diff --git a/core/src/test/scala/org/apache/spark/deploy/master/ui/MasterWebUISuite.scala b/core/src/test/scala/org/apache/spark/deploy/master/ui/MasterWebUISuite.scala index be83ec12f92..253517fdcf9 100644 --- a/core/src/test/scala/org/apache/spark/deploy/master/ui/MasterWebUISuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/master/ui/MasterWebUISuite.scala @@ -32,6 +32,7 @@ import org.apache.spark.deploy.DeployMessages.{DecommissionWorkersOnHosts, KillD import org.apache.spark.deploy.DeployTestUtils._ import org.apache.spark.deploy.master._ import org.apache.spark.rpc.{RpcEndpointRef, RpcEnv} +import org.apache.spark.util.Utils class MasterWebUISuite extends SparkFunSuite with BeforeAndAfterAll { @@ -68,7 +69,7 @@ class MasterWebUISuite extends SparkFunSuite with BeforeAndAfterAll { when(master.idToApp).thenReturn(HashMap[String, ApplicationInfo]((activeApp.id, activeApp))) - val url = s"http://localhost:${masterWebUI.boundPort}/app/kill/" + val url = s"http://${Utils.localCanonicalHostName()}:${masterWebUI.boundPort}/app/kill/" val body = convPostDataToString(Map(("id", activeApp.id), ("terminate", "true"))) val conn = sendHttpRequest(url, "POST", body) conn.getResponseCode @@ -79,7 +80,7 @@ class MasterWebUISuite extends SparkFunSuite with BeforeAndAfterAll { test("kill driver") { val activeDriverId = "driver-0" - val url = s"http://localhost:${masterWebUI.boundPort}/driver/kill/" + val url = s"http://${Utils.localCanonicalHostName()}:${masterWebUI.boundPort}/driver/kill/" val body = convPostDataToString(Map(("id", activeDriverId), ("terminate", "true"))) val conn = sendHttpRequest(url, "POST", body) conn.getResponseCode @@ -89,7 +90,7 @@ class MasterWebUISuite extends SparkFunSuite with BeforeAndAfterAll { } private def testKillWorkers(hostnames: Seq[String]): Unit = { - val url = s"http://localhost:${masterWebUI.boundPort}/workers/kill/" + val url = s"http://${Utils.localCanonicalHostName()}:${masterWebUI.boundPort}/workers/kill/" val body = convPostDataToString(hostnames.map(("host", _))) val conn = sendHttpRequest(url, "POST", body) // The master is mocked here, so cannot assert on the response code @@ -99,7 +100,7 @@ class MasterWebUISuite extends SparkFunSuite with BeforeAndAfterAll { } test("Kill one host") { - testKillWorkers(Seq("localhost")) + testKillWorkers(Seq("${Utils.localCanonicalHostName()}")) } test("Kill multiple hosts") { --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org