Repository: spark Updated Branches: refs/heads/branch-2.0 e30fe1c6a -> 491b47a16
[SPARK-19750][UI][BRANCH-2.1] Fix redirect issue from http to https ## What changes were proposed in this pull request? If spark ui port (4040) is not set, it will choose port number 0, this will make https port to also choose 0. And in Spark 2.1 code, it will use this https port (0) to do redirect, so when redirect triggered, it will point to a wrong url: like: ``` /tmp/temp$ wget http://172.27.25.134:55015 --2017-02-23 12:13:54-- http://172.27.25.134:55015/ Connecting to 172.27.25.134:55015... connected. HTTP request sent, awaiting response... 302 Found Location: https://172.27.25.134:0/ [following] --2017-02-23 12:13:54-- https://172.27.25.134:0/ Connecting to 172.27.25.134:0... failed: Can't assign requested address. Retrying. --2017-02-23 12:13:55-- (try: 2) https://172.27.25.134:0/ Connecting to 172.27.25.134:0... failed: Can't assign requested address. Retrying. --2017-02-23 12:13:57-- (try: 3) https://172.27.25.134:0/ Connecting to 172.27.25.134:0... failed: Can't assign requested address. Retrying. --2017-02-23 12:14:00-- (try: 4) https://172.27.25.134:0/ Connecting to 172.27.25.134:0... failed: Can't assign requested address. Retrying. ``` So instead of using 0 to do redirect, we should pick a bound port instead. This issue only exists in Spark 2.1-, and can be reproduced in yarn cluster mode. ## How was this patch tested? Current redirect UT doesn't verify this issue, so extend current UT to do correct verification. Author: jerryshao <ss...@hortonworks.com> Closes #17083 from jerryshao/SPARK-19750. (cherry picked from commit 3a7591ad5315308d24c0e444ce304ff78aef2304) Signed-off-by: Marcelo Vanzin <van...@cloudera.com> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/491b47a1 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/491b47a1 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/491b47a1 Branch: refs/heads/branch-2.0 Commit: 491b47a162030cb0dbb81ae32898e109707a4709 Parents: e30fe1c Author: jerryshao <ss...@hortonworks.com> Authored: Thu Mar 2 17:18:52 2017 -0800 Committer: Marcelo Vanzin <van...@cloudera.com> Committed: Thu Mar 2 17:19:04 2017 -0800 ---------------------------------------------------------------------- .../main/scala/org/apache/spark/TestUtils.scala | 23 ++++++++++++++++---- .../scala/org/apache/spark/ui/JettyUtils.scala | 10 +++++---- .../scala/org/apache/spark/ui/UISuite.scala | 6 ++++- 3 files changed, 30 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/491b47a1/core/src/main/scala/org/apache/spark/TestUtils.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/TestUtils.scala b/core/src/main/scala/org/apache/spark/TestUtils.scala index a76395b..3bf1d31 100644 --- a/core/src/main/scala/org/apache/spark/TestUtils.scala +++ b/core/src/main/scala/org/apache/spark/TestUtils.scala @@ -27,6 +27,7 @@ import java.util.Arrays import java.util.concurrent.{CountDownLatch, TimeUnit} import java.util.jar.{JarEntry, JarOutputStream} import javax.net.ssl._ +import javax.servlet.http.HttpServletResponse import javax.tools.{JavaFileObject, SimpleJavaFileObject, ToolProvider} import scala.collection.JavaConverters._ @@ -186,12 +187,12 @@ private[spark] object TestUtils { } /** - * Returns the response code from an HTTP(S) URL. + * Returns the response code and url (if redirected) from an HTTP(S) URL. */ - def httpResponseCode( + def httpResponseCodeAndURL( url: URL, method: String = "GET", - headers: Seq[(String, String)] = Nil): Int = { + headers: Seq[(String, String)] = Nil): (Int, Option[String]) = { val connection = url.openConnection().asInstanceOf[HttpURLConnection] connection.setRequestMethod(method) headers.foreach { case (k, v) => connection.setRequestProperty(k, v) } @@ -210,16 +211,30 @@ private[spark] object TestUtils { sslCtx.init(null, Array(trustManager), new SecureRandom()) connection.asInstanceOf[HttpsURLConnection].setSSLSocketFactory(sslCtx.getSocketFactory()) connection.asInstanceOf[HttpsURLConnection].setHostnameVerifier(verifier) + connection.setInstanceFollowRedirects(false) } try { connection.connect() - connection.getResponseCode() + if (connection.getResponseCode == HttpServletResponse.SC_FOUND) { + (connection.getResponseCode, Option(connection.getHeaderField("Location"))) + } else { + (connection.getResponseCode(), None) + } } finally { connection.disconnect() } } + /** + * Returns the response code from an HTTP(S) URL. + */ + def httpResponseCode( + url: URL, + method: String = "GET", + headers: Seq[(String, String)] = Nil): Int = { + httpResponseCodeAndURL(url, method, headers)._1 + } } http://git-wip-us.apache.org/repos/asf/spark/blob/491b47a1/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala index d989099..db213f1 100644 --- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala @@ -287,7 +287,7 @@ private[spark] object JettyUtils extends Logging { // redirect the HTTP requests to HTTPS port httpConnector.setName(REDIRECT_CONNECTOR_NAME) - collection.addHandler(createRedirectHttpsHandler(securePort, scheme)) + collection.addHandler(createRedirectHttpsHandler(connector, scheme)) Some(connector) case None => @@ -335,7 +335,9 @@ private[spark] object JettyUtils extends Logging { server.getHandler().asInstanceOf[ContextHandlerCollection]) } - private def createRedirectHttpsHandler(securePort: Int, scheme: String): ContextHandler = { + private def createRedirectHttpsHandler( + httpsConnector: ServerConnector, + scheme: String): ContextHandler = { val redirectHandler: ContextHandler = new ContextHandler redirectHandler.setContextPath("/") redirectHandler.setVirtualHosts(Array("@" + REDIRECT_CONNECTOR_NAME)) @@ -348,8 +350,8 @@ private[spark] object JettyUtils extends Logging { if (baseRequest.isSecure) { return } - val httpsURI = createRedirectURI(scheme, baseRequest.getServerName, securePort, - baseRequest.getRequestURI, baseRequest.getQueryString) + val httpsURI = createRedirectURI(scheme, baseRequest.getServerName, + httpsConnector.getLocalPort, baseRequest.getRequestURI, baseRequest.getQueryString) response.setContentLength(0) response.encodeRedirectURL(httpsURI) response.sendRedirect(httpsURI) http://git-wip-us.apache.org/repos/asf/spark/blob/491b47a1/core/src/test/scala/org/apache/spark/ui/UISuite.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/spark/ui/UISuite.scala b/core/src/test/scala/org/apache/spark/ui/UISuite.scala index 5ee8a5a..6a57361 100644 --- a/core/src/test/scala/org/apache/spark/ui/UISuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/UISuite.scala @@ -232,8 +232,12 @@ class UISuite extends SparkFunSuite { s"$scheme://localhost:$port/test1/root", s"$scheme://localhost:$port/test2/root") urls.foreach { url => - val rc = TestUtils.httpResponseCode(new URL(url)) + val (rc, redirectUrl) = TestUtils.httpResponseCodeAndURL(new URL(url)) assert(rc === expected, s"Unexpected status $rc for $url") + if (rc == HttpServletResponse.SC_FOUND) { + assert( + TestUtils.httpResponseCode(new URL(redirectUrl.get)) === HttpServletResponse.SC_OK) + } } } } finally { --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org