This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 7cfc40fff6a [SPARK-39468][CORE] Improve `RpcAddress` to add `[]` to `IPv6` if needed 7cfc40fff6a is described below commit 7cfc40fff6a2bb4025b29d8dd9eb66734030a901 Author: Dongjoon Hyun <dongj...@apache.org> AuthorDate: Tue Jun 14 20:00:05 2022 -0700 [SPARK-39468][CORE] Improve `RpcAddress` to add `[]` to `IPv6` if needed ### What changes were proposed in this pull request? This PR aims to extend the `IPv6` support in `RpcAddress` additionally when the input doesn't have `[]` properly. ### Why are the changes needed? Note that Apache Spark already depends on `java.net.URI` `getHost` and `getPort` and it assumpts `[]`-style IPv6. This PR additionally handles the case where the given host string doesn't have `[]`. - `RpcAddress.fromURIString` https://github.com/apache/spark/blob/683179c6813dbdccebd4063c3aac520020765692/core/src/main/scala/org/apache/spark/rpc/RpcAddress.scala#L40-L43 - `Utils.extractHostPortFromSparkUrl` https://github.com/apache/spark/blob/683179c6813dbdccebd4063c3aac520020765692/core/src/main/scala/org/apache/spark/util/Utils.scala#L2520-L2524 We need to handle Java URI IPv6 style additionally. ``` jshell> var uri = new java.net.URI("https://[::1]:80") uri ==> https://[::1]:80 jshell> uri.getHost() $4 ==> "[::1]" jshell> uri.getPort() $5 ==> 80 ``` ### Does this PR introduce _any_ user-facing change? No. This is `private[spark]` class. ### How was this patch tested? Pass the CIs with newly added test cases. This is also tested manually on IPv6-only environment with the following command. ``` $ SERIAL_SBT_TESTS=1 SPARK_LOCAL_HOSTNAME='[2600:.(omitted)..:60cd]' build/sbt "core/test" -Djava.net.preferIPv6Addresses=true -Dtest.exclude.tags=org.apache.spark.tags.ExtendedLevelDBTest ... [info] Run completed in 18 minutes, 43 seconds. [info] Total number of tests run: 2950 [info] Suites: completed 284, aborted 0 [info] Tests: succeeded 2950, failed 0, canceled 4, ignored 8, pending 0 [info] All tests passed. [info] Passed: Total 3214, Failed 0, Errors 0, Passed 3214, Ignored 8, Canceled 4 [success] Total time: 1189 s (19:49), completed Jun 14, 2022, 4:45:55 PM ``` Closes #36868 from dongjoon-hyun/SPARK-39468. Authored-by: Dongjoon Hyun <dongj...@apache.org> Signed-off-by: Dongjoon Hyun <dongj...@apache.org> --- .../main/scala/org/apache/spark/rpc/RpcAddress.scala | 4 +++- core/src/main/scala/org/apache/spark/util/Utils.scala | 2 +- .../scala/org/apache/spark/rpc/RpcAddressSuite.scala | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/rpc/RpcAddress.scala b/core/src/main/scala/org/apache/spark/rpc/RpcAddress.scala index eb0b26947f5..2a2d2051799 100644 --- a/core/src/main/scala/org/apache/spark/rpc/RpcAddress.scala +++ b/core/src/main/scala/org/apache/spark/rpc/RpcAddress.scala @@ -23,7 +23,9 @@ import org.apache.spark.util.Utils /** * Address for an RPC environment, with hostname and port. */ -private[spark] case class RpcAddress(host: String, port: Int) { +private[spark] case class RpcAddress(_host: String, port: Int) { + + val host: String = Utils.addBracketsIfNeeded(_host) def hostPort: String = host + ":" + port diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 3e4a7e727a8..cf93897f97d 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -1089,7 +1089,7 @@ private[spark] object Utils extends Logging { addBracketsIfNeeded(customHostname.getOrElse(InetAddresses.toUriString(localIpAddress))) } - private def addBracketsIfNeeded(addr: String): String = { + private[spark] def addBracketsIfNeeded(addr: String): String = { if (addr.contains(":") && !addr.contains("[")) { "[" + addr + "]" } else { diff --git a/core/src/test/scala/org/apache/spark/rpc/RpcAddressSuite.scala b/core/src/test/scala/org/apache/spark/rpc/RpcAddressSuite.scala index b3223ec61bf..0f7c9d71330 100644 --- a/core/src/test/scala/org/apache/spark/rpc/RpcAddressSuite.scala +++ b/core/src/test/scala/org/apache/spark/rpc/RpcAddressSuite.scala @@ -52,4 +52,22 @@ class RpcAddressSuite extends SparkFunSuite { val address = RpcAddress("1.2.3.4", 1234) assert(address.toSparkURL == "spark://1.2.3.4:1234") } + + test("SPARK-39468: IPv6 hostPort") { + val address = RpcAddress("::1", 1234) + assert(address.host == "[::1]") + assert(address.port == 1234) + assert(address.hostPort == "[::1]:1234") + } + + test("SPARK-39468: IPv6 fromSparkURL") { + val address = RpcAddress.fromSparkURL("spark://[::1]:1234") + assert(address.host == "[::1]") + assert(address.port == 1234) + } + + test("SPARK-39468: IPv6 toSparkURL") { + val address = RpcAddress("::1", 1234) + assert(address.toSparkURL == "spark://[::1]:1234") + } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org