This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new ba8abdda370 [SPARK-42474][CORE][K8S] Add extraJVMOptions JVM GC option K8s test cases ba8abdda370 is described below commit ba8abdda3703ce9d60e26678290739d080020418 Author: Dongjoon Hyun <dongj...@apache.org> AuthorDate: Fri Feb 17 01:11:17 2023 -0800 [SPARK-42474][CORE][K8S] Add extraJVMOptions JVM GC option K8s test cases ### What changes were proposed in this pull request? This PR aims to add JVM GC option test coverage to K8s Integration Suite. To reuse the existing code, `isG1GC` variable is moved from `MemoryManager` to `Utils`. ### Why are the changes needed? To provide more test coverage for JVM Options. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass the CIs ``` [info] KubernetesSuite: [info] - SPARK-42190: Run SparkPi with local[*] (4 seconds, 990 milliseconds) [info] - Run SparkPi with no resources (7 seconds, 101 milliseconds) [info] - Run SparkPi with no resources & statefulset allocation (7 seconds, 27 milliseconds) [info] - Run SparkPi with a very long application name. (7 seconds, 100 milliseconds) [info] - Use SparkLauncher.NO_RESOURCE (7 seconds, 947 milliseconds) [info] - Run SparkPi with a master URL without a scheme. (6 seconds, 932 milliseconds) [info] - Run SparkPi with an argument. (9 seconds, 47 milliseconds) [info] - Run SparkPi with custom labels, annotations, and environment variables. (6 seconds, 969 milliseconds) [info] - All pods have the same service account by default (6 seconds, 916 milliseconds) [info] - Run extraJVMOptions check on driver (3 seconds, 964 milliseconds) [info] - Run extraJVMOptions JVM GC option check - G1GC (3 seconds, 948 milliseconds) [info] - Run extraJVMOptions JVM GC option check - Other GC (4 seconds, 51 milliseconds) ... ``` Closes #40062 from dongjoon-hyun/SPARK-42474. Authored-by: Dongjoon Hyun <dongj...@apache.org> Signed-off-by: Dongjoon Hyun <dongj...@apache.org> --- .../org/apache/spark/memory/MemoryManager.scala | 23 +--------------------- .../main/scala/org/apache/spark/util/Utils.scala | 20 ++++++++++++++++++- .../spark/examples/DriverSubmissionTest.scala | 2 ++ .../k8s/integrationtest/BasicTestsSuite.scala | 16 +++++++++++++++ 4 files changed, 38 insertions(+), 23 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/memory/MemoryManager.scala b/core/src/main/scala/org/apache/spark/memory/MemoryManager.scala index 596974f338f..c33fca35764 100644 --- a/core/src/main/scala/org/apache/spark/memory/MemoryManager.scala +++ b/core/src/main/scala/org/apache/spark/memory/MemoryManager.scala @@ -17,11 +17,8 @@ package org.apache.spark.memory -import java.lang.management.{ManagementFactory, PlatformManagedObject} import javax.annotation.concurrent.GuardedBy -import scala.util.Try - import org.apache.spark.SparkConf import org.apache.spark.internal.Logging import org.apache.spark.internal.config._ @@ -263,7 +260,7 @@ private[spark] abstract class MemoryManager( } val size = ByteArrayMethods.nextPowerOf2(maxTungstenMemory / cores / safetyFactor) val chosenPageSize = math.min(maxPageSize, math.max(minPageSize, size)) - if (isG1GC && tungstenMemoryMode == MemoryMode.ON_HEAP) { + if (Utils.isG1GC && tungstenMemoryMode == MemoryMode.ON_HEAP) { chosenPageSize - Platform.LONG_ARRAY_OFFSET } else { chosenPageSize @@ -281,22 +278,4 @@ private[spark] abstract class MemoryManager( case MemoryMode.OFF_HEAP => MemoryAllocator.UNSAFE } } - - /** - * Return whether we are using G1GC or not - */ - private lazy val isG1GC: Boolean = { - Try { - val clazz = Utils.classForName("com.sun.management.HotSpotDiagnosticMXBean") - .asInstanceOf[Class[_ <: PlatformManagedObject]] - val vmOptionClazz = Utils.classForName("com.sun.management.VMOption") - val hotSpotDiagnosticMXBean = ManagementFactory.getPlatformMXBean(clazz) - val vmOptionMethod = clazz.getMethod("getVMOption", classOf[String]) - val valueMethod = vmOptionClazz.getMethod("getValue") - - val useG1GCObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseG1GC") - val useG1GC = valueMethod.invoke(useG1GCObject).asInstanceOf[String] - "true".equals(useG1GC) - }.getOrElse(false) - } } diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 9bf45ed3776..510486bc56b 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -19,7 +19,7 @@ package org.apache.spark.util import java.io._ import java.lang.{Byte => JByte} -import java.lang.management.{LockInfo, ManagementFactory, MonitorInfo, ThreadInfo} +import java.lang.management.{LockInfo, ManagementFactory, MonitorInfo, PlatformManagedObject, ThreadInfo} import java.lang.reflect.InvocationTargetException import java.math.{MathContext, RoundingMode} import java.net._ @@ -3278,6 +3278,24 @@ private[spark] object Utils extends Logging { case _ => math.max(sortedSize(len / 2), 1) } } + + /** + * Return whether we are using G1GC or not + */ + lazy val isG1GC: Boolean = { + Try { + val clazz = Utils.classForName("com.sun.management.HotSpotDiagnosticMXBean") + .asInstanceOf[Class[_ <: PlatformManagedObject]] + val vmOptionClazz = Utils.classForName("com.sun.management.VMOption") + val hotSpotDiagnosticMXBean = ManagementFactory.getPlatformMXBean(clazz) + val vmOptionMethod = clazz.getMethod("getVMOption", classOf[String]) + val valueMethod = vmOptionClazz.getMethod("getValue") + + val useG1GCObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseG1GC") + val useG1GC = valueMethod.invoke(useG1GCObject).asInstanceOf[String] + "true".equals(useG1GC) + }.getOrElse(false) + } } private[util] object CallerContext extends Logging { diff --git a/examples/src/main/scala/org/apache/spark/examples/DriverSubmissionTest.scala b/examples/src/main/scala/org/apache/spark/examples/DriverSubmissionTest.scala index 94fc755e0ca..f1d63fbcfb8 100644 --- a/examples/src/main/scala/org/apache/spark/examples/DriverSubmissionTest.scala +++ b/examples/src/main/scala/org/apache/spark/examples/DriverSubmissionTest.scala @@ -43,6 +43,8 @@ object DriverSubmissionTest { println("System properties containing spark.test:") properties.filter { case (k, _) => k.contains("spark.test") }.foreach(println) + println("JVM G1GC Flag: " + Utils.isG1GC) + for (i <- 1 until numSecondsToSleep) { println(s"Alive for $i out of $numSecondsToSleep seconds") Thread.sleep(1000) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala index 1f48d796067..ed38f8d70d7 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala @@ -135,6 +135,22 @@ private[spark] trait BasicTestsSuite { k8sSuite: KubernetesSuite => expectedJVMValue = Seq("(spark.test.foo,spark.test.bar)")) } + test("SPARK-42474: Run extraJVMOptions JVM GC option check - G1GC", k8sTestTag) { + sparkAppConf + .set("spark.driver.extraJavaOptions", "-XX:+UseG1GC") + .set("spark.executor.extraJavaOptions", "-XX:+UseG1GC") + runSparkJVMCheckAndVerifyCompletion( + expectedJVMValue = Seq("JVM G1GC Flag: true")) + } + + test("SPARK-42474: Run extraJVMOptions JVM GC option check - Other GC", k8sTestTag) { + sparkAppConf + .set("spark.driver.extraJavaOptions", "-XX:+UseParallelGC") + .set("spark.executor.extraJavaOptions", "-XX:+UseParallelGC") + runSparkJVMCheckAndVerifyCompletion( + expectedJVMValue = Seq("JVM G1GC Flag: false")) + } + test("Run SparkRemoteFileTest using a remote data file", k8sTestTag, localTestTag) { assert(sys.props.contains("spark.test.home"), "spark.test.home is not set!") TestUtils.withHttpServer(sys.props("spark.test.home")) { baseURL => --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org