This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push: new 8059a7e [SPARK-36195][BUILD] Set MaxMetaspaceSize JVM option to 2g 8059a7e is described below commit 8059a7e5e6726e7ca1401416be90b92c305c5060 Author: Dongjoon Hyun <dongj...@apache.org> AuthorDate: Sun Jul 18 10:15:15 2021 -0700 [SPARK-36195][BUILD] Set MaxMetaspaceSize JVM option to 2g ### What changes were proposed in this pull request? This PR aims to set `MaxMetaspaceSize` to `2g` because it's increasing the native memory consumption unlimitedly by default. The unlimited increasing memory causes GitHub Action flakiness. The value I observed during `hive` module test was over 1.8G and growing. - https://docs.oracle.com/javase/10/gctuning/other-considerations.htm#JSGCT-GUID-BFB89453-60C0-42AC-81CA-87D59B0ACE2E > Starting with JDK 8, the permanent generation was removed and the class metadata is allocated in native memory. The amount of native memory that can be used for class metadata is by default unlimited. Use the option -XX:MaxMetaspaceSize to put an upper limit on the amount of native memory used for class metadata. In addition, I increased the following memory limit to 4g consistently from two places. ```xml - <jvmArg>-Xms2048m</jvmArg> - <jvmArg>-Xmx2048m</jvmArg> + <jvmArg>-Xms4g</jvmArg> + <jvmArg>-Xmx4g</jvmArg> ``` ```scala - javaOptions += "-Xmx3g", + javaOptions ++= "-Xmx4g -XX:MaxMetaspaceSize=2g".split(" ").toSeq, ``` ### Why are the changes needed? This will reduce the flakiness in CI environment by limiting the memory usage explicitly. When we limit it with `1g`, Hive module fails with `OOM` like the following. ``` java.lang.OutOfMemoryError: Metaspace Error: Exception in thread "dispatcher-event-loop-110" java.lang.OutOfMemoryError: Metaspace ``` ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass the CIs. Closes #33405 from dongjoon-hyun/SPARK-36195. Lead-authored-by: Dongjoon Hyun <dongj...@apache.org> Co-authored-by: Kyle Bendickson <kbendick...@apple.com> Signed-off-by: Dongjoon Hyun <dongj...@apache.org> (cherry picked from commit d7df7a805fcbdf2435df1e78abd9899d3ca10dd2) Signed-off-by: Dongjoon Hyun <dongj...@apache.org> --- pom.xml | 9 +++++---- project/SparkBuild.scala | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index a49894e..3b32207 100644 --- a/pom.xml +++ b/pom.xml @@ -2611,8 +2611,9 @@ </args> <jvmArgs> <jvmArg>-Xss128m</jvmArg> - <jvmArg>-Xms2048m</jvmArg> - <jvmArg>-Xmx2048m</jvmArg> + <jvmArg>-Xms4g</jvmArg> + <jvmArg>-Xmx4g</jvmArg> + <jvmArg>-XX:MaxMetaspaceSize=2g</jvmArg> <jvmArg>-XX:ReservedCodeCacheSize=${CodeCacheSize}</jvmArg> </jvmArgs> <javacArgs> @@ -2661,7 +2662,7 @@ <include>**/*Suite.java</include> </includes> <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory> - <argLine>-ea -Xmx4g -Xss4m -XX:ReservedCodeCacheSize=${CodeCacheSize} -Dio.netty.tryReflectionSetAccessible=true</argLine> + <argLine>-ea -Xmx4g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} -Dio.netty.tryReflectionSetAccessible=true</argLine> <environmentVariables> <!-- Setting SPARK_DIST_CLASSPATH is a simple way to make sure any child processes @@ -2712,7 +2713,7 @@ <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory> <junitxml>.</junitxml> <filereports>SparkTestSuite.txt</filereports> - <argLine>-ea -Xmx4g -Xss4m -XX:ReservedCodeCacheSize=${CodeCacheSize} -Dio.netty.tryReflectionSetAccessible=true</argLine> + <argLine>-ea -Xmx4g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} -Dio.netty.tryReflectionSetAccessible=true</argLine> <stderr/> <environmentVariables> <!-- diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index f12e94e..b576dbd 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -1120,9 +1120,9 @@ object TestSettings { .map { case (k,v) => s"-D$k=$v" }.toSeq, (Test / javaOptions) += "-ea", // SPARK-29282 This is for consistency between JDK8 and JDK11. - (Test / javaOptions) ++= "-Xmx4g -Xss4m -XX:+UseParallelGC -XX:-UseDynamicNumberOfGCThreads" + (Test / javaOptions) ++= "-Xmx4g -Xss4m -XX:MaxMetaspaceSize=2g -XX:+UseParallelGC -XX:-UseDynamicNumberOfGCThreads" .split(" ").toSeq, - javaOptions += "-Xmx3g", + javaOptions ++= "-Xmx4g -XX:MaxMetaspaceSize=2g".split(" ").toSeq, (Test / javaOptions) ++= { val jdwpEnabled = sys.props.getOrElse("test.jdwp.enabled", "false").toBoolean --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org