This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 01d41e7de41 [SPARK-39879][SQL][TESTS] Reduce local-cluster maximum memory size in `BroadcastJoinSuite*` and `HiveSparkSubmitSuite` 01d41e7de41 is described below commit 01d41e7de418d0a40db7b16ddd0d8546f0794d17 Author: yangjie01 <yangji...@baidu.com> AuthorDate: Wed Jul 27 09:26:47 2022 +0900 [SPARK-39879][SQL][TESTS] Reduce local-cluster maximum memory size in `BroadcastJoinSuite*` and `HiveSparkSubmitSuite` ### What changes were proposed in this pull request? This pr change `local-cluster[2, 1, 1024]` in `BroadcastJoinSuite*` and `HiveSparkSubmitSuite` to `local-cluster[2, 1, 512]` to reduce test maximum memory usage. ### Why are the changes needed? Reduce the maximum memory usage of test cases. ### Does this PR introduce _any_ user-facing change? No, test-only. ### How was this patch tested? Should monitor CI Closes #37298 from LuciferYang/reduce-local-cluster-memory. Authored-by: yangjie01 <yangji...@baidu.com> Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> --- .../sql/execution/joins/BroadcastJoinSuite.scala | 4 +- .../spark/sql/hive/HiveSparkSubmitSuite.scala | 43 +++++++++++++++------- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala index 3cc43e2dd41..6333808b420 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql.execution.joins import scala.reflect.ClassTag import org.apache.spark.AccumulatorSuite +import org.apache.spark.internal.config.EXECUTOR_MEMORY import org.apache.spark.sql.{Dataset, QueryTest, Row, SparkSession} import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BitwiseAnd, BitwiseOr, Cast, Expression, Literal, ShiftLeft} import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide} @@ -56,7 +57,8 @@ abstract class BroadcastJoinSuiteBase extends QueryTest with SQLTestUtils override def beforeAll(): Unit = { super.beforeAll() spark = SparkSession.builder() - .master("local-cluster[2,1,1024]") + .master("local-cluster[2,1,512]") + .config(EXECUTOR_MEMORY.key, "512m") .appName("testing") .getOrCreate() } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala index 18202307fc4..fa4d1b78d1c 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala @@ -33,6 +33,7 @@ import org.scalatest.time.SpanSugar._ import org.apache.spark._ import org.apache.spark.deploy.SparkSubmitTestUtils import org.apache.spark.internal.Logging +import org.apache.spark.internal.config.EXECUTOR_MEMORY import org.apache.spark.internal.config.UI.UI_ENABLED import org.apache.spark.sql.{QueryTest, Row, SparkSession} import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} @@ -74,7 +75,8 @@ class HiveSparkSubmitSuite val args = Seq( "--class", TemporaryHiveUDFTest.getClass.getName.stripSuffix("$"), "--name", "TemporaryHiveUDFTest", - "--master", "local-cluster[2,1,1024]", + "--master", "local-cluster[2,1,512]", + "--conf", s"${EXECUTOR_MEMORY.key}=512m", "--conf", "spark.ui.enabled=false", "--conf", "spark.master.rest.enabled=false", "--driver-java-options", "-Dderby.system.durability=test", @@ -91,7 +93,8 @@ class HiveSparkSubmitSuite val args = Seq( "--class", PermanentHiveUDFTest1.getClass.getName.stripSuffix("$"), "--name", "PermanentHiveUDFTest1", - "--master", "local-cluster[2,1,1024]", + "--master", "local-cluster[2,1,512]", + "--conf", s"${EXECUTOR_MEMORY.key}=512m", "--conf", "spark.ui.enabled=false", "--conf", "spark.master.rest.enabled=false", "--driver-java-options", "-Dderby.system.durability=test", @@ -108,7 +111,8 @@ class HiveSparkSubmitSuite val args = Seq( "--class", PermanentHiveUDFTest2.getClass.getName.stripSuffix("$"), "--name", "PermanentHiveUDFTest2", - "--master", "local-cluster[2,1,1024]", + "--master", "local-cluster[2,1,512]", + "--conf", s"${EXECUTOR_MEMORY.key}=512m", "--conf", "spark.ui.enabled=false", "--conf", "spark.master.rest.enabled=false", "--driver-java-options", "-Dderby.system.durability=test", @@ -127,7 +131,8 @@ class HiveSparkSubmitSuite val args = Seq( "--class", SparkSubmitClassLoaderTest.getClass.getName.stripSuffix("$"), "--name", "SparkSubmitClassLoaderTest", - "--master", "local-cluster[2,1,1024]", + "--master", "local-cluster[2,1,512]", + "--conf", s"${EXECUTOR_MEMORY.key}=512m", "--conf", "spark.ui.enabled=false", "--conf", "spark.master.rest.enabled=false", "--driver-java-options", "-Dderby.system.durability=test", @@ -142,7 +147,8 @@ class HiveSparkSubmitSuite val args = Seq( "--class", SparkSQLConfTest.getClass.getName.stripSuffix("$"), "--name", "SparkSQLConfTest", - "--master", "local-cluster[2,1,1024]", + "--master", "local-cluster[2,1,512]", + "--conf", s"${EXECUTOR_MEMORY.key}=512m", "--conf", "spark.ui.enabled=false", "--conf", "spark.master.rest.enabled=false", "--conf", "spark.sql.hive.metastore.version=0.12", @@ -180,7 +186,8 @@ class HiveSparkSubmitSuite val args = Seq( "--class", SPARK_9757.getClass.getName.stripSuffix("$"), "--name", "SparkSQLConfTest", - "--master", "local-cluster[2,1,1024]", + "--master", "local-cluster[2,1,512]", + "--conf", s"${EXECUTOR_MEMORY.key}=512m", "--conf", "spark.ui.enabled=false", "--conf", "spark.master.rest.enabled=false", "--driver-java-options", "-Dderby.system.durability=test", @@ -193,7 +200,8 @@ class HiveSparkSubmitSuite val args = Seq( "--class", SPARK_11009.getClass.getName.stripSuffix("$"), "--name", "SparkSQLConfTest", - "--master", "local-cluster[2,1,1024]", + "--master", "local-cluster[2,1,512]", + "--conf", s"${EXECUTOR_MEMORY.key}=512m", "--conf", "spark.ui.enabled=false", "--conf", "spark.master.rest.enabled=false", "--driver-java-options", "-Dderby.system.durability=test", @@ -206,7 +214,8 @@ class HiveSparkSubmitSuite val args = Seq( "--class", SPARK_14244.getClass.getName.stripSuffix("$"), "--name", "SparkSQLConfTest", - "--master", "local-cluster[2,1,1024]", + "--master", "local-cluster[2,1,512]", + "--conf", s"${EXECUTOR_MEMORY.key}=512m", "--conf", "spark.ui.enabled=false", "--conf", "spark.master.rest.enabled=false", "--driver-java-options", "-Dderby.system.durability=test", @@ -219,7 +228,8 @@ class HiveSparkSubmitSuite val args = Seq( "--class", SetWarehouseLocationTest.getClass.getName.stripSuffix("$"), "--name", "SetSparkWarehouseLocationTest", - "--master", "local-cluster[2,1,1024]", + "--master", "local-cluster[2,1,512]", + "--conf", s"${EXECUTOR_MEMORY.key}=512m", "--conf", "spark.ui.enabled=false", "--conf", "spark.master.rest.enabled=false", "--driver-java-options", "-Dderby.system.durability=test", @@ -256,7 +266,8 @@ class HiveSparkSubmitSuite val args = Seq( "--class", SetWarehouseLocationTest.getClass.getName.stripSuffix("$"), "--name", "SetHiveWarehouseLocationTest", - "--master", "local-cluster[2,1,1024]", + "--master", "local-cluster[2,1,512]", + "--conf", s"${EXECUTOR_MEMORY.key}=512m", "--conf", "spark.ui.enabled=false", "--conf", "spark.master.rest.enabled=false", "--conf", s"spark.sql.test.expectedWarehouseDir=$hiveWarehouseLocation", @@ -314,7 +325,8 @@ class HiveSparkSubmitSuite val args = Seq( "--class", SPARK_18360.getClass.getName.stripSuffix("$"), "--name", "SPARK-18360", - "--master", "local-cluster[2,1,1024]", + "--master", "local-cluster[2,1,512]", + "--conf", s"${EXECUTOR_MEMORY.key}=512m", "--conf", "spark.ui.enabled=false", "--conf", "spark.master.rest.enabled=false", "--driver-java-options", "-Dderby.system.durability=test", @@ -328,7 +340,8 @@ class HiveSparkSubmitSuite val argsForCreateTable = Seq( "--class", SPARK_18989_CREATE_TABLE.getClass.getName.stripSuffix("$"), "--name", "SPARK-18947", - "--master", "local-cluster[2,1,1024]", + "--master", "local-cluster[2,1,512]", + "--conf", s"${EXECUTOR_MEMORY.key}=512m", "--conf", "spark.ui.enabled=false", "--conf", "spark.master.rest.enabled=false", "--jars", HiveTestJars.getHiveContribJar().getCanonicalPath, @@ -338,7 +351,8 @@ class HiveSparkSubmitSuite val argsForShowTables = Seq( "--class", SPARK_18989_DESC_TABLE.getClass.getName.stripSuffix("$"), "--name", "SPARK-18947", - "--master", "local-cluster[2,1,1024]", + "--master", "local-cluster[2,1,512]", + "--conf", s"${EXECUTOR_MEMORY.key}=512m", "--conf", "spark.ui.enabled=false", "--conf", "spark.master.rest.enabled=false", unusedJar.toString) @@ -359,7 +373,8 @@ class HiveSparkSubmitSuite val args = Seq( "--class", SPARK_34772.getClass.getName.stripSuffix("$"), "--name", "SPARK-34772", - "--master", "local-cluster[2,1,1024]", + "--master", "local-cluster[2,1,512]", + "--conf", s"${EXECUTOR_MEMORY.key}=512m", "--conf", s"${LEGACY_TIME_PARSER_POLICY.key}=LEGACY", "--conf", s"${HiveUtils.HIVE_METASTORE_VERSION.key}=1.2.1", "--conf", s"${HiveUtils.HIVE_METASTORE_JARS.key}=maven", --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org