This is an automated email from the ASF dual-hosted git repository. yao pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 07025f1c13c [SPARK-37825][SQL][LAUNCHER] Make spark beeline be able to handle javaOpts 07025f1c13c is described below commit 07025f1c13c9f6ed5dce11a1edb9868439e78cb3 Author: Kent Yao <y...@apache.org> AuthorDate: Fri May 27 10:15:08 2022 +0800 [SPARK-37825][SQL][LAUNCHER] Make spark beeline be able to handle javaOpts ### What changes were proposed in this pull request? Currently, we build the beeline command with SPARK_DRIVER_MEMORY only and are not able to set extra java opts. Besides, the beeline is not a DRIVER-type thing. In this PR, we add ```java # - SPARK_BEELINE_OPTS, to set config properties only for the beeline cli (e.g. "-Dx=y") # - SPARK_BEELINE_MEMORY, Memory for beeline (e.g. 1000M, 2G) (Default: 1G) ``` ### Why are the changes needed? Make spark beeline configurable for extra JVM settings, like GC ### Does this PR introduce _any_ user-facing change? yes, add 2 environment variables. ### How was this patch tested? add a new java ut Closes #35116 from yaooqinn/SPARK-37825. Authored-by: Kent Yao <y...@apache.org> Signed-off-by: Kent Yao <y...@apache.org> --- conf/spark-env.sh.template | 4 +++ .../spark/launcher/SparkClassCommandBuilder.java | 4 +++ .../launcher/SparkClassCommandBuilderSuite.java | 39 ++++++++++++++++++++++ pom.xml | 1 + project/SparkBuild.scala | 3 +- 5 files changed, 50 insertions(+), 1 deletion(-) diff --git a/conf/spark-env.sh.template b/conf/spark-env.sh.template index a2f1380692f..e9491995e72 100755 --- a/conf/spark-env.sh.template +++ b/conf/spark-env.sh.template @@ -75,3 +75,7 @@ # You might get better performance to enable these options if using native BLAS (see SPARK-21305). # - MKL_NUM_THREADS=1 Disable multi-threading of Intel MKL # - OPENBLAS_NUM_THREADS=1 Disable multi-threading of OpenBLAS + +# Options for beeline +# - SPARK_BEELINE_OPTS, to set config properties only for the beeline cli (e.g. "-Dx=y") +# - SPARK_BEELINE_MEMORY, Memory for beeline (e.g. 1000M, 2G) (Default: 1G) diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkClassCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkClassCommandBuilder.java index fd056bb90e0..a9daf0e2572 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkClassCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkClassCommandBuilder.java @@ -90,6 +90,10 @@ class SparkClassCommandBuilder extends AbstractCommandBuilder { extraClassPath = getenv("SPARK_DAEMON_CLASSPATH"); memKey = "SPARK_DAEMON_MEMORY"; break; + case "org.apache.hive.beeline.BeeLine": + javaOptsKeys.add("SPARK_BEELINE_OPTS"); + memKey = "SPARK_BEELINE_MEMORY"; + break; default: memKey = "SPARK_DRIVER_MEMORY"; break; diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkClassCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkClassCommandBuilderSuite.java new file mode 100644 index 00000000000..3f6d66bb5c9 --- /dev/null +++ b/launcher/src/test/java/org/apache/spark/launcher/SparkClassCommandBuilderSuite.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.launcher; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; + +import org.junit.Test; + +import static org.junit.Assert.*; + +public class SparkClassCommandBuilderSuite extends BaseSuite { + + @Test + public void testBeelineBuilder() throws Exception { + List<String> args = Arrays.asList("myBeelineArg"); + SparkClassCommandBuilder builder = + new SparkClassCommandBuilder("org.apache.hive.beeline.BeeLine", args); + List<String> strings = builder.buildCommand(new HashMap<>()); + assertTrue(strings.contains("-DmyKey=yourValue")); + assertTrue(strings.contains("myBeelineArg")); + } +} diff --git a/pom.xml b/pom.xml index e967d17d8fe..15675cb1a14 100644 --- a/pom.xml +++ b/pom.xml @@ -2950,6 +2950,7 @@ <SPARK_SCALA_VERSION>${scala.binary.version}</SPARK_SCALA_VERSION> <SPARK_TESTING>1</SPARK_TESTING> <JAVA_HOME>${test.java.home}</JAVA_HOME> + <SPARK_BEELINE_OPTS>-DmyKey=yourValue</SPARK_BEELINE_OPTS> </environmentVariables> <systemProperties> <log4j.configurationFile>file:src/test/resources/log4j2.properties</log4j.configurationFile> diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 3309b72a557..c205781ad10 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -1150,7 +1150,8 @@ object TestSettings { "SPARK_PREPEND_CLASSES" -> "1", "SPARK_SCALA_VERSION" -> scalaBinaryVersion.value, "SPARK_TESTING" -> "1", - "JAVA_HOME" -> sys.env.get("JAVA_HOME").getOrElse(sys.props("java.home"))), + "JAVA_HOME" -> sys.env.get("JAVA_HOME").getOrElse(sys.props("java.home")), + "SPARK_BEELINE_OPTS" -> "-DmyKey=yourValue"), (Test / javaOptions) += s"-Djava.io.tmpdir=$testTempDir", (Test / javaOptions) += "-Dspark.test.home=" + sparkHome, (Test / javaOptions) += "-Dspark.testing=1", --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org