This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 2154fe9ec01 HIVE-28869: added --add-opens JVM options to support JDK 9
or higher. (#5809). (Deepak Agrawal, reviewed by Ayush Saxena, Denys Kuzmenko,
Shohei Okumiya, Tanishq Chugh)
2154fe9ec01 is described below
commit 2154fe9ec011c70a654a1b91404d0689c9c77e3a
Author: deagrawa <[email protected]>
AuthorDate: Mon May 19 12:26:30 2025 +0530
HIVE-28869: added --add-opens JVM options to support JDK 9 or higher.
(#5809). (Deepak Agrawal, reviewed by Ayush Saxena, Denys Kuzmenko, Shohei
Okumiya, Tanishq Chugh)
---
.../hadoop/hive/common/JavaVersionUtils.java | 66 ++++++++++++++++++++++
.../templeton/tool/TempletonControllerJob.java | 8 ++-
.../apache/hadoop/hive/ql/exec/mr/ExecDriver.java | 15 +++++
.../apache/hadoop/hive/ql/exec/tez/DagUtils.java | 3 +-
.../hadoop/hive/ql/exec/tez/TezSessionState.java | 6 +-
5 files changed, 92 insertions(+), 6 deletions(-)
diff --git
a/common/src/java/org/apache/hadoop/hive/common/JavaVersionUtils.java
b/common/src/java/org/apache/hadoop/hive/common/JavaVersionUtils.java
new file mode 100644
index 00000000000..624a8c3028a
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/JavaVersionUtils.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common;
+
+public class JavaVersionUtils {
+ private JavaVersionUtils() {
+ throw new IllegalStateException("Utility class");
+ }
+
+ /**
+ * Returns the detected Java major version (e.g., 8, 11, 17).
+ */
+ public static int getJavaMajorVersion() {
+ return Math.max(8, Integer.parseInt(
+ System.getProperty("java.specification.version").split("\\.")[0]));
+ }
+
+ /**
+ * Returns the full --add-opens string if Java 9, or an empty string if Java
8 or below.
+ */
+ public static String getAddOpensFlagsIfNeeded() {
+ //TODO: Please remove this once the codebase has been migrated to JDK 9 or
a higher version.
+ int javaVersion = getJavaMajorVersion();
+ if (javaVersion >= 9) {
+ return " --add-opens=java.base/java.net=ALL-UNNAMED" +
+ " --add-opens=java.base/java.util=ALL-UNNAMED" +
+ " --add-opens=java.base/java.util.concurrent=ALL-UNNAMED" +
+ " --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED" +
+ " --add-opens=java.base/java.lang=ALL-UNNAMED" +
+ " --add-opens=java.base/java.io=ALL-UNNAMED" +
+ " --add-opens java.base/java.lang=ALL-UNNAMED" +
+ " --add-opens=java.base/java.lang.invoke=ALL-UNNAMED" +
+ " --add-opens=java.base/java.lang.reflect=ALL-UNNAMED" +
+ " --add-opens=java.base/java.math=ALL-UNNAMED" +
+ " --add-opens=java.base/java.nio=ALL-UNNAMED" +
+ " --add-opens=java.base/java.text=ALL-UNNAMED" +
+ " --add-opens=java.base/java.time=ALL-UNNAMED" +
+ " --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED" +
+ " --add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED" +
+ " --add-opens=java.sql/java.sql=ALL-UNNAMED" +
+ " --add-opens=java.base/sun.nio.ch=ALL-UNNAMED" +
+ " --add-opens=java.base/sun.nio.cs=ALL-UNNAMED" +
+ " --add-opens=java.base/java.util.regex=ALL-UNNAMED" +
+ " --add-opens=java.base/java.security=ALL-UNNAMED" +
+ " --add-opens=java.base/sun.security.provider=ALL-UNNAMED";
+ } else {
+ return ""; // Java 8 or lower does not need add-opens
+ }
+ }
+}
diff --git
a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
index 834b54b6c83..17453216f28 100644
---
a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
+++
b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
@@ -24,6 +24,8 @@
import java.sql.SQLException;
import java.util.Arrays;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hive.common.JavaVersionUtils;
import org.apache.hive.hcatalog.templeton.LauncherDelegator;
import org.apache.hive.jdbc.HiveConnection;
import org.slf4j.Logger;
@@ -121,8 +123,9 @@ public int run(String[] args) throws IOException,
InterruptedException, ClassNot
if (amMemoryMB != null && !amMemoryMB.isEmpty()) {
conf.set(AppConfig.HADOOP_MR_AM_MEMORY_MB, amMemoryMB);
}
- String amJavaOpts = appConf.controllerAMChildOpts();
- if (amJavaOpts != null && !amJavaOpts.isEmpty()) {
+ String amJavaOpts =
StringUtils.defaultString(appConf.controllerAMChildOpts());
+ amJavaOpts += JavaVersionUtils.getAddOpensFlagsIfNeeded();
+ if (!amJavaOpts.isEmpty()) {
conf.set(AppConfig.HADOOP_MR_AM_JAVA_OPTS, amJavaOpts);
}
@@ -167,6 +170,7 @@ public int run(String[] args) throws IOException,
InterruptedException, ClassNot
}
return 0;
}
+
private String addToken(Job job, String user, String type) throws
IOException, InterruptedException,
TException {
if(!secureMetastoreAccess) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
index e19b73dabcd..7071da726c1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
@@ -36,6 +36,7 @@
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hive.ql.exec.AddToClassPathAction;
import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
+import org.apache.hadoop.hive.common.JavaVersionUtils;
import org.apache.hadoop.hive.ql.log.LogDivertAppenderForTest;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.slf4j.Logger;
@@ -371,6 +372,7 @@ public int execute() {
}
}
+ addOpensFlagsIfNeeded(job);
jc = new JobClient(job);
// make this client wait if job tracker is not behaving well.
Throttle.checkJobTracker(job, LOG);
@@ -489,6 +491,19 @@ public int execute() {
return (returnVal);
}
+ private static void addOpensFlagsIfNeeded(JobConf job) {
+ String addOpens = JavaVersionUtils.getAddOpensFlagsIfNeeded();
+ if(!addOpens.isEmpty()) {
+ addJavaOpts(job, "mapreduce.map.java.opts", addOpens);
+ addJavaOpts(job, "mapreduce.reduce.java.opts", addOpens);
+ addJavaOpts(job, "yarn.app.mapreduce.am.command-opts", addOpens);
+ }
+ }
+ private static void addJavaOpts(JobConf job, String conf, String extraOpts) {
+ String javaOpts = StringUtils.defaultString(job.get(conf));
+ job.set(conf, javaOpts + extraOpts);
+ }
+
public static void propagateSplitSettings(JobConf job, MapWork work) {
if (work.getNumMapTasks() != null) {
job.setNumMapTasks(work.getNumMapTasks().intValue());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
index 09d2ff71933..22c7e777a30 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
@@ -53,6 +53,7 @@
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hive.common.JavaVersionUtils;
import org.apache.hive.common.util.HiveStringUtils;
import org.apache.tez.dag.api.OutputCommitterDescriptor;
import org.apache.tez.mapreduce.common.MRInputSplitDistributor;
@@ -709,7 +710,6 @@ Map<String, String> getContainerEnvironment(Configuration
conf, boolean isMap) {
*/
private static String getContainerJavaOpts(Configuration conf) {
String javaOpts = HiveConf.getVar(conf,
HiveConf.ConfVars.HIVE_TEZ_JAVA_OPTS);
-
String logLevel = HiveConf.getVar(conf,
HiveConf.ConfVars.HIVE_TEZ_LOG_LEVEL);
List<String> logProps = Lists.newArrayList();
TezUtils.addLog4jSystemProperties(logLevel, logProps);
@@ -729,6 +729,7 @@ private static String getContainerJavaOpts(Configuration
conf) {
}
finalOpts = logLevel + " " + MRHelpers.getJavaOptsForMRMapper(conf);
}
+ finalOpts += JavaVersionUtils.getAddOpensFlagsIfNeeded();
LOG.debug("Tez container final opts: {}", finalOpts);
return finalOpts;
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
index 9ee696e1e96..1f56d081e1e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hive.ql.exec.tez;
+import org.apache.hadoop.hive.common.JavaVersionUtils;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import java.io.File;
@@ -548,8 +549,8 @@ private void setupTezParamsBasedOnMR(TezConfiguration conf)
{
conf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_ENV, env);
}
- conf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
- org.apache.tez.mapreduce.hadoop.MRHelpers.getJavaOptsForMRAM(conf));
+ String mrAmJavaOpts = conf.get(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
MRHelpers.getJavaOptsForMRAM(conf));
+ conf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, mrAmJavaOpts +
JavaVersionUtils.getAddOpensFlagsIfNeeded());
String queueName = conf.get(JobContext.QUEUE_NAME,
YarnConfiguration.DEFAULT_QUEUE_NAME);
conf.setIfUnset(TezConfiguration.TEZ_QUEUE_NAME, queueName);
@@ -599,7 +600,6 @@ private void setupTezParamsBasedOnMR(TezConfiguration conf)
{
}
}
}
-
private void setupSessionAcls(Configuration tezConf, HiveConf hiveConf)
throws
IOException {