This is an automated email from the ASF dual-hosted git repository.

ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 2154fe9ec01 HIVE-28869: added --add-opens JVM options to support JDK 9 
or higher. (#5809).  (Deepak Agrawal, reviewed by Ayush Saxena, Denys Kuzmenko, 
Shohei Okumiya, Tanishq Chugh)
2154fe9ec01 is described below

commit 2154fe9ec011c70a654a1b91404d0689c9c77e3a
Author: deagrawa <[email protected]>
AuthorDate: Mon May 19 12:26:30 2025 +0530

    HIVE-28869: added --add-opens JVM options to support JDK 9 or higher. 
(#5809).  (Deepak Agrawal, reviewed by Ayush Saxena, Denys Kuzmenko, Shohei 
Okumiya, Tanishq Chugh)
---
 .../hadoop/hive/common/JavaVersionUtils.java       | 66 ++++++++++++++++++++++
 .../templeton/tool/TempletonControllerJob.java     |  8 ++-
 .../apache/hadoop/hive/ql/exec/mr/ExecDriver.java  | 15 +++++
 .../apache/hadoop/hive/ql/exec/tez/DagUtils.java   |  3 +-
 .../hadoop/hive/ql/exec/tez/TezSessionState.java   |  6 +-
 5 files changed, 92 insertions(+), 6 deletions(-)

diff --git 
a/common/src/java/org/apache/hadoop/hive/common/JavaVersionUtils.java 
b/common/src/java/org/apache/hadoop/hive/common/JavaVersionUtils.java
new file mode 100644
index 00000000000..624a8c3028a
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/JavaVersionUtils.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common;
+
+public class JavaVersionUtils {
+  private JavaVersionUtils() {
+    throw new IllegalStateException("Utility class");
+  }
+
+  /**
+   * Returns the detected Java major version (e.g., 8, 11, 17).
+   */
+  public static int getJavaMajorVersion() {
+    return Math.max(8, Integer.parseInt(
+        System.getProperty("java.specification.version").split("\\.")[0]));
+  }
+
+  /**
+   * Returns the full --add-opens string if Java 9, or an empty string if Java 
8 or below.
+   */
+  public static String getAddOpensFlagsIfNeeded() {
+    //TODO: Please remove this once the codebase has been migrated to JDK 9 or 
a higher version.
+    int javaVersion = getJavaMajorVersion();
+    if (javaVersion >= 9) {
+      return " --add-opens=java.base/java.net=ALL-UNNAMED" +
+          " --add-opens=java.base/java.util=ALL-UNNAMED" +
+          " --add-opens=java.base/java.util.concurrent=ALL-UNNAMED" +
+          " --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED" +
+          " --add-opens=java.base/java.lang=ALL-UNNAMED" +
+          " --add-opens=java.base/java.io=ALL-UNNAMED" +
+          " --add-opens java.base/java.lang=ALL-UNNAMED" +
+          " --add-opens=java.base/java.lang.invoke=ALL-UNNAMED" +
+          " --add-opens=java.base/java.lang.reflect=ALL-UNNAMED" +
+          " --add-opens=java.base/java.math=ALL-UNNAMED" +
+          " --add-opens=java.base/java.nio=ALL-UNNAMED" +
+          " --add-opens=java.base/java.text=ALL-UNNAMED" +
+          " --add-opens=java.base/java.time=ALL-UNNAMED" +
+          " --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED" +
+          " --add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED" +
+          " --add-opens=java.sql/java.sql=ALL-UNNAMED" +
+          " --add-opens=java.base/sun.nio.ch=ALL-UNNAMED" +
+          " --add-opens=java.base/sun.nio.cs=ALL-UNNAMED" +
+          " --add-opens=java.base/java.util.regex=ALL-UNNAMED" +
+          " --add-opens=java.base/java.security=ALL-UNNAMED" +
+          " --add-opens=java.base/sun.security.provider=ALL-UNNAMED";
+    } else {
+      return ""; // Java 8 or lower does not need add-opens
+    }
+  }
+}
diff --git 
a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
 
b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
index 834b54b6c83..17453216f28 100644
--- 
a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
+++ 
b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
@@ -24,6 +24,8 @@
 import java.sql.SQLException;
 import java.util.Arrays;
 
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hive.common.JavaVersionUtils;
 import org.apache.hive.hcatalog.templeton.LauncherDelegator;
 import org.apache.hive.jdbc.HiveConnection;
 import org.slf4j.Logger;
@@ -121,8 +123,9 @@ public int run(String[] args) throws IOException, 
InterruptedException, ClassNot
     if (amMemoryMB != null && !amMemoryMB.isEmpty()) {
       conf.set(AppConfig.HADOOP_MR_AM_MEMORY_MB, amMemoryMB);
     }
-    String amJavaOpts = appConf.controllerAMChildOpts();
-    if (amJavaOpts != null && !amJavaOpts.isEmpty()) {
+    String amJavaOpts = 
StringUtils.defaultString(appConf.controllerAMChildOpts());
+    amJavaOpts += JavaVersionUtils.getAddOpensFlagsIfNeeded();
+    if (!amJavaOpts.isEmpty()) {
       conf.set(AppConfig.HADOOP_MR_AM_JAVA_OPTS, amJavaOpts);
     }
 
@@ -167,6 +170,7 @@ public int run(String[] args) throws IOException, 
InterruptedException, ClassNot
     }
     return 0;
   }
+
   private String addToken(Job job, String user, String type) throws 
IOException, InterruptedException,
           TException {
     if(!secureMetastoreAccess) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
index e19b73dabcd..7071da726c1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
@@ -36,6 +36,7 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.hive.ql.exec.AddToClassPathAction;
 import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
+import org.apache.hadoop.hive.common.JavaVersionUtils;
 import org.apache.hadoop.hive.ql.log.LogDivertAppenderForTest;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.slf4j.Logger;
@@ -371,6 +372,7 @@ public int execute() {
         }
       }
 
+      addOpensFlagsIfNeeded(job);
       jc = new JobClient(job);
       // make this client wait if job tracker is not behaving well.
       Throttle.checkJobTracker(job, LOG);
@@ -489,6 +491,19 @@ public int execute() {
     return (returnVal);
   }
 
+  private static void addOpensFlagsIfNeeded(JobConf job) {
+    String addOpens = JavaVersionUtils.getAddOpensFlagsIfNeeded();
+    if(!addOpens.isEmpty()) {
+      addJavaOpts(job, "mapreduce.map.java.opts", addOpens);
+      addJavaOpts(job, "mapreduce.reduce.java.opts", addOpens);
+      addJavaOpts(job, "yarn.app.mapreduce.am.command-opts", addOpens);
+    }
+  }
+  private static void addJavaOpts(JobConf job, String conf, String extraOpts) {
+    String javaOpts = StringUtils.defaultString(job.get(conf));
+    job.set(conf, javaOpts + extraOpts);
+  }
+
   public static void propagateSplitSettings(JobConf job, MapWork work) {
     if (work.getNumMapTasks() != null) {
       job.setNumMapTasks(work.getNumMapTasks().intValue());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
index 09d2ff71933..22c7e777a30 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
@@ -53,6 +53,7 @@
 
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hive.common.JavaVersionUtils;
 import org.apache.hive.common.util.HiveStringUtils;
 import org.apache.tez.dag.api.OutputCommitterDescriptor;
 import org.apache.tez.mapreduce.common.MRInputSplitDistributor;
@@ -709,7 +710,6 @@ Map<String, String> getContainerEnvironment(Configuration 
conf, boolean isMap) {
    */
   private static String getContainerJavaOpts(Configuration conf) {
     String javaOpts = HiveConf.getVar(conf, 
HiveConf.ConfVars.HIVE_TEZ_JAVA_OPTS);
-
     String logLevel = HiveConf.getVar(conf, 
HiveConf.ConfVars.HIVE_TEZ_LOG_LEVEL);
     List<String> logProps = Lists.newArrayList();
     TezUtils.addLog4jSystemProperties(logLevel, logProps);
@@ -729,6 +729,7 @@ private static String getContainerJavaOpts(Configuration 
conf) {
       }
       finalOpts = logLevel + " " + MRHelpers.getJavaOptsForMRMapper(conf);
     }
+    finalOpts += JavaVersionUtils.getAddOpensFlagsIfNeeded();
     LOG.debug("Tez container final opts: {}", finalOpts);
     return finalOpts;
   }
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
index 9ee696e1e96..1f56d081e1e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hive.ql.exec.tez;
 
+import org.apache.hadoop.hive.common.JavaVersionUtils;
 import org.apache.hadoop.registry.client.api.RegistryOperations;
 
 import java.io.File;
@@ -548,8 +549,8 @@ private void setupTezParamsBasedOnMR(TezConfiguration conf) 
{
       conf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_ENV, env);
     }
 
-    conf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
-        org.apache.tez.mapreduce.hadoop.MRHelpers.getJavaOptsForMRAM(conf));
+    String mrAmJavaOpts = conf.get(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, 
MRHelpers.getJavaOptsForMRAM(conf));
+    conf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, mrAmJavaOpts + 
JavaVersionUtils.getAddOpensFlagsIfNeeded());
 
     String queueName = conf.get(JobContext.QUEUE_NAME, 
YarnConfiguration.DEFAULT_QUEUE_NAME);
     conf.setIfUnset(TezConfiguration.TEZ_QUEUE_NAME, queueName);
@@ -599,7 +600,6 @@ private void setupTezParamsBasedOnMR(TezConfiguration conf) 
{
       }
     }
   }
-
   private void setupSessionAcls(Configuration tezConf, HiveConf hiveConf) 
throws
       IOException {
 

Reply via email to