This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new ee5d85c02 [hive] Support read hadoop / hive conf from system
environment (#1839)
ee5d85c02 is described below
commit ee5d85c0214302765e9a2796a9a3655adaabb33a
Author: hehuiyuan <[email protected]>
AuthorDate: Wed Sep 6 15:20:16 2023 +0800
[hive] Support read hadoop / hive conf from system environment (#1839)
---
.../generated/hive_catalog_configuration.html | 9 +-
.../java/org/apache/paimon/hive/HiveCatalog.java | 28 +++++
.../org/apache/paimon/hive/HiveCatalogOptions.java | 7 +-
.../org/apache/paimon/hive/HiveCatalogTest.java | 33 ++++++
.../apache/paimon/hive/utils/CommonTestUtils.java | 117 +++++++++++++++++++++
5 files changed, 190 insertions(+), 4 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/hive_catalog_configuration.html
b/docs/layouts/shortcodes/generated/hive_catalog_configuration.html
index c94106ea7..0ce063dce 100644
--- a/docs/layouts/shortcodes/generated/hive_catalog_configuration.html
+++ b/docs/layouts/shortcodes/generated/hive_catalog_configuration.html
@@ -30,13 +30,18 @@ under the License.
<td><h5>hadoop-conf-dir</h5></td>
<td style="word-wrap: break-word;">(none)</td>
<td>String</td>
- <td>File directory of the
core-site.xml、hdfs-site.xml、yarn-site.xml、mapred-site.xml. Currently, only
local file system paths are supported.</td>
+ <td>File directory of the
core-site.xml、hdfs-site.xml、yarn-site.xml、mapred-site.xml. Currently, only
local file system paths are supported.
+If not configured, try to load from 'HADOOP_CONF_DIR' or 'HADOOP_HOME' system
environment.
+Configure Priority: 1.from 'hadoop-conf-dir' 2.from HADOOP_CONF_DIR 3.from
HADOOP_HOME/conf 4.HADOOP_HOME/etc/hadoop.
+</td>
</tr>
<tr>
<td><h5>hive-conf-dir</h5></td>
<td style="word-wrap: break-word;">(none)</td>
<td>String</td>
- <td>File directory of the hive-site.xml , used to create
HiveMetastoreClient and security authentication, such as Kerberos, LDAP, Ranger
and so on</td>
+ <td>File directory of the hive-site.xml , used to create
HiveMetastoreClient and security authentication, such as Kerberos, LDAP, Ranger
and so on.
+If not configured, try to load from 'HIVE_CONF_DIR' env.
+</td>
</tr>
<tr>
<td><h5>location-in-properties</h5></td>
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
index a4af771c5..b408214ca 100644
---
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
@@ -660,6 +660,14 @@ public class HiveCatalog extends AbstractCatalog {
public static HiveConf createHiveConf(
@Nullable String hiveConfDir, @Nullable String hadoopConfDir) {
+ // try to load from system env.
+ if (isNullOrWhitespaceOnly(hiveConfDir)) {
+ hiveConfDir = possibleHiveConfPath();
+ }
+ if (isNullOrWhitespaceOnly(hadoopConfDir)) {
+ hadoopConfDir = possibleHadoopConfPath();
+ }
+
// create HiveConf from hadoop configuration with hadoop conf
directory configured.
Configuration hadoopConf = null;
if (!isNullOrWhitespaceOnly(hadoopConfDir)) {
@@ -751,4 +759,24 @@ public class HiveCatalog extends AbstractCatalog {
}
return null;
}
+
+ public static String possibleHadoopConfPath() {
+ String possiblePath = null;
+ if (System.getenv("HADOOP_CONF_DIR") != null) {
+ possiblePath = System.getenv("HADOOP_CONF_DIR");
+ } else if (System.getenv("HADOOP_HOME") != null) {
+ String possiblePath1 = System.getenv("HADOOP_HOME") + "/conf";
+ String possiblePath2 = System.getenv("HADOOP_HOME") +
"/etc/hadoop";
+ if (new File(possiblePath1).exists()) {
+ possiblePath = possiblePath1;
+ } else if (new File(possiblePath2).exists()) {
+ possiblePath = possiblePath2;
+ }
+ }
+ return possiblePath;
+ }
+
+ public static String possibleHiveConfPath() {
+ return System.getenv("HIVE_CONF_DIR");
+ }
}
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java
index 431e74dcf..0f1b4d454 100644
---
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java
@@ -31,14 +31,17 @@ public final class HiveCatalogOptions {
.stringType()
.noDefaultValue()
.withDescription(
- "File directory of the hive-site.xml , used to
create HiveMetastoreClient and security authentication, such as Kerberos, LDAP,
Ranger and so on");
+ "File directory of the hive-site.xml , used to
create HiveMetastoreClient and security authentication, such as Kerberos, LDAP,
Ranger and so on.\n"
+ + "If not configured, try to load from
'HIVE_CONF_DIR' env.\n");
public static final ConfigOption<String> HADOOP_CONF_DIR =
ConfigOptions.key("hadoop-conf-dir")
.stringType()
.noDefaultValue()
.withDescription(
- "File directory of the
core-site.xml、hdfs-site.xml、yarn-site.xml、mapred-site.xml. Currently, only
local file system paths are supported.");
+ "File directory of the
core-site.xml、hdfs-site.xml、yarn-site.xml、mapred-site.xml. Currently, only
local file system paths are supported.\n"
+ + "If not configured, try to load from
'HADOOP_CONF_DIR' or 'HADOOP_HOME' system environment.\n"
+ + "Configure Priority: 1.from
'hadoop-conf-dir' 2.from HADOOP_CONF_DIR 3.from HADOOP_HOME/conf
4.HADOOP_HOME/etc/hadoop.\n");
public static final ConfigOption<Boolean> LOCATION_IN_PROPERTIES =
ConfigOptions.key("location-in-properties")
diff --git
a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java
b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java
index 4959866f1..ae57695e5 100644
---
a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java
+++
b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java
@@ -20,6 +20,7 @@ package org.apache.paimon.hive;
import org.apache.paimon.catalog.CatalogTestBase;
import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.hive.utils.CommonTestUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
@@ -28,7 +29,9 @@ import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
import static
org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTORECONNECTURLKEY;
@@ -115,4 +118,34 @@ public class HiveCatalogTest extends CatalogTestBase {
// reset back to default value
HiveConf.setHiveSiteLocation(HiveConf.class.getClassLoader().getResource("hive-site.xml"));
}
+
+ @Test
+ public void testHadoopConfDirFromEnv() {
+ Map<String, String> newEnv = new HashMap<>(System.getenv());
+ newEnv.put("HADOOP_CONF_DIR", HADOOP_CONF_DIR);
+ // add HADOOP_CONF_DIR to system environment
+ CommonTestUtils.setEnv(newEnv, false);
+
+ HiveConf hiveConf = HiveCatalog.createHiveConf(null, null);
+ assertThat(hiveConf.get("fs.defaultFS")).isEqualTo("dummy-fs");
+ }
+
+ @Test
+ public void testHiveConfDirFromEnv() {
+ try {
+ testHiveConfDirFromEnvImpl();
+ } finally {
+ cleanUpHiveConfDir();
+ }
+ }
+
+ private void testHiveConfDirFromEnvImpl() {
+ Map<String, String> newEnv = new HashMap<>(System.getenv());
+ newEnv.put("HIVE_CONF_DIR", HIVE_CONF_DIR);
+ // add HIVE_CONF_DIR to system environment
+ CommonTestUtils.setEnv(newEnv, false);
+
+ HiveConf hiveConf = HiveCatalog.createHiveConf(null, null);
+ assertThat(hiveConf.get("hive.metastore.uris")).isEqualTo("dummy-hms");
+ }
}
diff --git
a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/utils/CommonTestUtils.java
b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/utils/CommonTestUtils.java
new file mode 100644
index 000000000..6196e9e74
--- /dev/null
+++
b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/utils/CommonTestUtils.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.hive.utils;
+
+import java.lang.reflect.Field;
+import java.time.Duration;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Supplier;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** This class contains reusable utility methods for unit tests. */
+public class CommonTestUtils {
+
+ // ------------------------------------------------------------------------
+ // Manipulation of environment
+ // ------------------------------------------------------------------------
+
+ public static void setEnv(Map<String, String> newenv) {
+ setEnv(newenv, true);
+ }
+
+ // This code is taken slightly modified from:
http://stackoverflow.com/a/7201825/568695
+ // it changes the environment variables of this JVM. Use only for testing
purposes!
+ @SuppressWarnings("unchecked")
+ public static void setEnv(Map<String, String> newenv, boolean
clearExisting) {
+ try {
+ Map<String, String> env = System.getenv();
+ Class<?> clazz = env.getClass();
+ Field field = clazz.getDeclaredField("m");
+ field.setAccessible(true);
+ Map<String, String> map = (Map<String, String>) field.get(env);
+ if (clearExisting) {
+ map.clear();
+ }
+ map.putAll(newenv);
+
+ // only for Windows
+ Class<?> processEnvironmentClass =
Class.forName("java.lang.ProcessEnvironment");
+ try {
+ @SuppressWarnings("JavaReflectionMemberAccess")
+ Field theCaseInsensitiveEnvironmentField =
+
processEnvironmentClass.getDeclaredField("theCaseInsensitiveEnvironment");
+ theCaseInsensitiveEnvironmentField.setAccessible(true);
+ Map<String, String> cienv =
+ (Map<String, String>)
theCaseInsensitiveEnvironmentField.get(null);
+ if (clearExisting) {
+ cienv.clear();
+ }
+ cienv.putAll(newenv);
+ } catch (NoSuchFieldException ignored) {
+ }
+
+ } catch (Exception e1) {
+ throw new RuntimeException(e1);
+ }
+ }
+
+ /** Checks whether an exception with a message occurs when running a piece
of code. */
+ public static void assertThrows(
+ String msg, Class<? extends Exception> expected, Callable<?> code)
{
+
assertThatThrownBy(code::call).isInstanceOf(expected).hasMessageContaining(msg);
+ }
+
+ public static void waitUtil(Supplier<Boolean> condition, Duration timeout,
Duration pause)
+ throws TimeoutException, InterruptedException {
+ waitUtil(condition, timeout, pause, "Failed to wait for condition to
be true.");
+ }
+
+ /**
+ * Wait util the given condition is met or timeout.
+ *
+ * @param condition the condition to wait for.
+ * @param timeout the maximum time to wait for the condition to become
true.
+ * @param pause delay between condition checks.
+ * @param errorMsg the error message to include in the
<code>TimeoutException</code> if the
+ * condition was not met before timeout.
+ * @throws TimeoutException if the condition is not met before timeout.
+ * @throws InterruptedException if the thread is interrupted.
+ */
+ @SuppressWarnings("BusyWait")
+ public static void waitUtil(
+ Supplier<Boolean> condition, Duration timeout, Duration pause,
String errorMsg)
+ throws TimeoutException, InterruptedException {
+ long timeoutMs = timeout.toMillis();
+ if (timeoutMs <= 0) {
+ throw new IllegalArgumentException("The timeout must be
positive.");
+ }
+ long startingTime = System.currentTimeMillis();
+ boolean conditionResult = condition.get();
+ while (!conditionResult && System.currentTimeMillis() - startingTime <
timeoutMs) {
+ conditionResult = condition.get();
+ Thread.sleep(pause.toMillis());
+ }
+ if (!conditionResult) {
+ throw new TimeoutException(errorMsg);
+ }
+ }
+}
