This is an automated email from the ASF dual-hosted git repository. forwardxu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push: new 495b6fbb062 [HUDI-5332] HiveSyncTool can avoid initializing all permanent custom functions of Hive (#7385) 495b6fbb062 is described below commit 495b6fbb062c843d19de420acfefd3a6a2ee3c58 Author: cxzl25 <cxz...@users.noreply.github.com> AuthorDate: Thu Dec 29 19:17:43 2022 +0800 [HUDI-5332] HiveSyncTool can avoid initializing all permanent custom functions of Hive (#7385) --- .../main/java/org/apache/hudi/hive/ddl/HMSDDLExecutor.java | 11 ++++++++++- .../java/org/apache/hudi/hive/ddl/HiveQueryDDLExecutor.java | 11 ++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HMSDDLExecutor.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HMSDDLExecutor.java index c14536a2774..fbba5861741 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HMSDDLExecutor.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HMSDDLExecutor.java @@ -30,6 +30,7 @@ import org.apache.hudi.sync.common.model.PartitionValueExtractor; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.Database; @@ -48,6 +49,7 @@ import org.apache.log4j.Logger; import org.apache.parquet.schema.MessageType; import org.apache.thrift.TException; +import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; @@ -78,7 +80,14 @@ public class HMSDDLExecutor implements DDLExecutor { public HMSDDLExecutor(HiveSyncConfig syncConfig) throws HiveException, MetaException { this.syncConfig = syncConfig; this.databaseName = syncConfig.getStringOrDefault(META_SYNC_DATABASE_NAME); - this.client = Hive.get(syncConfig.getHiveConf()).getMSC(); + HiveConf hiveConf = syncConfig.getHiveConf(); + IMetaStoreClient tempMetaStoreClient; + try { + tempMetaStoreClient = ((Hive) Hive.class.getMethod("getWithoutRegisterFns", HiveConf.class).invoke(null, hiveConf)).getMSC(); + } catch (NoSuchMethodException | IllegalAccessException | IllegalArgumentException | InvocationTargetException ex) { + tempMetaStoreClient = Hive.get(hiveConf).getMSC(); + } + this.client = tempMetaStoreClient; try { this.partitionValueExtractor = (PartitionValueExtractor) Class.forName(syncConfig.getStringOrDefault(META_SYNC_PARTITION_EXTRACTOR_CLASS)).newInstance(); diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HiveQueryDDLExecutor.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HiveQueryDDLExecutor.java index 93ae3cfbf73..e0f7dab5f35 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HiveQueryDDLExecutor.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HiveQueryDDLExecutor.java @@ -23,6 +23,7 @@ import org.apache.hudi.hive.HiveSyncConfig; import org.apache.hudi.hive.HoodieHiveSyncException; import org.apache.hudi.hive.util.HivePartitionUtil; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; @@ -37,6 +38,7 @@ import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -59,7 +61,14 @@ public class HiveQueryDDLExecutor extends QueryBasedDDLExecutor { public HiveQueryDDLExecutor(HiveSyncConfig config) throws HiveException, MetaException { super(config); - this.metaStoreClient = Hive.get(config.getHiveConf()).getMSC(); + HiveConf hiveConf = config.getHiveConf(); + IMetaStoreClient tempMetaStoreClient; + try { + tempMetaStoreClient = ((Hive) Hive.class.getMethod("getWithoutRegisterFns", HiveConf.class).invoke(null, hiveConf)).getMSC(); + } catch (NoSuchMethodException | IllegalAccessException | IllegalArgumentException | InvocationTargetException ex) { + tempMetaStoreClient = Hive.get(hiveConf).getMSC(); + } + this.metaStoreClient = tempMetaStoreClient; try { this.sessionState = new SessionState(config.getHiveConf(), UserGroupInformation.getCurrentUser().getShortUserName());