Repository: hive Updated Branches: refs/heads/branch-2 61bbad6ea -> 5c2e3b4d8
HIVE-17621: Hive-site settings are ignored during HCatInputFormat split-calculation (Chris Drome, reviewed by Mithun Radhakrishnan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5c2e3b4d Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5c2e3b4d Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5c2e3b4d Branch: refs/heads/branch-2 Commit: 5c2e3b4d80b8fe13b41aa43ea0c2dbbefa283037 Parents: 61bbad6 Author: Mithun RK <mit...@apache.org> Authored: Wed Sep 27 11:10:36 2017 -0700 Committer: Mithun RK <mith...@oath.com> Committed: Thu Sep 28 16:10:21 2017 -0700 ---------------------------------------------------------------------- .../apache/hive/hcatalog/common/HCatUtil.java | 24 +++++++++++++++++--- .../hcatalog/mapreduce/HCatBaseInputFormat.java | 8 +++++-- 2 files changed, 27 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/5c2e3b4d/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java index 8b927af..d18e206 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java @@ -33,6 +33,8 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import javax.security.auth.login.LoginException; + import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -76,8 +78,6 @@ import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.security.auth.login.LoginException; - public class HCatUtil { private static final Logger LOG = LoggerFactory.getLogger(HCatUtil.class); @@ -642,6 +642,25 @@ public class HCatUtil { return jobConf; } + public static Map<String,String> getHCatKeyHiveConf(JobConf conf) { + Map<String,String> hiveProperties = new HashMap<String,String>(); + if (conf.get(HCatConstants.HCAT_KEY_HIVE_CONF) != null) { + try { + Properties properties = (Properties) HCatUtil.deserialize( + conf.get(HCatConstants.HCAT_KEY_HIVE_CONF)); + for (Map.Entry<Object, Object> prop : properties.entrySet()) { + if (conf.get((String)prop.getKey()) == null) { + hiveProperties.put(prop.getKey().toString(), prop.getValue().toString()); + } + } + } catch (IOException e) { + throw new IllegalStateException( + "Failed to deserialize hive conf", e); + } + } + return hiveProperties; + } + public static void copyJobPropertiesToJobConf( Map<String, String> jobProperties, JobConf jobConf) { for (Map.Entry<String, String> entry : jobProperties.entrySet()) { @@ -649,7 +668,6 @@ public class HCatUtil { } } - public static boolean isHadoop23() { String version = org.apache.hadoop.util.VersionInfo.getVersion(); if (version.matches("\\b0\\.23\\..+\\b")||version.matches("\\b2\\..*")) http://git-wip-us.apache.org/repos/asf/hive/blob/5c2e3b4d/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java index 9caff7a..ec1e1ca 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatBaseInputFormat.java @@ -126,16 +126,20 @@ public abstract class HCatBaseInputFormat } HiveStorageHandler storageHandler; - JobConf jobConf; + Map<String,String> hiveProps = null; //For each matching partition, call getSplits on the underlying InputFormat for (PartInfo partitionInfo : partitionInfoList) { - jobConf = HCatUtil.getJobConfFromContext(jobContext); + JobConf jobConf = HCatUtil.getJobConfFromContext(jobContext); + if (hiveProps == null) { + hiveProps = HCatUtil.getHCatKeyHiveConf(jobConf); + } List<String> setInputPath = setInputPath(jobConf, partitionInfo.getLocation()); if (setInputPath.isEmpty()) { continue; } Map<String, String> jobProperties = partitionInfo.getJobProperties(); + HCatUtil.copyJobPropertiesToJobConf(hiveProps, jobConf); HCatUtil.copyJobPropertiesToJobConf(jobProperties, jobConf); storageHandler = HCatUtil.getStorageHandler(
