Repository: hive Updated Branches: refs/heads/master c29ecc664 -> d72121f0c
HIVE-17112: Reduce logging in HiveSparkClientFactory and RemoteHiveSparkClient (Sahil Takiar, reviewed by Peter Vary) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d72121f0 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d72121f0 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d72121f0 Branch: refs/heads/master Commit: d72121f0c2afdd0bbc68c0f7a9b5e89560a3a4aa Parents: c29ecc6 Author: Sahil Takiar <takiar.sa...@gmail.com> Authored: Wed Sep 20 21:18:25 2017 -0700 Committer: Sahil Takiar <stak...@cloudera.com> Committed: Wed Sep 20 21:18:25 2017 -0700 ---------------------------------------------------------------------- .../hive/ql/exec/spark/HiveSparkClientFactory.java | 16 ++++++++-------- .../hive/ql/exec/spark/RemoteHiveSparkClient.java | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/d72121f0/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java index 6e9ba7c..194585e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java @@ -95,21 +95,21 @@ public class HiveSparkClientFactory { inputStream = HiveSparkClientFactory.class.getClassLoader() .getResourceAsStream(SPARK_DEFAULT_CONF_FILE); if (inputStream != null) { - LOG.info("loading spark properties from:" + SPARK_DEFAULT_CONF_FILE); + LOG.info("loading spark properties from: " + SPARK_DEFAULT_CONF_FILE); Properties properties = new Properties(); properties.load(new InputStreamReader(inputStream, CharsetNames.UTF_8)); for (String propertyName : properties.stringPropertyNames()) { if (propertyName.startsWith("spark")) { String value = properties.getProperty(propertyName); sparkConf.put(propertyName, properties.getProperty(propertyName)); - LOG.info(String.format( + LOG.debug(String.format( "load spark property from %s (%s -> %s).", SPARK_DEFAULT_CONF_FILE, propertyName, LogUtils.maskIfPassword(propertyName,value))); } } } } catch (IOException e) { - LOG.info("Failed to open spark configuration file:" + LOG.info("Failed to open spark configuration file: " + SPARK_DEFAULT_CONF_FILE, e); } finally { if (inputStream != null) { @@ -156,7 +156,7 @@ public class HiveSparkClientFactory { if (propertyName.startsWith("spark")) { String value = hiveConf.get(propertyName); sparkConf.put(propertyName, value); - LOG.info(String.format( + LOG.debug(String.format( "load spark property from hive configuration (%s -> %s).", propertyName, LogUtils.maskIfPassword(propertyName,value))); } else if (propertyName.startsWith("yarn") && @@ -166,7 +166,7 @@ public class HiveSparkClientFactory { // started with spark prefix, Spark would remove spark.hadoop prefix lately and add // it to its hadoop configuration. sparkConf.put("spark.hadoop." + propertyName, value); - LOG.info(String.format( + LOG.debug(String.format( "load yarn property from hive configuration in %s mode (%s -> %s).", sparkMaster, propertyName, LogUtils.maskIfPassword(propertyName,value))); } else if (propertyName.equals(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY)) { @@ -180,19 +180,19 @@ public class HiveSparkClientFactory { // Spark problem. String value = hiveConf.get(propertyName); sparkConf.put("spark.hadoop." + propertyName, value); - LOG.info(String.format( + LOG.debug(String.format( "load HBase configuration (%s -> %s).", propertyName, LogUtils.maskIfPassword(propertyName,value))); } else if (propertyName.startsWith("oozie")) { String value = hiveConf.get(propertyName); sparkConf.put("spark." + propertyName, value); - LOG.info(String.format( + LOG.debug(String.format( "Pass Oozie configuration (%s -> %s).", propertyName, LogUtils.maskIfPassword(propertyName,value))); } if (RpcConfiguration.HIVE_SPARK_RSC_CONFIGS.contains(propertyName)) { String value = RpcConfiguration.getValue(hiveConf, propertyName); sparkConf.put(propertyName, value); - LOG.info(String.format( + LOG.debug(String.format( "load RPC property from hive configuration (%s -> %s).", propertyName, LogUtils.maskIfPassword(propertyName,value))); } http://git-wip-us.apache.org/repos/asf/hive/blob/d72121f0/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java index dbcf26a..102e41b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java @@ -359,12 +359,12 @@ public class RemoteHiveSparkClient implements HiveSparkClient { } private void logConfigurations(JobConf localJobConf) { - if (LOG.isInfoEnabled()) { - LOG.info("Logging job configuration: "); + if (LOG.isDebugEnabled()) { + LOG.debug("Logging job configuration: "); StringBuilder outWriter = new StringBuilder(); // redact sensitive information before logging HiveConfUtil.dumpConfig(localJobConf, outWriter); - LOG.info(outWriter.toString()); + LOG.debug(outWriter.toString()); } } }
