Repository: hive Updated Branches: refs/heads/master 1e45c4537 -> 24f7d2473
HIVE-20644 : Avoid exposing sensitive infomation through a Hive Runtime exception (Ashutosh Bapat reviewed by Thejas Nair, Sankar Hariappan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/24f7d247 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/24f7d247 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/24f7d247 Branch: refs/heads/master Commit: 24f7d2473c64053247c539af41288cf0e9110917 Parents: 1e45c45 Author: Ashutosh Bapat <aba...@hortonworks.com> Authored: Fri Oct 12 07:28:21 2018 -0500 Committer: Thejas M Nair <the...@hortonworks.com> Committed: Fri Oct 12 07:28:21 2018 -0500 ---------------------------------------------------------------------- .../hadoop/hive/ql/exec/FunctionRegistry.java | 7 ++-- .../apache/hadoop/hive/ql/exec/MapOperator.java | 10 ++++-- .../hadoop/hive/ql/exec/mr/ExecReducer.java | 9 +++-- .../ql/exec/spark/SparkReduceRecordHandler.java | 38 +++++++++++++------- .../hive/ql/exec/tez/ReduceRecordSource.java | 9 +++-- 5 files changed, 53 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/24f7d247/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index 0bc8d84..b7ca7c7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -1115,8 +1115,11 @@ public final class FunctionRegistry { String detailedMsg = e instanceof java.lang.reflect.InvocationTargetException ? e.getCause().getMessage() : e.getMessage(); - throw new HiveException("Unable to execute method " + m + " with arguments " - + argumentString + ":" + detailedMsg, e); + // Log the arguments into a debug message for the ease of debugging. But when exposed through + // an error message they can leak sensitive information, even to the client application. + LOG.trace("Unable to execute method " + m + " with arguments " + + argumentString); + throw new HiveException("Unable to execute method " + m + ":" + detailedMsg, e); } return o; } http://git-wip-us.apache.org/repos/asf/hive/blob/24f7d247/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java index b9986d3..1cbc272 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java @@ -562,9 +562,15 @@ public class MapOperator extends AbstractMapOperator { } if (row == null) { deserialize_error_count.set(deserialize_error_count.get() + 1); - throw new HiveException("Hive Runtime Error while processing writable " + message, e); + LOG.trace("Hive Runtime Error while processing writable " + message); + throw new HiveException("Hive Runtime Error while processing writable", e); } - throw new HiveException("Hive Runtime Error while processing row " + message, e); + + // Log the contents of the row that caused exception so that it's available for debugging. But + // when exposed through an error message it can leak sensitive information, even to the + // client application. + LOG.trace("Hive Runtime Error while processing row " + message); + throw new HiveException("Hive Runtime Error while processing row", e); } } rowsForwarded(childrenDone, 1); http://git-wip-us.apache.org/repos/asf/hive/blob/24f7d247/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java index 829006d..e106bc9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java @@ -240,8 +240,13 @@ public class ExecReducer extends MapReduceBase implements Reducer { rowString = "[Error getting row data with exception " + StringUtils.stringifyException(e2) + " ]"; } - throw new HiveException("Hive Runtime Error while processing row (tag=" - + tag + ") " + rowString, e); + + // Log the contents of the row that caused exception so that it's available for debugging. But + // when exposed through an error message it can leak sensitive information, even to the + // client application. + LOG.trace("Hive Runtime Error while processing row (tag=" + + tag + ") " + rowString); + throw new HiveException("Hive Runtime Error while processing row", e); } } http://git-wip-us.apache.org/repos/asf/hive/blob/24f7d247/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java index 6a7e1df..20e7ea0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java @@ -346,11 +346,14 @@ public class SparkReduceRecordHandler extends SparkRecordHandler { try { keyObject = inputKeyDeserializer.deserialize(keyWritable); } catch (Exception e) { - throw new HiveException( - "Hive Runtime Error: Unable to deserialize reduce input key from " + // Log the input key which caused exception so that it's available for debugging. But when + // exposed through an error message it can leak sensitive information, even to the client + // application. + LOG.trace("Hive Runtime Error: Unable to deserialize reduce input key from " + Utilities.formatBinaryString(keyWritable.get(), 0, keyWritable.getSize()) + " with properties " - + keyTableDesc.getProperties(), e); + + keyTableDesc.getProperties()); + throw new HiveException("Hive Runtime Error: Unable to deserialize reduce input key ", e); } groupKey.set(keyWritable.get(), 0, keyWritable.getSize()); @@ -384,13 +387,16 @@ public class SparkReduceRecordHandler extends SparkRecordHandler { try { valueObject[tag] = inputValueDeserializer[tag].deserialize(valueWritable); } catch (SerDeException e) { - throw new HiveException( - "Hive Runtime Error: Unable to deserialize reduce input value (tag=" + // Log the input value which caused exception so that it's available for debugging. But when + // exposed through an error message it can leak sensitive information, even to the client + // application. + LOG.trace("Hive Runtime Error: Unable to deserialize reduce input value (tag=" + tag + ") from " + Utilities.formatBinaryString(valueWritable.get(), 0, valueWritable.getSize()) + " with properties " - + valueTableDesc[tag].getProperties(), e); + + valueTableDesc[tag].getProperties()); + throw new HiveException("Hive Runtime Error: Unable to deserialize reduce input value ", e); } row.clear(); row.add(keyObject); @@ -408,8 +414,12 @@ public class SparkReduceRecordHandler extends SparkRecordHandler { rowString = "[Error getting row data with exception " + StringUtils.stringifyException(e2) + " ]"; } - throw new HiveException("Error while processing row (tag=" - + tag + ") " + rowString, e); + + // Log contents of the row which caused exception so that it's available for debugging. But + // when exposed through an error message it can leak sensitive information, even to the + // client application. + LOG.trace("Hive exception while processing row (tag=" + tag + ") " + rowString); + throw new HiveException("Error while processing row ", e); } } return true; // give me more @@ -570,10 +580,14 @@ public class SparkReduceRecordHandler extends SparkRecordHandler { try { return inputValueDeserializer[tag].deserialize(valueWritable); } catch (SerDeException e) { - throw new HiveException("Error: Unable to deserialize reduce input value (tag=" - + tag + ") from " - + Utilities.formatBinaryString(valueWritable.getBytes(), 0, valueWritable.getLength()) - + " with properties " + valueTableDesc[tag].getProperties(), e); + // Log the input value which caused exception so that it's available for debugging. But when + // exposed through an error message it can leak sensitive information, even to the client + // application. + LOG.trace("Error: Unable to deserialize reduce input value (tag=" + tag + ") from " + + Utilities.formatBinaryString(valueWritable.getBytes(), 0, + valueWritable.getLength()) + + " with properties " + valueTableDesc[tag].getProperties()); + throw new HiveException("Error: Unable to deserialize reduce input value ", e); } } http://git-wip-us.apache.org/repos/asf/hive/blob/24f7d247/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java index 5698639..72446af 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java @@ -369,8 +369,13 @@ public class ReduceRecordSource implements RecordSource { rowString = "[Error getting row data with exception " + StringUtils.stringifyException(e2) + " ]"; } - throw new HiveException("Hive Runtime Error while processing row (tag=" - + tag + ") " + rowString, e); + + // Log the contents of the row that caused exception so that it's available for debugging. But + // when exposed through an error message it can leak sensitive information, even to the + // client application. + l4j.trace("Hive Runtime Error while processing row (tag=" + + tag + ") " + rowString); + throw new HiveException("Hive Runtime Error while processing row", e); } } }