[CARBONDATA-1470] csv data should not show in error log when data column length is greater than 100000 characters
Added method to handle TextParsingException and stop the leak of the sensitive data in logs This closes #1349 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/33ecca9b Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/33ecca9b Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/33ecca9b Branch: refs/heads/branch-1.2 Commit: 33ecca9b7c256285b2f10a080b1c6bae14fef5a3 Parents: 1551a7c Author: kapilreja <kapilr...@gmail.com> Authored: Tue Sep 12 06:32:36 2017 +0000 Committer: Ravindra Pesala <ravi.pes...@gmail.com> Committed: Tue Sep 12 15:34:31 2017 +0530 ---------------------------------------------------------------------- .../processing/csvload/CSVRecordReaderIterator.java | 8 ++++++++ .../processing/util/CarbonDataProcessorUtil.java | 16 ++++++++++++++++ 2 files changed, 24 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/33ecca9b/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVRecordReaderIterator.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVRecordReaderIterator.java b/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVRecordReaderIterator.java index 10a036a..efe75ef 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVRecordReaderIterator.java +++ b/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVRecordReaderIterator.java @@ -19,8 +19,12 @@ package org.apache.carbondata.processing.csvload; import java.io.IOException; + import org.apache.carbondata.common.CarbonIterator; import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException; +import org.apache.carbondata.processing.util.CarbonDataProcessorUtil; + +import com.univocity.parsers.common.TextParsingException; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.InputSplit; @@ -62,6 +66,10 @@ public class CSVRecordReaderIterator extends CarbonIterator<Object []> { } return true; } catch (Exception e) { + if (e instanceof TextParsingException) { + throw new CarbonDataLoadingException( + CarbonDataProcessorUtil.trimErrorMessage(e.getMessage())); + } throw new CarbonDataLoadingException(e); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/33ecca9b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java index 45461e3..e91cf44 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java +++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java @@ -610,4 +610,20 @@ public final class CarbonDataProcessorUtil { } return outArr; } + + /** + * This method returns String if exception is TextParsingException + * + * @param input + * @return + */ + public static String trimErrorMessage(String input) { + String errorMessage = input; + if (input != null) { + if (input.split("Hint").length > 0) { + errorMessage = input.split("Hint")[0]; + } + } + return errorMessage; + } } \ No newline at end of file
