Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/23253#discussion_r240036498
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
---
@@ -347,17 +347,28 @@ class JacksonParser(
schema: StructType,
fieldConverters: Array[ValueConverter]): InternalRow = {
val row = new GenericInternalRow(schema.length)
+ var badRecordException: Option[Throwable] = None
+
while (nextUntil(parser, JsonToken.END_OBJECT)) {
schema.getFieldIndex(parser.getCurrentName) match {
case Some(index) =>
- row.update(index, fieldConverters(index).apply(parser))
-
+ try {
+ row.update(index, fieldConverters(index).apply(parser))
+ } catch {
+ case NonFatal(e) =>
+ badRecordException = badRecordException.orElse(Some(e))
+ parser.skipChildren()
+ }
case None =>
parser.skipChildren()
}
}
- row
+ if (badRecordException.isEmpty) {
+ row
+ } else {
+ throw BadRecordException(() => UTF8String.EMPTY_UTF8, () =>
Some(row), badRecordException.get)
--- End diff --
or we can create a new exception type and use it here, which only carries
the row and the exception
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
