Github user HyukjinKwon commented on a diff in the pull request:
https://github.com/apache/spark/pull/20937#discussion_r178476321
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/CreateJacksonParser.scala
---
@@ -39,11 +40,36 @@ private[sql] object CreateJacksonParser extends
Serializable {
jsonFactory.createParser(new InputStreamReader(bain, "UTF-8"))
}
- def text(jsonFactory: JsonFactory, record: Text): JsonParser = {
- jsonFactory.createParser(record.getBytes, 0, record.getLength)
+ def text(jsonFactory: JsonFactory, record: Text, encoding:
Option[String] = None): JsonParser = {
+ encoding match {
+ case Some(enc) =>
+ val bain = new ByteArrayInputStream(record.getBytes, 0,
record.getLength)
+ jsonFactory.createParser(new InputStreamReader(bain, enc))
+ case _ =>
+ jsonFactory.createParser(record.getBytes, 0, record.getLength)
+ }
}
- def inputStream(jsonFactory: JsonFactory, record: InputStream):
JsonParser = {
- jsonFactory.createParser(record)
+ def inputStream(
+ jsonFactory: JsonFactory,
+ is: InputStream,
+ encoding: Option[String] = None): JsonParser = {
+ encoding match {
+ case Some(enc) =>
+ jsonFactory.createParser(new InputStreamReader(is, enc))
+ case _ =>
+ jsonFactory.createParser(is)
+ }
+ }
+
+ def internalRow(
+ jsonFactory: JsonFactory,
+ row: InternalRow,
+ field: Int,
+ encoding: Option[String] = None
+ ): JsonParser = {
--- End diff --
Let's move this line up.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
