Github user ueshin commented on a diff in the pull request:
https://github.com/apache/spark/pull/22646#discussion_r225806457
--- Diff: sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala ---
@@ -1115,9 +1126,38 @@ object SQLContext {
})
}
}
- def createConverter(cls: Class[_], dataType: DataType): Any => Any =
dataType match {
- case struct: StructType => createStructConverter(cls,
struct.map(_.dataType))
- case _ => CatalystTypeConverters.createToCatalystConverter(dataType)
+ def createConverter(t: Type, dataType: DataType): Any => Any = (t,
dataType) match {
+ case (cls: Class[_], struct: StructType) =>
+ // bean type
+ createStructConverter(cls, struct.map(_.dataType))
+ case (arrayType: Class[_], array: ArrayType) if arrayType.isArray =>
+ // array type
+ val converter = createConverter(arrayType.getComponentType,
array.elementType)
+ value => new GenericArrayData(
+ (0 until JavaArray.getLength(value)).map(i =>
+ converter(JavaArray.get(value, i))).toArray)
+ case (_, array: ArrayType) =>
+ // java.util.List type
+ val cls = classOf[java.util.List[_]]
--- End diff --
On second thoughts, we should use `java.lang.Iterable` here. We can convert
`Iterable` to `ArrayType` as `ArrayConverter` is trying. If we use
`java.util.List` here, it leads behavior changes for list of primitives.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
