Github user viirya commented on a diff in the pull request: https://github.com/apache/spark/pull/18875#discussion_r137227713 --- Diff: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala --- @@ -22,24 +22,44 @@ import java.io.Writer import com.fasterxml.jackson.core._ import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult import org.apache.spark.sql.catalyst.expressions.SpecializedGetters import org.apache.spark.sql.catalyst.util.{ArrayData, DateTimeUtils, MapData} import org.apache.spark.sql.types._ private[sql] class JacksonGenerator( - schema: StructType, + rowSchema: DataType, writer: Writer, options: JSONOptions) { + // A `ValueWriter` is responsible for writing a field of an `InternalRow` to appropriate // JSON data. Here we are using `SpecializedGetters` rather than `InternalRow` so that // we can directly access data in `ArrayData` without the help of `SpecificMutableRow`. private type ValueWriter = (SpecializedGetters, Int) => Unit + // `JackGenerator` only supports to write out a struct, an array of struct or an arbitrary map + rowSchema match { + case _: StructType | _: MapType => + TypeCheckResult.TypeCheckSuccess + case _ => TypeCheckResult.TypeCheckFailure( + s"Input type ${rowSchema.simpleString} must be a struct or a map") + } + // `ValueWriter`s for all fields of the schema - private val rootFieldWriters: Array[ValueWriter] = schema.map(_.dataType).map(makeWriter).toArray + private lazy val rootFieldWriters: Array[ValueWriter] = { + rowSchema.asInstanceOf[StructType].map(_.dataType).map(makeWriter).toArray + } + // `ValueWriter` for array data storing rows of the schema. - private val arrElementWriter: ValueWriter = (arr: SpecializedGetters, i: Int) => { - writeObject(writeFields(arr.getStruct(i, schema.length), schema, rootFieldWriters)) + private lazy val arrElementWriter: ValueWriter = { + (arr: SpecializedGetters, i: Int) => { + val schema: StructType = rowSchema.asInstanceOf[StructType] + writeObject(writeFields(arr.getStruct(i, schema.length), schema, rootFieldWriters)) + } + } + + private lazy val mapElementWriter: ValueWriter = { + makeWriter(rowSchema.asInstanceOf[MapType].valueType) --- End diff -- ditto.
--- --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org