Repository: spark Updated Branches: refs/heads/master f6d06adf0 -> 42d225f44
[SPARK-11216][SQL][FOLLOW-UP] add encoder/decoder for external row address comments in https://github.com/apache/spark/pull/9184 Author: Wenchen Fan <wenc...@databricks.com> Closes #9212 from cloud-fan/encoder. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/42d225f4 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/42d225f4 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/42d225f4 Branch: refs/heads/master Commit: 42d225f449c633be7465493c57b9881303ee14ba Parents: f6d06ad Author: Wenchen Fan <wenc...@databricks.com> Authored: Thu Oct 22 10:53:59 2015 -0700 Committer: Michael Armbrust <mich...@databricks.com> Committed: Thu Oct 22 10:53:59 2015 -0700 ---------------------------------------------------------------------- .../spark/sql/catalyst/encoders/ClassEncoder.scala | 14 +++----------- .../spark/sql/catalyst/encoders/RowEncoder.scala | 9 ++++++--- .../spark/sql/catalyst/expressions/objects.scala | 8 +++++++- .../spark/sql/catalyst/encoders/RowEncoderSuite.scala | 2 +- 4 files changed, 17 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/42d225f4/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ClassEncoder.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ClassEncoder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ClassEncoder.scala index f3a1063..54096f1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ClassEncoder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ClassEncoder.scala @@ -48,20 +48,12 @@ case class ClassEncoder[T]( private val dataType = ObjectType(clsTag.runtimeClass) override def toRow(t: T): InternalRow = { - if (t == null) { - null - } else { - inputRow(0) = t - extractProjection(inputRow) - } + inputRow(0) = t + extractProjection(inputRow) } override def fromRow(row: InternalRow): T = { - if (row eq null) { - null.asInstanceOf[T] - } else { - constructProjection(row).get(0, dataType).asInstanceOf[T] - } + constructProjection(row).get(0, dataType).asInstanceOf[T] } override def bind(schema: Seq[Attribute]): ClassEncoder[T] = { http://git-wip-us.apache.org/repos/asf/spark/blob/42d225f4/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala index 3e74aab..5142856 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala @@ -26,8 +26,11 @@ import org.apache.spark.sql.catalyst.util.DateTimeUtils import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String +/** + * A factory for constructing encoders that convert external row to/from the Spark SQL + * internal binary representation. + */ object RowEncoder { - def apply(schema: StructType): ClassEncoder[Row] = { val cls = classOf[Row] val inputObject = BoundReference(0, ObjectType(cls), nullable = true) @@ -136,7 +139,7 @@ object RowEncoder { constructorFor(BoundReference(i, f.dataType, f.nullable), f.dataType) ) } - CreateRow(fields) + CreateExternalRow(fields) } private def constructorFor(input: Expression, dataType: DataType): Expression = dataType match { @@ -195,7 +198,7 @@ object RowEncoder { Literal.create(null, externalDataTypeFor(f.dataType)), constructorFor(getField(input, i, f.dataType), f.dataType)) } - CreateRow(convertedFields) + CreateExternalRow(convertedFields) } private def getField( http://git-wip-us.apache.org/repos/asf/spark/blob/42d225f4/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala index 8fc00ad..b42d6c5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala @@ -456,7 +456,13 @@ case class MapObjects( } } -case class CreateRow(children: Seq[Expression]) extends Expression { +/** + * Constructs a new external row, using the result of evaluating the specified expressions + * as content. + * + * @param children A list of expression to use as content of the external row. + */ +case class CreateExternalRow(children: Seq[Expression]) extends Expression { override def dataType: DataType = ObjectType(classOf[Row]) override def nullable: Boolean = false http://git-wip-us.apache.org/repos/asf/spark/blob/42d225f4/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala index 6041b62..e8301e8 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala @@ -73,7 +73,7 @@ class RowEncoderSuite extends SparkFunSuite { private def encodeDecodeTest(schema: StructType): Unit = { test(s"encode/decode: ${schema.simpleString}") { val encoder = RowEncoder(schema) - val inputGenerator = RandomDataGenerator.forType(schema).get + val inputGenerator = RandomDataGenerator.forType(schema, nullable = false).get var input: Row = null try { --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
