Repository: spark Updated Branches: refs/heads/master 3dfa4ea52 -> 87f28fc24
[SPARK-11379][SQL] ExpressionEncoder can't handle top level primitive type correctly For inner primitive type(e.g. inside `Product`), we use `schemaFor` to get the catalyst type for it, https://github.com/apache/spark/blob/master/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala#L403. However, for top level primitive type, we use `dataTypeFor`, which is wrong. Author: Wenchen Fan <wenc...@databricks.com> Closes #9337 from cloud-fan/encoder. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/87f28fc2 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/87f28fc2 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/87f28fc2 Branch: refs/heads/master Commit: 87f28fc24003ad60c52f899d10f38032631624dc Parents: 3dfa4ea Author: Wenchen Fan <wenc...@databricks.com> Authored: Thu Oct 29 11:17:03 2015 +0100 Committer: Michael Armbrust <mich...@databricks.com> Committed: Thu Oct 29 11:17:03 2015 +0100 ---------------------------------------------------------------------- .../main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala | 2 +- .../spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/87f28fc2/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala index 9cbb7c2..0b8a8ab 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala @@ -170,7 +170,7 @@ trait ScalaReflection { .getOrElse(BoundReference(ordinal, dataType, false)) /** Returns the current path or throws an error. */ - def getPath = path.getOrElse(BoundReference(0, dataTypeFor(tpe), true)) + def getPath = path.getOrElse(BoundReference(0, schemaFor(tpe).dataType, true)) tpe match { case t if !dataTypeFor(t).isInstanceOf[ObjectType] => http://git-wip-us.apache.org/repos/asf/spark/blob/87f28fc2/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala index a374da4..b0dacf7 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala @@ -57,6 +57,7 @@ class ExpressionEncoderSuite extends SparkFunSuite { encodeDecodeTest(false) encodeDecodeTest(1.toShort) encodeDecodeTest(1.toByte) + encodeDecodeTest("hello") encodeDecodeTest(PrimitiveData(1, 1, 1, 1, 1, 1, true)) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
