Repository: spark Updated Branches: refs/heads/master a9928277d -> b270bccff
[SPARK-25096][SQL] Loosen nullability if the cast is force-nullable. ## What changes were proposed in this pull request? In type coercion for complex types, if the found type is force-nullable to cast, we should loosen the nullability to be able to cast. Also for map key type, we can't use the type. ## How was this patch tested? Added some test. Closes #22086 from ueshin/issues/SPARK-25096/fix_type_coercion. Authored-by: Takuya UESHIN <ues...@databricks.com> Signed-off-by: hyukjinkwon <gurwls...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b270bccf Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b270bccf Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b270bccf Branch: refs/heads/master Commit: b270bccffffb233331b814e77ae55c1b74bc25d7 Parents: a992827 Author: Takuya UESHIN <ues...@databricks.com> Authored: Mon Aug 13 19:27:17 2018 +0800 Committer: hyukjinkwon <gurwls...@apache.org> Committed: Mon Aug 13 19:27:17 2018 +0800 ---------------------------------------------------------------------- .../sql/catalyst/analysis/TypeCoercion.scala | 21 +++++++++++++------- .../catalyst/analysis/TypeCoercionSuite.scala | 16 +++++++++++++++ 2 files changed, 30 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/b270bccf/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala index 27839d7..10d9ee5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala @@ -153,19 +153,26 @@ object TypeCoercion { t2: DataType, findTypeFunc: (DataType, DataType) => Option[DataType]): Option[DataType] = (t1, t2) match { case (ArrayType(et1, containsNull1), ArrayType(et2, containsNull2)) => - findTypeFunc(et1, et2).map(ArrayType(_, containsNull1 || containsNull2)) + findTypeFunc(et1, et2).map { et => + ArrayType(et, containsNull1 || containsNull2 || + Cast.forceNullable(et1, et) || Cast.forceNullable(et2, et)) + } case (MapType(kt1, vt1, valueContainsNull1), MapType(kt2, vt2, valueContainsNull2)) => - findTypeFunc(kt1, kt2).flatMap { kt => - findTypeFunc(vt1, vt2).map { vt => - MapType(kt, vt, valueContainsNull1 || valueContainsNull2) - } + findTypeFunc(kt1, kt2) + .filter { kt => !Cast.forceNullable(kt1, kt) && !Cast.forceNullable(kt2, kt) } + .flatMap { kt => + findTypeFunc(vt1, vt2).map { vt => + MapType(kt, vt, valueContainsNull1 || valueContainsNull2 || + Cast.forceNullable(vt1, vt) || Cast.forceNullable(vt2, vt)) + } } case (StructType(fields1), StructType(fields2)) if fields1.length == fields2.length => val resolver = SQLConf.get.resolver fields1.zip(fields2).foldLeft(Option(new StructType())) { case (Some(struct), (field1, field2)) if resolver(field1.name, field2.name) => - findTypeFunc(field1.dataType, field2.dataType).map { - dt => struct.add(field1.name, dt, field1.nullable || field2.nullable) + findTypeFunc(field1.dataType, field2.dataType).map { dt => + struct.add(field1.name, dt, field1.nullable || field2.nullable || + Cast.forceNullable(field1.dataType, dt) || Cast.forceNullable(field2.dataType, dt)) } case _ => None } http://git-wip-us.apache.org/repos/asf/spark/blob/b270bccf/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala index d71bbb3..2c6cb3a 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala @@ -499,6 +499,10 @@ class TypeCoercionSuite extends AnalysisTest { ArrayType(new StructType().add("num", ShortType), containsNull = false), ArrayType(new StructType().add("num", LongType), containsNull = false), Some(ArrayType(new StructType().add("num", LongType), containsNull = false))) + widenTestWithStringPromotion( + ArrayType(IntegerType, containsNull = false), + ArrayType(DecimalType.IntDecimal, containsNull = false), + Some(ArrayType(DecimalType.IntDecimal, containsNull = true))) // MapType widenTestWithStringPromotion( @@ -517,6 +521,14 @@ class TypeCoercionSuite extends AnalysisTest { MapType(IntegerType, new StructType().add("num", ShortType), valueContainsNull = false), MapType(LongType, new StructType().add("num", LongType), valueContainsNull = false), Some(MapType(LongType, new StructType().add("num", LongType), valueContainsNull = false))) + widenTestWithStringPromotion( + MapType(StringType, IntegerType, valueContainsNull = false), + MapType(StringType, DecimalType.IntDecimal, valueContainsNull = false), + Some(MapType(StringType, DecimalType.IntDecimal, valueContainsNull = true))) + widenTestWithStringPromotion( + MapType(IntegerType, StringType, valueContainsNull = false), + MapType(DecimalType.IntDecimal, StringType, valueContainsNull = false), + None) // StructType widenTestWithStringPromotion( @@ -540,6 +552,10 @@ class TypeCoercionSuite extends AnalysisTest { .add("map", MapType(DoubleType, StringType, valueContainsNull = false), nullable = false), Some(new StructType() .add("map", MapType(DoubleType, StringType, valueContainsNull = true), nullable = false))) + widenTestWithStringPromotion( + new StructType().add("num", IntegerType, nullable = false), + new StructType().add("num", DecimalType.IntDecimal, nullable = false), + Some(new StructType().add("num", DecimalType.IntDecimal, nullable = true))) widenTestWithStringPromotion( new StructType().add("num", IntegerType), --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
