This is an automated email from the ASF dual-hosted git repository. maxgekk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new e7520fc58e1 [SPARK-41166][SQL][TESTS] Check errorSubClass of DataTypeMismatch in *ExpressionSuites e7520fc58e1 is described below commit e7520fc58e18c45e43e07dc63f1f03cfd4da0fcc Author: panbingkun <pbk1...@gmail.com> AuthorDate: Fri Nov 18 13:30:48 2022 +0300 [SPARK-41166][SQL][TESTS] Check errorSubClass of DataTypeMismatch in *ExpressionSuites ### What changes were proposed in this pull request? The pr aims to check errorSubClass of DataTypeMismatch in `*ExpressionSuites`. ### Why are the changes needed? The changes improve the error framework. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass GA. Closes #38688 from panbingkun/SPARK-41166. Authored-by: panbingkun <pbk1...@gmail.com> Signed-off-by: Max Gekk <max.g...@gmail.com> --- .../expressions/CallMethodViaReflectionSuite.scala | 30 ++++- .../sql/catalyst/expressions/CastSuiteBase.scala | 71 ++++++++++-- .../catalyst/expressions/CastWithAnsiOnSuite.scala | 118 +++++++++++++++++++- .../expressions/CollectionExpressionsSuite.scala | 32 +++++- .../catalyst/expressions/ComplexTypeSuite.scala | 52 ++++++++- .../expressions/GeneratorExpressionSuite.scala | 36 +++++- .../expressions/JsonExpressionsSuite.scala | 14 ++- .../expressions/MiscExpressionsSuite.scala | 13 ++- .../expressions/StringExpressionsSuite.scala | 81 ++++++++++++-- .../aggregate/AggregateExpressionSuite.scala | 121 +++++++++++++++++---- .../ApproxCountDistinctForIntervalsSuite.scala | 26 ++++- .../aggregate/ApproximatePercentileSuite.scala | 23 +++- .../expressions/aggregate/PercentileSuite.scala | 62 ++++++++++- 13 files changed, 610 insertions(+), 69 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflectionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflectionSuite.scala index c8b99f6f026..e65b81ee166 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflectionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflectionSuite.scala @@ -97,10 +97,34 @@ class CallMethodViaReflectionSuite extends SparkFunSuite with ExpressionEvalHelp } test("input type checking") { - assert(CallMethodViaReflection(Seq.empty).checkInputDataTypes().isFailure) - assert(CallMethodViaReflection(Seq(Literal(staticClassName))).checkInputDataTypes().isFailure) + assert(CallMethodViaReflection(Seq.empty).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "WRONG_NUM_ARGS", + messageParameters = Map( + "functionName" -> "`reflect`", + "expectedNum" -> "> 1", + "actualNum" -> "0") + ) + ) + assert(CallMethodViaReflection(Seq(Literal(staticClassName))).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "WRONG_NUM_ARGS", + messageParameters = Map( + "functionName" -> "`reflect`", + "expectedNum" -> "> 1", + "actualNum" -> "1") + ) + ) assert(CallMethodViaReflection( - Seq(Literal(staticClassName), Literal(1))).checkInputDataTypes().isFailure) + Seq(Literal(staticClassName), Literal(1))).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "NON_FOLDABLE_INPUT", + messageParameters = Map( + "inputName" -> "method", + "inputType" -> "\"STRING\"", + "inputExpr" -> "\"1\"") + ) + ) assert(createExpr(staticClassName, "method1").checkInputDataTypes().isSuccess) } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala index a60491b0ab8..6d972a8482a 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala @@ -29,6 +29,7 @@ import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch import org.apache.spark.sql.catalyst.analysis.TypeCoercion.numericPrecedence +import org.apache.spark.sql.catalyst.expressions.Cast._ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext import org.apache.spark.sql.catalyst.util.DateTimeConstants._ import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._ @@ -542,18 +543,74 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper { } protected def checkInvalidCastFromNumericType(to: DataType): Unit = { - assert(cast(1.toByte, to).checkInputDataTypes().isFailure) - assert(cast(1.toShort, to).checkInputDataTypes().isFailure) - assert(cast(1, to).checkInputDataTypes().isFailure) - assert(cast(1L, to).checkInputDataTypes().isFailure) - assert(cast(1.0.toFloat, to).checkInputDataTypes().isFailure) - assert(cast(1.0, to).checkInputDataTypes().isFailure) + cast(1.toByte, to).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "CAST_WITH_FUN_SUGGESTION", + messageParameters = Map( + "srcType" -> toSQLType(Literal(1.toByte).dataType), + "targetType" -> toSQLType(to), + "functionNames" -> "`DATE_FROM_UNIX_DATE`" + ) + ) + cast(1.toShort, to).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "CAST_WITH_FUN_SUGGESTION", + messageParameters = Map( + "srcType" -> toSQLType(Literal(1.toShort).dataType), + "targetType" -> toSQLType(to), + "functionNames" -> "`DATE_FROM_UNIX_DATE`" + ) + ) + cast(1, to).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "CAST_WITH_FUN_SUGGESTION", + messageParameters = Map( + "srcType" -> toSQLType(Literal(1).dataType), + "targetType" -> toSQLType(to), + "functionNames" -> "`DATE_FROM_UNIX_DATE`" + ) + ) + cast(1L, to).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "CAST_WITH_FUN_SUGGESTION", + messageParameters = Map( + "srcType" -> toSQLType(Literal(1L).dataType), + "targetType" -> toSQLType(to), + "functionNames" -> "`DATE_FROM_UNIX_DATE`" + ) + ) + cast(1.0.toFloat, to).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "CAST_WITH_FUN_SUGGESTION", + messageParameters = Map( + "srcType" -> toSQLType(Literal(1.0.toFloat).dataType), + "targetType" -> toSQLType(to), + "functionNames" -> "`DATE_FROM_UNIX_DATE`" + ) + ) + cast(1.0, to).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "CAST_WITH_FUN_SUGGESTION", + messageParameters = Map( + "srcType" -> toSQLType(Literal(1.0).dataType), + "targetType" -> toSQLType(to), + "functionNames" -> "`DATE_FROM_UNIX_DATE`" + ) + ) } test("SPARK-16729 type checking for casting to date type") { assert(cast("1234", DateType).checkInputDataTypes().isSuccess) assert(cast(new Timestamp(1), DateType).checkInputDataTypes().isSuccess) - assert(cast(false, DateType).checkInputDataTypes().isFailure) + assert(cast(false, DateType).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "CAST_WITHOUT_SUGGESTION", + messageParameters = Map( + "srcType" -> "\"BOOLEAN\"", + "targetType" -> "\"DATE\"" + ) + ) + ) checkInvalidCastFromNumericType(DateType) } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastWithAnsiOnSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastWithAnsiOnSuite.scala index fe742a9a4c6..fe5151fcfff 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastWithAnsiOnSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastWithAnsiOnSuite.scala @@ -170,19 +170,127 @@ class CastWithAnsiOnSuite extends CastSuiteBase with QueryErrorsBase { checkInvalidCastFromNumericType(BinaryType) val binaryLiteral = Literal(new Array[Byte](1.toByte), BinaryType) numericTypes.foreach { numericType => - assert(cast(binaryLiteral, numericType).checkInputDataTypes().isFailure) + assert(cast(binaryLiteral, numericType).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "CAST_WITHOUT_SUGGESTION", + messageParameters = Map( + "srcType" -> "\"BINARY\"", + "targetType" -> toSQLType(numericType) + ) + ) + ) } } test("ANSI mode: disallow type conversions between Datatime types and Boolean types") { val timestampLiteral = Literal(1L, TimestampType) - assert(cast(timestampLiteral, BooleanType).checkInputDataTypes().isFailure) + val checkResult1 = cast(timestampLiteral, BooleanType).checkInputDataTypes() + evalMode match { + case EvalMode.ANSI => + assert(checkResult1 == + DataTypeMismatch( + errorSubClass = "CAST_WITH_CONF_SUGGESTION", + messageParameters = Map( + "srcType" -> "\"TIMESTAMP\"", + "targetType" -> "\"BOOLEAN\"", + "config" -> "\"spark.sql.ansi.enabled\"", + "configVal" -> "'false'" + ) + ) + ) + case EvalMode.TRY => + assert(checkResult1 == + DataTypeMismatch( + errorSubClass = "CAST_WITHOUT_SUGGESTION", + messageParameters = Map( + "srcType" -> "\"TIMESTAMP\"", + "targetType" -> "\"BOOLEAN\"" + ) + ) + ) + case _ => + } + val dateLiteral = Literal(1, DateType) - assert(cast(dateLiteral, BooleanType).checkInputDataTypes().isFailure) + val checkResult2 = cast(dateLiteral, BooleanType).checkInputDataTypes() + evalMode match { + case EvalMode.ANSI => + assert(checkResult2 == + DataTypeMismatch( + errorSubClass = "CAST_WITH_CONF_SUGGESTION", + messageParameters = Map( + "srcType" -> "\"DATE\"", + "targetType" -> "\"BOOLEAN\"", + "config" -> "\"spark.sql.ansi.enabled\"", + "configVal" -> "'false'" + ) + ) + ) + case EvalMode.TRY => + assert(checkResult2 == + DataTypeMismatch( + errorSubClass = "CAST_WITHOUT_SUGGESTION", + messageParameters = Map( + "srcType" -> "\"DATE\"", + "targetType" -> "\"BOOLEAN\"" + ) + ) + ) + case _ => + } val booleanLiteral = Literal(true, BooleanType) - assert(cast(booleanLiteral, TimestampType).checkInputDataTypes().isFailure) - assert(cast(booleanLiteral, DateType).checkInputDataTypes().isFailure) + val checkResult3 = cast(booleanLiteral, TimestampType).checkInputDataTypes() + evalMode match { + case EvalMode.ANSI => + assert(checkResult3 == + DataTypeMismatch( + errorSubClass = "CAST_WITH_CONF_SUGGESTION", + messageParameters = Map( + "srcType" -> "\"BOOLEAN\"", + "targetType" -> "\"TIMESTAMP\"", + "config" -> "\"spark.sql.ansi.enabled\"", + "configVal" -> "'false'" + ) + ) + ) + case EvalMode.TRY => + assert(checkResult3 == + DataTypeMismatch( + errorSubClass = "CAST_WITHOUT_SUGGESTION", + messageParameters = Map( + "srcType" -> "\"BOOLEAN\"", + "targetType" -> "\"TIMESTAMP\"" + ) + ) + ) + case _ => + } + + val checkResult4 = cast(booleanLiteral, DateType).checkInputDataTypes() + evalMode match { + case EvalMode.ANSI => + assert(checkResult4 == + DataTypeMismatch( + errorSubClass = "CAST_WITHOUT_SUGGESTION", + messageParameters = Map( + "srcType" -> "\"BOOLEAN\"", + "targetType" -> "\"DATE\"" + ) + ) + ) + case EvalMode.TRY => + assert(checkResult4 == + DataTypeMismatch( + errorSubClass = "CAST_WITHOUT_SUGGESTION", + messageParameters = Map( + "srcType" -> "\"BOOLEAN\"", + "targetType" -> "\"DATE\"" + ) + ) + ) + case _ => + } } private def castErrMsg(v: Any, to: DataType, from: DataType = StringType): String = { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala index 807927dba88..55456e309b6 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala @@ -232,8 +232,24 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper // argument checking assert(MapConcat(Seq(m0, m1)).checkInputDataTypes().isSuccess) assert(MapConcat(Seq(m5, m6)).checkInputDataTypes().isSuccess) - assert(MapConcat(Seq(m0, m5)).checkInputDataTypes().isFailure) - assert(MapConcat(Seq(m0, Literal(12))).checkInputDataTypes().isFailure) + assert(MapConcat(Seq(m0, m5)).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "DATA_DIFF_TYPES", + messageParameters = Map( + "functionName" -> "`map_concat`", + "dataType" -> "(\"MAP<STRING, STRING>\" or \"MAP<STRING, INT>\")" + ) + ) + ) + assert(MapConcat(Seq(m0, Literal(12))).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "MAP_CONCAT_DIFF_TYPES", + messageParameters = Map( + "functionName" -> "`map_concat`", + "dataType" -> "[\"MAP<STRING, STRING>\", \"INT\"]" + ) + ) + ) assert(MapConcat(Seq(m0, m1)).dataType.keyType == StringType) assert(MapConcat(Seq(m0, m1)).dataType.valueType == StringType) assert(!MapConcat(Seq(m0, m1)).dataType.valueContainsNull) @@ -1538,7 +1554,17 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper val m1 = Literal.create(Map[String, String](), MapType(StringType, StringType)) val m2 = Literal.create(null, MapType(StringType, StringType)) - assert(ElementAt(m0, Literal(1.0)).checkInputDataTypes().isFailure) + assert(ElementAt(m0, Literal(1.0)).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "MAP_FUNCTION_DIFF_TYPES", + messageParameters = Map( + "functionName" -> "`element_at`", + "dataType" -> "\"MAP\"", + "leftType" -> "\"MAP<STRING, STRING>\"", + "rightType" -> "\"DOUBLE\"" + ) + ) + ) withSQLConf(SQLConf.ANSI_ENABLED.key -> false.toString) { checkEvaluation(ElementAt(m0, Literal("d")), null) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala index f1f781b7137..b9f6ca1b191 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala @@ -527,7 +527,7 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper { val s6 = Literal("a=1&b=2&c=3") val m6 = Map("a" -> "1", "b" -> "2", "c" -> "3") - checkEvaluation(new StringToMap(s6, NonFoldableLiteral("&"), NonFoldableLiteral("=")), m6) + checkEvaluation(StringToMap(s6, NonFoldableLiteral("&"), NonFoldableLiteral("=")), m6) checkExceptionInExpression[RuntimeException]( new StringToMap(Literal("a:1,b:2,a:3")), "Duplicate map key") @@ -540,11 +540,51 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper { // arguments checking assert(new StringToMap(Literal("a:1,b:2,c:3")).checkInputDataTypes().isSuccess) - assert(new StringToMap(Literal(null)).checkInputDataTypes().isFailure) - assert(new StringToMap(Literal("a:1,b:2,c:3"), Literal(null)).checkInputDataTypes().isFailure) - assert(StringToMap(Literal("a:1,b:2,c:3"), Literal(null), Literal(null)) - .checkInputDataTypes().isFailure) - assert(new StringToMap(Literal(null), Literal(null)).checkInputDataTypes().isFailure) + assert(new StringToMap(Literal(null)).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "1", + "requiredType" -> "\"STRING\"", + "inputSql" -> "\"NULL\"", + "inputType" -> "\"VOID\"" + ) + ) + ) + assert(new StringToMap(Literal("a:1,b:2,c:3"), Literal(null)).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "2", + "requiredType" -> "\"STRING\"", + "inputSql" -> "\"NULL\"", + "inputType" -> "\"VOID\"" + ) + ) + ) + assert(StringToMap(Literal("a:1,b:2,c:3"), Literal(null), + Literal(null)).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "2", + "requiredType" -> "\"STRING\"", + "inputSql" -> "\"NULL\"", + "inputType" -> "\"VOID\"" + ) + ) + ) + assert(new StringToMap(Literal(null), Literal(null)).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "1", + "requiredType" -> "\"STRING\"", + "inputSql" -> "\"NULL\"", + "inputType" -> "\"VOID\"" + ) + ) + ) } test("SPARK-22693: CreateNamedStruct should not use global variables") { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/GeneratorExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/GeneratorExpressionSuite.scala index e29dfa41f1c..4b134dffe97 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/GeneratorExpressionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/GeneratorExpressionSuite.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.catalyst.expressions import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch import org.apache.spark.sql.types._ class GeneratorExpressionSuite extends SparkFunSuite with ExpressionEvalHelper { @@ -76,9 +77,38 @@ class GeneratorExpressionSuite extends SparkFunSuite with ExpressionEvalHelper { Stack(Seq(3, 1, 1.0, "a", 2, 2.0, "b", 3, 3.0, "c").map(Literal(_))), Seq(create_row(1, 1.0, "a"), create_row(2, 2.0, "b"), create_row(3, 3.0, "c"))) - assert(Stack(Seq(Literal(1))).checkInputDataTypes().isFailure) - assert(Stack(Seq(Literal(1.0))).checkInputDataTypes().isFailure) + assert(Stack(Seq(Literal(1))).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "WRONG_NUM_ARGS", + messageParameters = Map( + "functionName" -> "`stack`", + "expectedNum" -> "> 1", + "actualNum" -> "1" + ) + ) + ) + assert(Stack(Seq(Literal(1.0))).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "WRONG_NUM_ARGS", + messageParameters = Map( + "functionName" -> "`stack`", + "expectedNum" -> "> 1", + "actualNum" -> "1" + ) + ) + ) assert(Stack(Seq(Literal(1), Literal(1), Literal(1.0))).checkInputDataTypes().isSuccess) - assert(Stack(Seq(Literal(2), Literal(1), Literal(1.0))).checkInputDataTypes().isFailure) + assert(Stack(Seq(Literal(2), Literal(1), Literal(1.0))).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "STACK_COLUMN_DIFF_TYPES", + messageParameters = Map( + "rightParamIndex" -> "2", + "leftType" -> "\"INT\"", + "leftParamIndex" -> "1", + "columnIndex" -> "0", + "rightType" -> "\"DOUBLE\"" + ) + ) + ) } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala index 6aebc6b31d1..beeb01619aa 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala @@ -26,6 +26,7 @@ import org.apache.spark.{SparkException, SparkFunSuite} import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch +import org.apache.spark.sql.catalyst.expressions.Cast._ import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection import org.apache.spark.sql.catalyst.plans.PlanTestBase import org.apache.spark.sql.catalyst.util._ @@ -884,9 +885,16 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with """"test": {"1": "test"}"""), (ArrayType(MapType(IntegerType, StringType)), """[{"1": "test"}]"""), (MapType(StringType, MapType(IntegerType, StringType)), """{"key": {"1" : "test"}}""") - ).foreach{ + ).foreach { case(schema, jsonData) => - assert(JsonToStructs(schema, Map.empty, Literal(jsonData)).checkInputDataTypes().isFailure) - } + assert(JsonToStructs(schema, Map.empty, Literal(jsonData)).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "INVALID_JSON_MAP_KEY_TYPE", + messageParameters = Map( + "schema" -> toSQLType(schema) + ) + ) + ) + } } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala index d42081024c1..15a0695943b 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala @@ -22,6 +22,7 @@ import java.io.PrintStream import scala.util.Random import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch import org.apache.spark.sql.types._ class MiscExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { @@ -40,7 +41,17 @@ class MiscExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { ) // Expects a string - assert(RaiseError(Literal(5)).checkInputDataTypes().isFailure) + assert(RaiseError(Literal(5)).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "1", + "requiredType" -> "\"STRING\"", + "inputSql" -> "\"5\"", + "inputType" -> "\"INT\"" + ) + ) + ) } test("uuid") { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala index 6a95270adcf..8bdbcb26e83 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala @@ -64,10 +64,27 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { .checkInputDataTypes().isSuccess) assert(Concat(Literal.create("a".getBytes) :: Literal.create("b".getBytes) :: Nil) .checkInputDataTypes().isSuccess) - assert(Concat(Literal.create(1) :: Literal.create(2) :: Nil) - .checkInputDataTypes().isFailure) + assert(Concat(Literal.create(1) :: Literal.create(2) :: Nil).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "1", + "requiredType" -> "(\"STRING\" or \"BINARY\" or \"ARRAY\")", + "inputSql" -> "\"1\"", + "inputType" -> "\"INT\"" + ) + ) + ) assert(Concat(Literal.create("a") :: Literal.create("b".getBytes) :: Nil) - .checkInputDataTypes().isFailure) + .checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "DATA_DIFF_TYPES", + messageParameters = Map( + "functionName" -> "`concat`", + "dataType" -> "(\"STRING\" or \"BINARY\")" + ) + ) + ) } test("concat_ws") { @@ -132,10 +149,38 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { } // type checking - assert(Elt(Seq.empty).checkInputDataTypes().isFailure) - assert(Elt(Seq(Literal(1))).checkInputDataTypes().isFailure) + assert(Elt(Seq.empty).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "WRONG_NUM_ARGS", + messageParameters = Map( + "functionName" -> "`elt`", + "expectedNum" -> "> 1", + "actualNum" -> "0" + ) + ) + ) + assert(Elt(Seq(Literal(1))).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "WRONG_NUM_ARGS", + messageParameters = Map( + "functionName" -> "`elt`", + "expectedNum" -> "> 1", + "actualNum" -> "1" + ) + ) + ) assert(Elt(Seq(Literal(1), Literal("A"))).checkInputDataTypes().isSuccess) - assert(Elt(Seq(Literal(1), Literal(2))).checkInputDataTypes().isFailure) + assert(Elt(Seq(Literal(1), Literal(2))).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "2...", + "requiredType" -> "\"STRING\" or \"BINARY\"", + "inputSql" -> "\"2\"", + "inputType" -> "\"INT\"" + ) + ) + ) } test("SPARK-22550: Elt should not generate codes beyond 64KB") { @@ -553,9 +598,29 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { Literal.create(7, IntegerType), Literal.create(0, IntegerType)) .checkInputDataTypes().isSuccess) assert(new Overlay(Literal.create(1), Literal.create(2), Literal.create(0, IntegerType)) - .checkInputDataTypes().isFailure) + .checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "1", + "requiredType" -> "(\"STRING\" or \"BINARY\")", + "inputSql" -> "\"1\"", + "inputType" -> "\"INT\"" + ) + ) + ) assert(Overlay(Literal("Spark SQL"), Literal.create(2), Literal.create(7, IntegerType), - Literal.create(0, IntegerType)).checkInputDataTypes().isFailure) + Literal.create(0, IntegerType)).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "2", + "requiredType" -> "(\"STRING\" or \"BINARY\")", + "inputSql" -> "\"2\"", + "inputType" -> "\"INT\"" + ) + ) + ) } test("translate") { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/AggregateExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/AggregateExpressionSuite.scala index 813ab746486..91b580282ec 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/AggregateExpressionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/AggregateExpressionSuite.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.catalyst.expressions.aggregate import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, UnresolvedAttribute} +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch import org.apache.spark.sql.catalyst.expressions.{Add, AttributeSet, Literal} class AggregateExpressionSuite extends SparkFunSuite { @@ -32,37 +33,115 @@ class AggregateExpressionSuite extends SparkFunSuite { } test("test regr_r2 input types") { - Seq( - RegrR2(Literal("a"), Literal(1d)), - RegrR2(Literal(3.0D), Literal('b')), - RegrR2(Literal(3.0D), Literal(Array(0))) - ).foreach { expr => - assert(expr.checkInputDataTypes().isFailure) - } + assert(RegrR2(Literal("a"), Literal(1d)).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "1", + "requiredType" -> "\"DOUBLE\"", + "inputSql" -> "\"a\"", + "inputType" -> "\"STRING\"" + ) + ) + ) + assert(RegrR2(Literal(3.0D), Literal('b')).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "2", + "requiredType" -> "\"DOUBLE\"", + "inputSql" -> "\"b\"", + "inputType" -> "\"STRING\"" + ) + ) + ) + assert(RegrR2(Literal(3.0D), Literal(Array(0))).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "2", + "requiredType" -> "\"DOUBLE\"", + "inputSql" -> "\"ARRAY(0)\"", + "inputType" -> "\"ARRAY<INT>\"" + ) + ) + ) assert(RegrR2(Literal(3.0D), Literal(1d)).checkInputDataTypes() === TypeCheckResult.TypeCheckSuccess) } test("test regr_slope input types") { - Seq( - RegrSlope(Literal("a"), Literal(1)), - RegrSlope(Literal(3.0D), Literal('b')), - RegrSlope(Literal(3.0D), Literal(Array(0))) - ).foreach { expr => - assert(expr.checkInputDataTypes().isFailure) - } + assert(RegrSlope(Literal("a"), Literal(1)).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "1", + "requiredType" -> "\"DOUBLE\"", + "inputSql" -> "\"a\"", + "inputType" -> "\"STRING\"" + ) + ) + ) + assert(RegrSlope(Literal(3.0D), Literal('b')).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "2", + "requiredType" -> "\"DOUBLE\"", + "inputSql" -> "\"b\"", + "inputType" -> "\"STRING\"" + ) + ) + ) + assert(RegrSlope(Literal(3.0D), Literal(Array(0))).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "2", + "requiredType" -> "\"DOUBLE\"", + "inputSql" -> "\"ARRAY(0)\"", + "inputType" -> "\"ARRAY<INT>\"" + ) + ) + ) assert(RegrSlope(Literal(3.0D), Literal(1D)).checkInputDataTypes() === TypeCheckResult.TypeCheckSuccess) } test("test regr_intercept input types") { - Seq( - RegrIntercept(Literal("a"), Literal(1)), - RegrIntercept(Literal(3.0D), Literal('b')), - RegrIntercept(Literal(3.0D), Literal(Array(0))) - ).foreach { expr => - assert(expr.checkInputDataTypes().isFailure) - } + assert(RegrIntercept(Literal("a"), Literal(1)).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "1", + "requiredType" -> "\"DOUBLE\"", + "inputSql" -> "\"a\"", + "inputType" -> "\"STRING\"" + ) + ) + ) + assert(RegrIntercept(Literal(3.0D), Literal('b')).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "2", + "requiredType" -> "\"DOUBLE\"", + "inputSql" -> "\"b\"", + "inputType" -> "\"STRING\"" + ) + ) + ) + assert(RegrIntercept(Literal(3.0D), Literal(Array(0))).checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "2", + "requiredType" -> "\"DOUBLE\"", + "inputSql" -> "\"ARRAY(0)\"", + "inputType" -> "\"ARRAY<INT>\"" + ) + ) + ) assert(RegrIntercept(Literal(3.0D), Literal(1D)).checkInputDataTypes() === TypeCheckResult.TypeCheckSuccess) } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxCountDistinctForIntervalsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxCountDistinctForIntervalsSuite.scala index bb99e1c1e8e..70c1be47c42 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxCountDistinctForIntervalsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxCountDistinctForIntervalsSuite.scala @@ -24,6 +24,7 @@ import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BoundReference, CreateArray, Literal, SpecificInternalRow} +import org.apache.spark.sql.catalyst.expressions.Cast._ import org.apache.spark.sql.catalyst.util.{ArrayData, DateTimeUtils} import org.apache.spark.sql.types._ @@ -36,13 +37,34 @@ class ApproxCountDistinctForIntervalsSuite extends SparkFunSuite { val wrongColumn = ApproxCountDistinctForIntervals( AttributeReference("a", dataType)(), endpointsExpression = CreateArray(Seq(1, 10).map(Literal(_)))) - assert(wrongColumn.checkInputDataTypes().isFailure) + assert(wrongColumn.checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "1", + "requiredType" -> ("(\"NUMERIC\" or \"TIMESTAMP\" or \"DATE\" or \"TIMESTAMP_NTZ\"" + + " or \"INTERVAL YEAR TO MONTH\" or \"INTERVAL DAY TO SECOND\")"), + "inputSql" -> "\"a\"", + "inputType" -> toSQLType(dataType) + ) + ) + ) } var wrongEndpoints = ApproxCountDistinctForIntervals( AttributeReference("a", DoubleType)(), endpointsExpression = Literal(0.5d)) - assert(wrongEndpoints.checkInputDataTypes().isFailure) + assert(wrongEndpoints.checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "2", + "requiredType" -> "\"ARRAY\"", + "inputSql" -> "\"0.5\"", + "inputType" -> "\"DOUBLE\"" + ) + ) + ) wrongEndpoints = ApproxCountDistinctForIntervals( AttributeReference("a", DoubleType)(), diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentileSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentileSuite.scala index 533f60da9b4..2aa6104eb96 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentileSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentileSuite.scala @@ -375,7 +375,16 @@ class ApproximatePercentileSuite extends SparkFunSuite { AttributeReference("a", DoubleType)(), percentageExpression = percentageExpression, accuracyExpression = Literal(100)) - assert(wrongPercentage.checkInputDataTypes().isFailure) + assert(wrongPercentage.checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "2", + "requiredType" -> "(\"DOUBLE\" or \"ARRAY<DOUBLE>\")", + "inputSql" -> toSQLExpr(percentageExpression), + "inputType" -> "\"ARRAY<VOID>\"") + ) + ) } } @@ -386,7 +395,17 @@ class ApproximatePercentileSuite extends SparkFunSuite { AttributeReference("a", DoubleType)(), percentageExpression = Literal(0.5), accuracyExpression = Literal(acc)) - assert(wrongPercentage.checkInputDataTypes().isFailure) + assert(wrongPercentage.checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "3", + "requiredType" -> "\"INTEGRAL\"", + "inputSql" -> toSQLExpr(Literal(acc)), + "inputType" -> toSQLType(Literal(acc).dataType) + ) + ) + ) } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PercentileSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PercentileSuite.scala index 622f7ad548e..4efc65cf7fa 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PercentileSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PercentileSuite.scala @@ -170,7 +170,18 @@ class PercentileSuite extends SparkFunSuite { invalidDataTypes.foreach { dataType => val child = AttributeReference("a", dataType)() val percentile = new Percentile(child, percentage) - assert(percentile.checkInputDataTypes().isFailure) + assert(percentile.checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "1", + "requiredType" -> ("(\"NUMERIC\" or \"INTERVAL DAY TO SECOND\" " + + "or \"INTERVAL YEAR TO MONTH\")"), + "inputSql" -> "\"a\"", + "inputType" -> toSQLType(dataType) + ) + ) + ) } val invalidFrequencyDataTypes = Seq(FloatType, DoubleType, BooleanType, @@ -182,7 +193,18 @@ class PercentileSuite extends SparkFunSuite { val child = AttributeReference("a", dataType)() val frq = AttributeReference("frq", frequencyType)() val percentile = new Percentile(child, percentage, frq) - assert(percentile.checkInputDataTypes().isFailure) + assert(percentile.checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "1", + "requiredType" -> ("(\"NUMERIC\" or \"INTERVAL DAY TO SECOND\" " + + "or \"INTERVAL YEAR TO MONTH\")"), + "inputSql" -> "\"a\"", + "inputType" -> toSQLType(dataType) + ) + ) + ) } for(dataType <- validDataTypes; @@ -190,7 +212,17 @@ class PercentileSuite extends SparkFunSuite { val child = AttributeReference("a", dataType)() val frq = AttributeReference("frq", frequencyType)() val percentile = new Percentile(child, percentage, frq) - assert(percentile.checkInputDataTypes().isFailure) + assert(percentile.checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "3", + "requiredType" -> "\"INTEGRAL\"", + "inputSql" -> "\"frq\"", + "inputType" -> toSQLType(frequencyType) + ) + ) + ) } } @@ -260,7 +292,17 @@ class PercentileSuite extends SparkFunSuite { val percentage = Literal.default(dataType) val percentile4 = new Percentile(child, percentage) val checkResult = percentile4.checkInputDataTypes() - assert(checkResult.isFailure) + assert(checkResult == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "2", + "requiredType" -> "\"DOUBLE\"", + "inputSql" -> toSQLExpr(percentage), + "inputType" -> toSQLType(dataType) + ) + ) + ) } } @@ -300,7 +342,17 @@ class PercentileSuite extends SparkFunSuite { val wrongPercentage = new Percentile( AttributeReference("a", DoubleType)(), percentageExpression = percentageExpression) - assert(wrongPercentage.checkInputDataTypes().isFailure) + assert(wrongPercentage.checkInputDataTypes() == + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> "2", + "requiredType" -> "\"ARRAY<DOUBLE>\"", + "inputSql" -> toSQLExpr(percentageExpression), + "inputType" -> "\"ARRAY<VOID>\"" + ) + ) + ) } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org