This is an automated email from the ASF dual-hosted git repository. maxgekk pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.3 by this push: new b39ed56193e [SPARK-38687][SQL][3.3] Use error classes in the compilation errors of generators b39ed56193e is described below commit b39ed56193ea0ade80960cd920536426d85680f1 Author: panbingkun <pbk1...@gmail.com> AuthorDate: Wed Jun 22 18:52:14 2022 +0300 [SPARK-38687][SQL][3.3] Use error classes in the compilation errors of generators ## What changes were proposed in this pull request? Migrate the following errors in QueryCompilationErrors onto use error classes: - nestedGeneratorError => UNSUPPORTED_GENERATOR.NESTED_IN_EXPRESSIONS - moreThanOneGeneratorError => UNSUPPORTED_GENERATOR.MULTI_GENERATOR - generatorOutsideSelectError => UNSUPPORTED_GENERATOR.OUTSIDE_SELECT - generatorNotExpectedError => UNSUPPORTED_GENERATOR.NOT_GENERATOR This is a backport of https://github.com/apache/spark/pull/36617. ### Why are the changes needed? Porting compilation errors of generator to new error framework, improve test coverage, and document expected error messages in tests. ### Does this PR introduce any user-facing change? No ### How was this patch tested? By running new test: ``` $ build/sbt "sql/testOnly *QueryCompilationErrorsSuite*" ``` Closes #36956 from panbingkun/branch-3.3-SPARK-38687. Authored-by: panbingkun <pbk1...@gmail.com> Signed-off-by: Max Gekk <max.g...@gmail.com> --- core/src/main/resources/error/error-classes.json | 17 ++++++++ .../spark/sql/errors/QueryCompilationErrors.scala | 22 +++++------ .../sql/catalyst/analysis/AnalysisErrorSuite.scala | 23 +++++------ .../apache/spark/sql/GeneratorFunctionSuite.scala | 9 +++-- .../sql/errors/QueryCompilationErrorsSuite.scala | 46 ++++++++++++++++++++++ 5 files changed, 90 insertions(+), 27 deletions(-) diff --git a/core/src/main/resources/error/error-classes.json b/core/src/main/resources/error/error-classes.json index 5eab18dfd35..31ec5aaa05e 100644 --- a/core/src/main/resources/error/error-classes.json +++ b/core/src/main/resources/error/error-classes.json @@ -198,6 +198,23 @@ "message" : [ "The feature is not supported: <feature>" ], "sqlState" : "0A000" }, + "UNSUPPORTED_GENERATOR" : { + "message" : [ "The generator is not supported: " ], + "subClass" : { + "MULTI_GENERATOR" : { + "message" : [ "only one generator allowed per <clause> clause but found <num>: <generators>" ] + }, + "NESTED_IN_EXPRESSIONS" : { + "message" : [ "nested in expressions <expression>" ] + }, + "NOT_GENERATOR" : { + "message" : [ "<functionName> is expected to be a generator. However, its class is <classCanonicalName>, which is not a generator." ] + }, + "OUTSIDE_SELECT" : { + "message" : [ "outside the SELECT clause, found: <plan>" ] + } + } + }, "UNSUPPORTED_GROUPING_EXPRESSION" : { "message" : [ "grouping()/grouping_id() can only be used with GroupingSets/Cube/Rollup" ] }, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala index 04b1d5f796d..6946f9dfc98 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala @@ -30,7 +30,7 @@ import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeRef import org.apache.spark.sql.catalyst.plans.JoinType import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoStatement, Join, LogicalPlan, SerdeInfo, Window} import org.apache.spark.sql.catalyst.trees.{Origin, TreeNode} -import org.apache.spark.sql.catalyst.util.{toPrettySQL, FailFastMode, ParseMode, PermissiveMode} +import org.apache.spark.sql.catalyst.util.{FailFastMode, ParseMode, PermissiveMode} import org.apache.spark.sql.connector.catalog._ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ import org.apache.spark.sql.connector.catalog.functions.{BoundFunction, UnboundFunction} @@ -113,21 +113,19 @@ private[sql] object QueryCompilationErrors extends QueryErrorsBase { } def nestedGeneratorError(trimmedNestedGenerator: Expression): Throwable = { - new AnalysisException( - "Generators are not supported when it's nested in " + - "expressions, but got: " + toPrettySQL(trimmedNestedGenerator)) + new AnalysisException(errorClass = "UNSUPPORTED_GENERATOR", + messageParameters = Array("NESTED_IN_EXPRESSIONS", toSQLExpr(trimmedNestedGenerator))) } def moreThanOneGeneratorError(generators: Seq[Expression], clause: String): Throwable = { - new AnalysisException( - s"Only one generator allowed per $clause clause but found " + - generators.size + ": " + generators.map(toPrettySQL).mkString(", ")) + new AnalysisException(errorClass = "UNSUPPORTED_GENERATOR", + messageParameters = Array("MULTI_GENERATOR", + clause, generators.size.toString, generators.map(toSQLExpr).mkString(", "))) } def generatorOutsideSelectError(plan: LogicalPlan): Throwable = { - new AnalysisException( - "Generators are not supported outside the SELECT clause, but " + - "got: " + plan.simpleString(SQLConf.get.maxToStringFields)) + new AnalysisException(errorClass = "UNSUPPORTED_GENERATOR", + messageParameters = Array("OUTSIDE_SELECT", plan.simpleString(SQLConf.get.maxToStringFields))) } def legacyStoreAssignmentPolicyError(): Throwable = { @@ -324,8 +322,8 @@ private[sql] object QueryCompilationErrors extends QueryErrorsBase { } def generatorNotExpectedError(name: FunctionIdentifier, classCanonicalName: String): Throwable = { - new AnalysisException(s"$name is expected to be a generator. However, " + - s"its class is $classCanonicalName, which is not a generator.") + new AnalysisException(errorClass = "UNSUPPORTED_GENERATOR", + messageParameters = Array("NOT_GENERATOR", toSQLId(name.toString), classCanonicalName)) } def functionWithUnsupportedSyntaxError(prettyName: String, syntax: String): Throwable = { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala index a5b8663f5e6..89034432452 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala @@ -483,7 +483,7 @@ class AnalysisErrorSuite extends AnalysisTest { errorTest( "generator nested in expressions", listRelation.select(Explode($"list") + 1), - "Generators are not supported when it's nested in expressions, but got: (explode(list) + 1)" + "The generator is not supported: nested in expressions \"(explode(list) + 1)\"" :: Nil ) @@ -494,29 +494,29 @@ class AnalysisErrorSuite extends AnalysisTest { AttributeReference("nestedList", ArrayType(ArrayType(IntegerType)))()) nestedListRelation.select(Explode(Explode($"nestedList"))) }, - "Generators are not supported when it's nested in expressions, but got: " + - "explode(explode(nestedList))" :: Nil + "The generator is not supported: nested in expressions " + + """"explode(explode(nestedList))"""" :: Nil ) errorTest( "SPARK-30998: unsupported nested inner generators for aggregates", testRelation.select(Explode(Explode( CreateArray(CreateArray(min($"a") :: max($"a") :: Nil) :: Nil)))), - "Generators are not supported when it's nested in expressions, but got: " + - "explode(explode(array(array(min(a), max(a)))))" :: Nil + "The generator is not supported: nested in expressions " + + """"explode(explode(array(array(min(a), max(a)))))"""" :: Nil ) errorTest( "generator nested in expressions for aggregates", testRelation.select(Explode(CreateArray(min($"a") :: max($"a") :: Nil)) + 1), - "Generators are not supported when it's nested in expressions, but got: " + - "(explode(array(min(a), max(a))) + 1)" :: Nil + "The generator is not supported: nested in expressions " + + """"(explode(array(min(a), max(a))) + 1)"""" :: Nil ) errorTest( "generator appears in operator which is not Project", listRelation.sortBy(Explode($"list").asc), - "Generators are not supported outside the SELECT clause, but got: Sort" :: Nil + "The generator is not supported: outside the SELECT clause, found: Sort" :: Nil ) errorTest( @@ -534,15 +534,16 @@ class AnalysisErrorSuite extends AnalysisTest { errorTest( "more than one generators in SELECT", listRelation.select(Explode($"list"), Explode($"list")), - "Only one generator allowed per select clause but found 2: explode(list), explode(list)" :: Nil + "The generator is not supported: only one generator allowed per select clause but found 2: " + + """"explode(list)", "explode(list)"""" :: Nil ) errorTest( "more than one generators for aggregates in SELECT", testRelation.select(Explode(CreateArray(min($"a") :: Nil)), Explode(CreateArray(max($"a") :: Nil))), - "Only one generator allowed per select clause but found 2: " + - "explode(array(min(a))), explode(array(max(a)))" :: Nil + "The generator is not supported: only one generator allowed per select clause but found 2: " + + """"explode(array(min(a)))", "explode(array(max(a)))"""" :: Nil ) errorTest( diff --git a/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala index 09afedea7a5..08280c08cd2 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala @@ -332,7 +332,7 @@ class GeneratorFunctionSuite extends QueryTest with SharedSparkSession { val msg1 = intercept[AnalysisException] { sql("select 1 + explode(array(min(c2), max(c2))) from t1 group by c1") }.getMessage - assert(msg1.contains("Generators are not supported when it's nested in expressions")) + assert(msg1.contains("The generator is not supported: nested in expressions")) val msg2 = intercept[AnalysisException] { sql( @@ -342,7 +342,8 @@ class GeneratorFunctionSuite extends QueryTest with SharedSparkSession { |from t1 group by c1 """.stripMargin) }.getMessage - assert(msg2.contains("Only one generator allowed per aggregate clause")) + assert(msg2.contains("The generator is not supported: " + + "only one generator allowed per aggregate clause")) } } @@ -350,8 +351,8 @@ class GeneratorFunctionSuite extends QueryTest with SharedSparkSession { val errMsg = intercept[AnalysisException] { sql("SELECT array(array(1, 2), array(3)) v").select(explode(explode($"v"))).collect }.getMessage - assert(errMsg.contains("Generators are not supported when it's nested in expressions, " + - "but got: explode(explode(v))")) + assert(errMsg.contains("The generator is not supported: " + + "nested in expressions \"explode(explode(v))\"")) } test("SPARK-30997: generators in aggregate expressions for dataframe") { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala index 3da5202a2ad..9e18e4e6692 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala @@ -205,4 +205,50 @@ class QueryCompilationErrorsSuite extends QueryTest with SharedSparkSession { "The deserializer is not supported: try to map \"STRUCT<a: STRING, b: INT>\" " + "to Tuple1, but failed as the number of fields does not line up.") } + + test("UNSUPPORTED_GENERATOR: " + + "generators are not supported when it's nested in expressions") { + val e = intercept[AnalysisException]( + sql("""select explode(Array(1, 2, 3)) + 1""").collect() + ) + assert(e.errorClass === Some("UNSUPPORTED_GENERATOR")) + assert(e.message === + """The generator is not supported: """ + + """nested in expressions "(explode(array(1, 2, 3)) + 1)"""") + } + + test("UNSUPPORTED_GENERATOR: only one generator allowed") { + val e = intercept[AnalysisException]( + sql("""select explode(Array(1, 2, 3)), explode(Array(1, 2, 3))""").collect() + ) + assert(e.errorClass === Some("UNSUPPORTED_GENERATOR")) + assert(e.message === + "The generator is not supported: only one generator allowed per select clause " + + """but found 2: "explode(array(1, 2, 3))", "explode(array(1, 2, 3))"""") + } + + test("UNSUPPORTED_GENERATOR: generators are not supported outside the SELECT clause") { + val e = intercept[AnalysisException]( + sql("""select 1 from t order by explode(Array(1, 2, 3))""").collect() + ) + assert(e.errorClass === Some("UNSUPPORTED_GENERATOR")) + assert(e.message === + "The generator is not supported: outside the SELECT clause, found: " + + "'Sort [explode(array(1, 2, 3)) ASC NULLS FIRST], true") + } + + test("UNSUPPORTED_GENERATOR: not a generator") { + val e = intercept[AnalysisException]( + sql( + """ + |SELECT explodedvalue.* + |FROM VALUES array(1, 2, 3) AS (value) + |LATERAL VIEW array_contains(value, 1) AS explodedvalue""".stripMargin).collect() + ) + assert(e.errorClass === Some("UNSUPPORTED_GENERATOR")) + assert(e.message === + """The generator is not supported: `array_contains` is expected to be a generator. """ + + "However, its class is org.apache.spark.sql.catalyst.expressions.ArrayContains, " + + "which is not a generator.") + } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org