This is an automated email from the ASF dual-hosted git repository. maxgekk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 1c8c47cb55d [SPARK-43914][SQL] Assign names to the error class _LEGACY_ERROR_TEMP_[2433-2437] 1c8c47cb55d is described below commit 1c8c47cb55da75526fef4dd41ed0734b01e71814 Author: Jiaan Geng <belie...@163.com> AuthorDate: Wed Jun 28 08:22:01 2023 +0300 [SPARK-43914][SQL] Assign names to the error class _LEGACY_ERROR_TEMP_[2433-2437] ### What changes were proposed in this pull request? The pr aims to assign names to the error class _LEGACY_ERROR_TEMP_[2433-2437]. ### Why are the changes needed? Improve the error framework. ### Does this PR introduce _any_ user-facing change? 'No'. ### How was this patch tested? Exists test cases updated. Closes #41476 from beliefer/SPARK-43914. Authored-by: Jiaan Geng <belie...@163.com> Signed-off-by: Max Gekk <max.g...@gmail.com> --- core/src/main/resources/error/error-classes.json | 34 ---------- .../sql/catalyst/analysis/CheckAnalysis.scala | 65 +++++++++++-------- .../sql/catalyst/analysis/AnalysisErrorSuite.scala | 74 ++++++++++++++++++++-- .../org/apache/spark/sql/DataFrameSuite.scala | 14 ++++ 4 files changed, 120 insertions(+), 67 deletions(-) diff --git a/core/src/main/resources/error/error-classes.json b/core/src/main/resources/error/error-classes.json index 342af0ffa6c..e441686432a 100644 --- a/core/src/main/resources/error/error-classes.json +++ b/core/src/main/resources/error/error-classes.json @@ -5637,40 +5637,6 @@ "Cannot change nullable column to non-nullable: <fieldName>." ] }, - "_LEGACY_ERROR_TEMP_2433" : { - "message" : [ - "Only a single table generating function is allowed in a SELECT clause, found:", - "<sqlExprs>." - ] - }, - "_LEGACY_ERROR_TEMP_2434" : { - "message" : [ - "Failure when resolving conflicting references in Join:", - "<plan>", - "Conflicting attributes: <conflictingAttributes>." - ] - }, - "_LEGACY_ERROR_TEMP_2435" : { - "message" : [ - "Failure when resolving conflicting references in Intersect:", - "<plan>", - "Conflicting attributes: <conflictingAttributes>." - ] - }, - "_LEGACY_ERROR_TEMP_2436" : { - "message" : [ - "Failure when resolving conflicting references in Except:", - "<plan>", - "Conflicting attributes: <conflictingAttributes>." - ] - }, - "_LEGACY_ERROR_TEMP_2437" : { - "message" : [ - "Failure when resolving conflicting references in AsOfJoin:", - "<plan>", - "Conflicting attributes: <conflictingAttributes>." - ] - }, "_LEGACY_ERROR_TEMP_2446" : { "message" : [ "Operation not allowed: <cmd> only works on table with location provided: <tableIdentWithDB>" diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala index 7c0e8f1490d..a0296d27361 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala @@ -674,9 +674,8 @@ trait CheckAnalysis extends PredicateHelper with LookupCatalog with QueryErrorsB } case p @ Project(exprs, _) if containsMultipleGenerators(exprs) => - p.failAnalysis( - errorClass = "_LEGACY_ERROR_TEMP_2433", - messageParameters = Map("sqlExprs" -> exprs.map(_.sql).mkString(","))) + val generators = exprs.filter(expr => expr.exists(_.isInstanceOf[Generator])) + throw QueryCompilationErrors.moreThanOneGeneratorError(generators, "SELECT") case p @ Project(projectList, _) => projectList.foreach(_.transformDownWithPruning( @@ -686,36 +685,48 @@ trait CheckAnalysis extends PredicateHelper with LookupCatalog with QueryErrorsB }) case j: Join if !j.duplicateResolved => - val conflictingAttributes = j.left.outputSet.intersect(j.right.outputSet) - j.failAnalysis( - errorClass = "_LEGACY_ERROR_TEMP_2434", - messageParameters = Map( - "plan" -> plan.toString, - "conflictingAttributes" -> conflictingAttributes.mkString(","))) + val conflictingAttributes = + j.left.outputSet.intersect(j.right.outputSet).map(toSQLExpr(_)).mkString(", ") + throw SparkException.internalError( + msg = s""" + |Failure when resolving conflicting references in ${j.nodeName}: + |${planToString(plan)} + |Conflicting attributes: $conflictingAttributes.""".stripMargin, + context = j.origin.getQueryContext, + summary = j.origin.context.summary) case i: Intersect if !i.duplicateResolved => - val conflictingAttributes = i.left.outputSet.intersect(i.right.outputSet) - i.failAnalysis( - errorClass = "_LEGACY_ERROR_TEMP_2435", - messageParameters = Map( - "plan" -> plan.toString, - "conflictingAttributes" -> conflictingAttributes.mkString(","))) + val conflictingAttributes = + i.left.outputSet.intersect(i.right.outputSet).map(toSQLExpr(_)).mkString(", ") + throw SparkException.internalError( + msg = s""" + |Failure when resolving conflicting references in ${i.nodeName}: + |${planToString(plan)} + |Conflicting attributes: $conflictingAttributes.""".stripMargin, + context = i.origin.getQueryContext, + summary = i.origin.context.summary) case e: Except if !e.duplicateResolved => - val conflictingAttributes = e.left.outputSet.intersect(e.right.outputSet) - e.failAnalysis( - errorClass = "_LEGACY_ERROR_TEMP_2436", - messageParameters = Map( - "plan" -> plan.toString, - "conflictingAttributes" -> conflictingAttributes.mkString(","))) + val conflictingAttributes = + e.left.outputSet.intersect(e.right.outputSet).map(toSQLExpr(_)).mkString(", ") + throw SparkException.internalError( + msg = s""" + |Failure when resolving conflicting references in ${e.nodeName}: + |${planToString(plan)} + |Conflicting attributes: $conflictingAttributes.""".stripMargin, + context = e.origin.getQueryContext, + summary = e.origin.context.summary) case j: AsOfJoin if !j.duplicateResolved => - val conflictingAttributes = j.left.outputSet.intersect(j.right.outputSet) - j.failAnalysis( - errorClass = "_LEGACY_ERROR_TEMP_2437", - messageParameters = Map( - "plan" -> plan.toString, - "conflictingAttributes" -> conflictingAttributes.mkString(","))) + val conflictingAttributes = + j.left.outputSet.intersect(j.right.outputSet).map(toSQLExpr(_)).mkString(", ") + throw SparkException.internalError( + msg = s""" + |Failure when resolving conflicting references in ${j.nodeName}: + |${planToString(plan)} + |Conflicting attributes: $conflictingAttributes.""".stripMargin, + context = j.origin.getQueryContext, + summary = j.origin.context.summary) // TODO: although map type is not orderable, technically map type should be able to be // used in equality comparison, remove this type check once we support it. diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala index 5d26fa5ea81..f994c999903 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.dsl.plans._ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate.{Count, Max} import org.apache.spark.sql.catalyst.parser.CatalystSqlParser -import org.apache.spark.sql.catalyst.plans.{Cross, LeftOuter, RightOuter} +import org.apache.spark.sql.catalyst.plans.{AsOfJoinDirection, Cross, Inner, LeftOuter, RightOuter} import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, GenericArrayData, MapData} import org.apache.spark.sql.internal.SQLConf @@ -781,11 +781,73 @@ class AnalysisErrorSuite extends AnalysisTest { test("error test for self-join") { val join = Join(testRelation, testRelation, Cross, None, JoinHint.NONE) - val error = intercept[AnalysisException] { - SimpleAnalyzer.checkAnalysis(join) - } - assert(error.message.contains("Failure when resolving conflicting references in Join")) - assert(error.message.contains("Conflicting attributes")) + checkError( + exception = intercept[SparkException] { + SimpleAnalyzer.checkAnalysis(join) + }, + errorClass = "INTERNAL_ERROR", + parameters = Map("message" -> + """ + |Failure when resolving conflicting references in Join: + |'Join Cross + |:- LocalRelation <empty>, [a#x] + |+- LocalRelation <empty>, [a#x] + | + |Conflicting attributes: "a".""".stripMargin)) + } + + test("error test for self-intersect") { + val intersect = Intersect(testRelation, testRelation, true) + checkError( + exception = intercept[SparkException] { + SimpleAnalyzer.checkAnalysis(intersect) + }, + errorClass = "INTERNAL_ERROR", + parameters = Map("message" -> + """ + |Failure when resolving conflicting references in Intersect All: + |'Intersect All true + |:- LocalRelation <empty>, [a#x] + |+- LocalRelation <empty>, [a#x] + | + |Conflicting attributes: "a".""".stripMargin)) + } + + test("error test for self-except") { + val except = Except(testRelation, testRelation, true) + checkError( + exception = intercept[SparkException] { + SimpleAnalyzer.checkAnalysis(except) + }, + errorClass = "INTERNAL_ERROR", + parameters = Map("message" -> + """ + |Failure when resolving conflicting references in Except All: + |'Except All true + |:- LocalRelation <empty>, [a#x] + |+- LocalRelation <empty>, [a#x] + | + |Conflicting attributes: "a".""".stripMargin)) + } + + test("error test for self-asOfJoin") { + val asOfJoin = + AsOfJoin(testRelation, testRelation, testRelation.output(0), testRelation.output(0), + None, Inner, tolerance = None, allowExactMatches = true, + direction = AsOfJoinDirection("backward")) + checkError( + exception = intercept[SparkException] { + SimpleAnalyzer.checkAnalysis(asOfJoin) + }, + errorClass = "INTERNAL_ERROR", + parameters = Map("message" -> + """ + |Failure when resolving conflicting references in AsOfJoin: + |'AsOfJoin (a#x >= a#x), Inner + |:- LocalRelation <empty>, [a#x] + |+- LocalRelation <empty>, [a#x] + | + |Conflicting attributes: "a".""".stripMargin)) } test("check grouping expression data types") { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala index 99f2ce9706d..6a90b3409d3 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala @@ -364,6 +364,20 @@ class DataFrameSuite extends QueryTest Row("a", Seq("a"), 1) :: Nil) } + test("more than one generator in SELECT clause") { + val df = Seq((Array("a"), 1)).toDF("a", "b") + + checkError( + exception = intercept[AnalysisException] { + df.select(explode($"a").as("a"), explode($"a").as("b")) + }, + errorClass = "UNSUPPORTED_GENERATOR.MULTI_GENERATOR", + parameters = Map( + "clause" -> "SELECT", + "num" -> "2", + "generators" -> "\"explode(a)\", \"explode(a)\"")) + } + test("sort after generate with join=true") { val df = Seq((Array("a"), 1)).toDF("a", "b") --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org