This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 38897b1ce96 [SPARK-41095][SQL] Convert unresolved operators to
internal errors
38897b1ce96 is described below
commit 38897b1ce96a1e24629875619c165b4fd1fa2d8f
Author: Max Gekk <max.g...@gmail.com>
AuthorDate: Fri Nov 11 09:37:10 2022 +0300
[SPARK-41095][SQL] Convert unresolved operators to internal errors
### What changes were proposed in this pull request?
In the PR, I propose to interpret the `unresolved operator` issue as an
internal error, and throw `SparkException` w/ the error class `INTERNAL_ERROR`.
### Why are the changes needed?
The issues that leads to `unresolved operator` should be solved earlier w/
proper user-facing errors. If we reach the point when we cannot resolve an
operator, we should interpret this as Spark SQL internal error.
### Does this PR introduce _any_ user-facing change?
No, in most regular cases.
### How was this patch tested?
By running the affected and modified test suites:
```
$ PYSPARK_PYTHON=python3 build/sbt "sql/testOnly
org.apache.spark.sql.SQLQueryTestSuite"
$ build/sbt "test:testOnly *AnalysisErrorSuite"
```
Closes #38582 from MaxGekk/unresolved-op-internal-error.
Authored-by: Max Gekk <max.g...@gmail.com>
Signed-off-by: Max Gekk <max.g...@gmail.com>
---
core/src/main/resources/error/error-classes.json | 5 -----
core/src/main/scala/org/apache/spark/SparkException.scala | 10 ++++++++--
.../apache/spark/sql/catalyst/analysis/CheckAnalysis.scala | 8 +++++---
.../spark/sql/catalyst/analysis/AnalysisErrorSuite.scala | 12 ++++++++----
4 files changed, 21 insertions(+), 14 deletions(-)
diff --git a/core/src/main/resources/error/error-classes.json
b/core/src/main/resources/error/error-classes.json
index 2b626ba5761..63978e6be66 100644
--- a/core/src/main/resources/error/error-classes.json
+++ b/core/src/main/resources/error/error-classes.json
@@ -5107,11 +5107,6 @@
"Invalid expressions: [<invalidExprSqls>]"
]
},
- "_LEGACY_ERROR_TEMP_2442" : {
- "message" : [
- "unresolved operator <operator>"
- ]
- },
"_LEGACY_ERROR_TEMP_2443" : {
"message" : [
"Multiple definitions of observed metrics named '<name>': <plan>"
diff --git a/core/src/main/scala/org/apache/spark/SparkException.scala
b/core/src/main/scala/org/apache/spark/SparkException.scala
index 03938444e12..2f05b2ad6a7 100644
--- a/core/src/main/scala/org/apache/spark/SparkException.scala
+++ b/core/src/main/scala/org/apache/spark/SparkException.scala
@@ -68,11 +68,17 @@ class SparkException(
}
object SparkException {
- def internalError(msg: String): SparkException = {
+ def internalError(msg: String, context: Array[QueryContext], summary:
String): SparkException = {
new SparkException(
errorClass = "INTERNAL_ERROR",
messageParameters = Map("message" -> msg),
- cause = null)
+ cause = null,
+ context,
+ summary)
+ }
+
+ def internalError(msg: String): SparkException = {
+ internalError(msg, context = Array.empty[QueryContext], summary = "")
}
def internalError(msg: String, cause: Throwable): SparkException = {
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
index f88ef522f34..285c7396124 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
@@ -18,6 +18,7 @@ package org.apache.spark.sql.catalyst.analysis
import scala.collection.mutable
+import org.apache.spark.SparkException
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.SubExprUtils._
@@ -713,9 +714,10 @@ trait CheckAnalysis extends PredicateHelper with
LookupCatalog with QueryErrorsB
extendedCheckRules.foreach(_(plan))
plan.foreachUp {
case o if !o.resolved =>
- o.failAnalysis(
- errorClass = "_LEGACY_ERROR_TEMP_2442",
- messageParameters = Map("operator" ->
o.simpleString(SQLConf.get.maxToStringFields)))
+ throw SparkException.internalError(
+ msg = s"Found the unresolved operator:
${o.simpleString(SQLConf.get.maxToStringFields)}",
+ context = o.origin.getQueryContext,
+ summary = o.origin.context.summary)
case _ =>
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
index 30116d410f7..93006224814 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.catalyst.analysis
import org.scalatest.Assertions._
+import org.apache.spark.SparkException
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.dsl.expressions._
@@ -383,10 +384,13 @@ class AnalysisErrorSuite extends AnalysisTest {
Map("fieldName" -> "`c`", "fields" -> "`aField`, `bField`, `cField`"),
caseSensitive = false)
- errorTest(
- "catch all unresolved plan",
- UnresolvedTestPlan(),
- "unresolved" :: Nil)
+ checkError(
+ exception = intercept[SparkException] {
+ val analyzer = getAnalyzer
+ analyzer.checkAnalysis(analyzer.execute(UnresolvedTestPlan()))
+ },
+ errorClass = "INTERNAL_ERROR",
+ parameters = Map("message" -> "Found the unresolved operator:
'UnresolvedTestPlan"))
errorTest(
"union with unequal number of columns",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
