This is an automated email from the ASF dual-hosted git repository. gengliang pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.3 by this push: new a210e39 [SPARK-38716][SQL] Provide query context in map key not exists error a210e39 is described below commit a210e3929bb96086894a9a5a72f0fe5946b1659d Author: Gengliang Wang <gengli...@apache.org> AuthorDate: Fri Apr 1 11:40:32 2022 +0800 [SPARK-38716][SQL] Provide query context in map key not exists error ### What changes were proposed in this pull request? Provide query context in `map key does not exist` runtime error with ANSI SQL mode on, including - operator `[]` - function `element_at()` ### Why are the changes needed? Provide SQL query context of runtime errors to users, so that they can understand it better. ### Does this PR introduce _any_ user-facing change? Yes, improve the runtime error message of "map key does not exist" ### How was this patch tested? UT Closes #36025 from gengliangwang/mapKeyContext. Authored-by: Gengliang Wang <gengli...@apache.org> Signed-off-by: Gengliang Wang <gengli...@apache.org> (cherry picked from commit f0fc991f1d2e7b04b0e3967481f7e75e4322ae15) Signed-off-by: Gengliang Wang <gengli...@apache.org> --- core/src/main/resources/error/error-classes.json | 4 ++-- .../scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala | 5 ++++- .../spark/sql/catalyst/expressions/complexTypeExtractors.scala | 6 ++++-- .../scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala | 9 ++++++--- sql/core/src/test/resources/sql-tests/results/ansi/map.sql.out | 9 +++++++++ 5 files changed, 25 insertions(+), 8 deletions(-) diff --git a/core/src/main/resources/error/error-classes.json b/core/src/main/resources/error/error-classes.json index d7d7702..c8c1841 100644 --- a/core/src/main/resources/error/error-classes.json +++ b/core/src/main/resources/error/error-classes.json @@ -121,10 +121,10 @@ "sqlState" : "42000" }, "MAP_KEY_DOES_NOT_EXIST" : { - "message" : [ "Key %s does not exist. If necessary set %s to false to bypass this error." ] + "message" : [ "Key %s does not exist. If necessary set %s to false to bypass this error.%s" ] }, "MAP_KEY_DOES_NOT_EXIST_IN_ELEMENT_AT" : { - "message" : [ "Key %s does not exist. To return NULL instead, use 'try_element_at'. If necessary set %s to false to bypass this error." ] + "message" : [ "Key %s does not exist. To return NULL instead, use 'try_element_at'. If necessary set %s to false to bypass this error.%s" ] }, "MISSING_COLUMN" : { "message" : [ "Column '%s' does not exist. Did you mean one of the following? [%s]" ], diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 6d95067..6b44483 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -41,6 +41,7 @@ import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules._ import org.apache.spark.sql.catalyst.streaming.StreamingRelationV2 import org.apache.spark.sql.catalyst.trees.{AlwaysProcess, CurrentOrigin} +import org.apache.spark.sql.catalyst.trees.CurrentOrigin.withOrigin import org.apache.spark.sql.catalyst.trees.TreePattern._ import org.apache.spark.sql.catalyst.util.{toPrettySQL, CharVarcharUtils} import org.apache.spark.sql.connector.catalog._ @@ -1749,7 +1750,9 @@ class Analyzer(override val catalogManager: CatalogManager) case u @ UnresolvedExtractValue(child, fieldName) => val newChild = innerResolve(child, isTopLevel = false) if (newChild.resolved) { - ExtractValue(newChild, fieldName, resolver) + withOrigin(u.origin) { + ExtractValue(newChild, fieldName, resolver) + } } else { u.copy(child = newChild) } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala index cb7e06b..3cd404a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala @@ -367,7 +367,7 @@ trait GetMapValueUtil extends BinaryExpression with ImplicitCastInputTypes { if (!found) { if (failOnError) { - throw QueryExecutionErrors.mapKeyNotExistError(ordinal, isElementAtFunction) + throw QueryExecutionErrors.mapKeyNotExistError(ordinal, isElementAtFunction, origin.context) } else { null } @@ -400,9 +400,11 @@ trait GetMapValueUtil extends BinaryExpression with ImplicitCastInputTypes { } val keyJavaType = CodeGenerator.javaType(keyType) + lazy val errorContext = ctx.addReferenceObj("errCtx", origin.context) nullSafeCodeGen(ctx, ev, (eval1, eval2) => { val keyNotFoundBranch = if (failOnError) { - s"throw QueryExecutionErrors.mapKeyNotExistError($eval2, $isElementAtFunction);" + s"throw QueryExecutionErrors.mapKeyNotExistError(" + + s"$eval2, $isElementAtFunction, $errorContext);" } else { s"${ev.isNull} = true;" } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index 304801e..2a9b3c0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -165,13 +165,16 @@ object QueryExecutionErrors { messageParameters = Array(index.toString, numElements.toString, SQLConf.ANSI_ENABLED.key)) } - def mapKeyNotExistError(key: Any, isElementAtFunction: Boolean): NoSuchElementException = { + def mapKeyNotExistError( + key: Any, + isElementAtFunction: Boolean, + context: String): NoSuchElementException = { if (isElementAtFunction) { new SparkNoSuchElementException(errorClass = "MAP_KEY_DOES_NOT_EXIST_IN_ELEMENT_AT", - messageParameters = Array(key.toString, SQLConf.ANSI_ENABLED.key)) + messageParameters = Array(key.toString, SQLConf.ANSI_ENABLED.key, context)) } else { new SparkNoSuchElementException(errorClass = "MAP_KEY_DOES_NOT_EXIST", - messageParameters = Array(key.toString, SQLConf.ANSI_STRICT_INDEX_OPERATOR.key)) + messageParameters = Array(key.toString, SQLConf.ANSI_STRICT_INDEX_OPERATOR.key, context)) } } diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/map.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/map.sql.out index 5f7bd9f..5ba3727 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/map.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/map.sql.out @@ -9,6 +9,9 @@ struct<> -- !query output org.apache.spark.SparkNoSuchElementException Key 5 does not exist. To return NULL instead, use 'try_element_at'. If necessary set spark.sql.ansi.enabled to false to bypass this error. +== SQL(line 1, position 7) == +select element_at(map(1, 'a', 2, 'b'), 5) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- !query @@ -18,6 +21,9 @@ struct<> -- !query output org.apache.spark.SparkNoSuchElementException Key 5 does not exist. If necessary set spark.sql.ansi.strictIndexOperator to false to bypass this error. +== SQL(line 1, position 7) == +select map(1, 'a', 2, 'b')[5] + ^^^^^^^^^^^^^^^^^^^^^^ -- !query @@ -109,3 +115,6 @@ struct<> -- !query output org.apache.spark.SparkNoSuchElementException Key 5 does not exist. To return NULL instead, use 'try_element_at'. If necessary set spark.sql.ansi.enabled to false to bypass this error. +== SQL(line 1, position 7) == +select element_at(map(1, 'a', 2, 'b'), 5) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org