This is an automated email from the ASF dual-hosted git repository. maxgekk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new f291125ae08 [SPARK-40324][SQL] Provide a query context of `ParseException` f291125ae08 is described below commit f291125ae08ff98fed9106c4273349ceb7966697 Author: Max Gekk <max.g...@gmail.com> AuthorDate: Tue Sep 6 12:18:08 2022 +0300 [SPARK-40324][SQL] Provide a query context of `ParseException` ### What changes were proposed in this pull request? In the PR, I propose to extend `ParseException` and add new field `queryContext` which contains an array of query contexts. By default, the field is initialized by the context of currently processed tree node. Also I removed the wrapper `validateParsingError()` of `checkError()` to use last one directly. ### Why are the changes needed? To improve user experience with Spark SQL by providing more context of parsing errors. ### Does this PR introduce _any_ user-facing change? No, it extends existing functionality. ### How was this patch tested? By running the affected test suites: ``` $ build/sbt "sql/testOnly *QueryParsingErrorsSuite" $ build/sbt "sql/testOnly org.apache.spark.sql.SQLQueryTestSuite" $ build/sbt -Phive -Phive-thriftserver "test:testOnly *TruncateTableParserSuite" $ build/sbt -Phive-2.3 -Phive-thriftserver "test:testOnly *ShowPartitionsParserSuite" ``` Closes #37794 from MaxGekk/query-context-parse-exception. Authored-by: Max Gekk <max.g...@gmail.com> Signed-off-by: Max Gekk <max.g...@gmail.com> --- .../scala/org/apache/spark/SparkFunSuite.scala | 17 + .../spark/sql/catalyst/parser/ParseDriver.scala | 30 +- .../spark/sql/catalyst/trees/SQLQueryContext.scala | 2 +- .../resources/sql-tests/results/describe.sql.out | 9 +- .../sql-tests/results/join-lateral.sql.out | 18 +- .../resources/sql-tests/results/transform.sql.out | 18 +- .../resources/sql-tests/results/window.sql.out | 9 +- .../spark/sql/errors/QueryErrorsSuiteBase.scala | 15 - .../spark/sql/errors/QueryParsingErrorsSuite.scala | 468 +++++++++++++-------- .../command/ShowPartitionsParserSuite.scala | 9 +- .../command/TruncateTableParserSuite.scala | 6 +- 11 files changed, 398 insertions(+), 203 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/SparkFunSuite.scala b/core/src/test/scala/org/apache/spark/SparkFunSuite.scala index 6e44645379b..c15ebab511b 100644 --- a/core/src/test/scala/org/apache/spark/SparkFunSuite.scala +++ b/core/src/test/scala/org/apache/spark/SparkFunSuite.scala @@ -362,6 +362,23 @@ abstract class SparkFunSuite context: QueryContext): Unit = checkError(exception, errorClass, None, None, parameters, false, Array(context)) + protected def checkError( + exception: SparkThrowable, + errorClass: String, + errorSubClass: String, + sqlState: String, + context: QueryContext): Unit = + checkError(exception, errorClass, Some(errorSubClass), None, Map.empty, false, Array(context)) + + protected def checkError( + exception: SparkThrowable, + errorClass: String, + errorSubClass: String, + sqlState: String, + parameters: Map[String, String], + context: QueryContext): Unit = + checkError(exception, errorClass, Some(errorSubClass), None, parameters, false, Array(context)) + class LogAppender(msg: String = "", maxEvents: Int = 1000) extends AbstractAppender("logAppender", null, null, true, Property.EMPTY_ARRAY) { private val _loggingEvents = new ArrayBuffer[LogEvent]() diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala index 76757f9fc21..5eaef76a747 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala @@ -21,14 +21,14 @@ import org.antlr.v4.runtime.atn.PredictionMode import org.antlr.v4.runtime.misc.{Interval, ParseCancellationException} import org.antlr.v4.runtime.tree.TerminalNodeImpl -import org.apache.spark.SparkThrowableHelper +import org.apache.spark.{QueryContext, SparkThrowableHelper} import org.apache.spark.internal.Logging import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.{FunctionIdentifier, SQLConfHelper, TableIdentifier} import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.parser.ParserUtils.withOrigin import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.catalyst.trees.Origin +import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, Origin} import org.apache.spark.sql.errors.QueryParsingErrors import org.apache.spark.sql.types.{DataType, StructType} @@ -238,7 +238,8 @@ class ParseException( val stop: Origin, errorClass: Option[String] = None, errorSubClass: Option[String] = None, - messageParameters: Array[String] = Array.empty) + messageParameters: Array[String] = Array.empty, + queryContext: Array[QueryContext] = ParseException.getQueryContext()) extends AnalysisException( message, start.line, @@ -315,12 +316,23 @@ class ParseException( } def withCommand(cmd: String): ParseException = { - // PARSE_EMPTY_STATEMENT error class overrides the PARSE_SYNTAX_ERROR when cmd is empty - if (cmd.trim().isEmpty && errorClass.isDefined && errorClass.get == "PARSE_SYNTAX_ERROR") { - new ParseException(Option(cmd), start, stop, "PARSE_EMPTY_STATEMENT", Array[String]()) - } else { - new ParseException(Option(cmd), message, start, stop, errorClass, None, messageParameters) - } + val (cls, subCls, params) = + if (errorClass == Some("PARSE_SYNTAX_ERROR") && cmd.trim().isEmpty) { + // PARSE_EMPTY_STATEMENT error class overrides the PARSE_SYNTAX_ERROR when cmd is empty + (Some("PARSE_EMPTY_STATEMENT"), None, Array[String]()) + } else { + (errorClass, errorSubClass, messageParameters) + } + new ParseException(Option(cmd), message, start, stop, cls, subCls, params, queryContext) + } + + override def getQueryContext: Array[QueryContext] = queryContext +} + +object ParseException { + def getQueryContext(): Array[QueryContext] = { + val context = CurrentOrigin.get.context + if (context.isValid) Array(context) else Array.empty } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/SQLQueryContext.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/SQLQueryContext.scala index 2f6b0f885a0..99889cf7dae 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/SQLQueryContext.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/SQLQueryContext.scala @@ -124,7 +124,7 @@ case class SQLQueryContext( } } - private def isValid: Boolean = { + def isValid: Boolean = { sqlText.isDefined && originStartIndex.isDefined && originStopIndex.isDefined && originStartIndex.get >= 0 && originStopIndex.get < sqlText.get.length && originStartIndex.get <= originStopIndex.get diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out index 5cbc80d932e..45df378495c 100644 --- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out @@ -387,7 +387,14 @@ org.apache.spark.sql.catalyst.parser.ParseException "sqlState" : "42000", "messageParameters" : { "inputString" : "PARTITION specification is incomplete: `d`" - } + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 28, + "fragment" : "DESC t PARTITION (c='Us', d)" + } ] } diff --git a/sql/core/src/test/resources/sql-tests/results/join-lateral.sql.out b/sql/core/src/test/resources/sql-tests/results/join-lateral.sql.out index 12982778565..eee62aba35f 100644 --- a/sql/core/src/test/resources/sql-tests/results/join-lateral.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/join-lateral.sql.out @@ -152,7 +152,14 @@ org.apache.spark.sql.catalyst.parser.ParseException { "errorClass" : "UNSUPPORTED_FEATURE", "errorSubClass" : "LATERAL_NATURAL_JOIN", - "sqlState" : "0A000" + "sqlState" : "0A000", + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 18, + "stopIndex" : 60, + "fragment" : "NATURAL JOIN LATERAL (SELECT c1 + c2 AS c2)" + } ] } @@ -165,7 +172,14 @@ org.apache.spark.sql.catalyst.parser.ParseException { "errorClass" : "UNSUPPORTED_FEATURE", "errorSubClass" : "LATERAL_JOIN_USING", - "sqlState" : "0A000" + "sqlState" : "0A000", + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 18, + "stopIndex" : 63, + "fragment" : "JOIN LATERAL (SELECT c1 + c2 AS c2) USING (c2)" + } ] } diff --git a/sql/core/src/test/resources/sql-tests/results/transform.sql.out b/sql/core/src/test/resources/sql-tests/results/transform.sql.out index 4b6c12e992f..190e8a4cd68 100644 --- a/sql/core/src/test/resources/sql-tests/results/transform.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/transform.sql.out @@ -718,7 +718,14 @@ org.apache.spark.sql.catalyst.parser.ParseException { "errorClass" : "UNSUPPORTED_FEATURE", "errorSubClass" : "TRANSFORM_DISTINCT_ALL", - "sqlState" : "0A000" + "sqlState" : "0A000", + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 92, + "fragment" : "SELECT TRANSFORM(DISTINCT b, a, c)\n USING 'cat' AS (a, b, c)\nFROM script_trans\nWHERE a <= 4" + } ] } @@ -734,7 +741,14 @@ org.apache.spark.sql.catalyst.parser.ParseException { "errorClass" : "UNSUPPORTED_FEATURE", "errorSubClass" : "TRANSFORM_DISTINCT_ALL", - "sqlState" : "0A000" + "sqlState" : "0A000", + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 1, + "stopIndex" : 87, + "fragment" : "SELECT TRANSFORM(ALL b, a, c)\n USING 'cat' AS (a, b, c)\nFROM script_trans\nWHERE a <= 4" + } ] } diff --git a/sql/core/src/test/resources/sql-tests/results/window.sql.out b/sql/core/src/test/resources/sql-tests/results/window.sql.out index dd7a159936e..59345a4773f 100644 --- a/sql/core/src/test/resources/sql-tests/results/window.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/window.sql.out @@ -915,7 +915,14 @@ org.apache.spark.sql.catalyst.parser.ParseException "sqlState" : "42000", "messageParameters" : { "inputString" : "The definition of window `w` is repetitive." - } + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 226, + "stopIndex" : 394, + "fragment" : "WINDOW\n w AS (ORDER BY salary DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING),\n w AS (ORDER BY salary DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 2 FOLLOWING)" + } ] } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryErrorsSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryErrorsSuiteBase.scala index 525771f3038..02463f9e65a 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryErrorsSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryErrorsSuiteBase.scala @@ -18,25 +18,10 @@ package org.apache.spark.sql.errors import org.apache.spark.QueryContext -import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.test.SharedSparkSession trait QueryErrorsSuiteBase extends SharedSparkSession { - def validateParsingError( - sqlText: String, - errorClass: String, - errorSubClass: Option[String] = None, - sqlState: String, - parameters: Map[String, String] = Map.empty): Unit = { - checkError( - exception = intercept[ParseException](sql(sqlText)), - errorClass = errorClass, - errorSubClass = errorSubClass, - sqlState = Some(sqlState), - parameters = parameters) - } - case class ExpectedContext( objectType: String, objectName: String, diff --git a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryParsingErrorsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryParsingErrorsSuite.scala index e9379b461ec..96ebf3ccf0b 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryParsingErrorsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryParsingErrorsSuite.scala @@ -17,287 +17,383 @@ package org.apache.spark.sql.errors +import org.apache.spark.SparkThrowable import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.catalyst.parser.ParseException // Turn of the length check because most of the tests check entire error messages // scalastyle:off line.size.limit class QueryParsingErrorsSuite extends QueryTest with QueryErrorsSuiteBase { + private def parseException(sqlText: String): SparkThrowable = { + intercept[ParseException](sql(sqlText).collect()) + } + test("UNSUPPORTED_FEATURE: LATERAL join with NATURAL join not supported") { - validateParsingError( - sqlText = "SELECT * FROM t1 NATURAL JOIN LATERAL (SELECT c1 + c2 AS c2)", + checkError( + exception = parseException("SELECT * FROM t1 NATURAL JOIN LATERAL (SELECT c1 + c2 AS c2)"), errorClass = "UNSUPPORTED_FEATURE", - errorSubClass = Some("LATERAL_NATURAL_JOIN"), - sqlState = "0A000") + errorSubClass = "LATERAL_NATURAL_JOIN", + sqlState = "0A000", + context = ExpectedContext( + fragment = "NATURAL JOIN LATERAL (SELECT c1 + c2 AS c2)", + start = 17, + stop = 59)) } test("UNSUPPORTED_FEATURE: LATERAL join with USING join not supported") { - validateParsingError( - sqlText = "SELECT * FROM t1 JOIN LATERAL (SELECT c1 + c2 AS c2) USING (c2)", + checkError( + exception = parseException("SELECT * FROM t1 JOIN LATERAL (SELECT c1 + c2 AS c2) USING (c2)"), errorClass = "UNSUPPORTED_FEATURE", - errorSubClass = Some("LATERAL_JOIN_USING"), - sqlState = "0A000") + errorSubClass = "LATERAL_JOIN_USING", + sqlState = "0A000", + context = ExpectedContext( + fragment = "JOIN LATERAL (SELECT c1 + c2 AS c2) USING (c2)", + start = 17, + stop = 62)) } test("UNSUPPORTED_FEATURE: Unsupported LATERAL join type") { - Seq("RIGHT OUTER", "FULL OUTER", "LEFT SEMI", "LEFT ANTI").foreach { joinType => - validateParsingError( - sqlText = s"SELECT * FROM t1 $joinType JOIN LATERAL (SELECT c1 + c2 AS c3) ON c2 = c3", + Seq( + "RIGHT OUTER" -> (17, 74), + "FULL OUTER" -> (17, 73), + "LEFT SEMI" -> (17, 72), + "LEFT ANTI" -> (17, 72)).foreach { case (joinType, (start, stop)) => + checkError( + exception = parseException(s"SELECT * FROM t1 $joinType JOIN LATERAL (SELECT c1 + c2 AS c3) ON c2 = c3"), errorClass = "UNSUPPORTED_FEATURE", - errorSubClass = Some("LATERAL_JOIN_OF_TYPE"), + errorSubClass = "LATERAL_JOIN_OF_TYPE", sqlState = "0A000", - parameters = Map("joinType" -> joinType)) + parameters = Map("joinType" -> joinType), + context = ExpectedContext( + fragment = s"$joinType JOIN LATERAL (SELECT c1 + c2 AS c3) ON c2 = c3", + start = start, + stop = stop)) } } test("INVALID_SQL_SYNTAX: LATERAL can only be used with subquery") { Seq( - "SELECT * FROM t1, LATERAL t2" -> 26, - "SELECT * FROM t1 JOIN LATERAL t2" -> 30, - "SELECT * FROM t1, LATERAL (t2 JOIN t3)" -> 26, - "SELECT * FROM t1, LATERAL (LATERAL t2)" -> 26, - "SELECT * FROM t1, LATERAL VALUES (0, 1)" -> 26, - "SELECT * FROM t1, LATERAL RANGE(0, 1)" -> 26 - ).foreach { case (sqlText, pos) => - validateParsingError( - sqlText = sqlText, + ", LATERAL t2" -> ("FROM t1, LATERAL t2", 9, 27), + " JOIN LATERAL t2" -> ("JOIN LATERAL t2", 17, 31), + ", LATERAL (t2 JOIN t3)" -> ("FROM t1, LATERAL (t2 JOIN t3)", 9, 37), + ", LATERAL (LATERAL t2)" -> ("FROM t1, LATERAL (LATERAL t2)", 9, 37), + ", LATERAL VALUES (0, 1)" -> ("FROM t1, LATERAL VALUES (0, 1)", 9, 38), + ", LATERAL RANGE(0, 1)" -> ("FROM t1, LATERAL RANGE(0, 1)", 9, 36) + ).foreach { case (sqlText, (fragment, start, stop)) => + checkError( + exception = parseException(s"SELECT * FROM t1$sqlText"), errorClass = "INVALID_SQL_SYNTAX", sqlState = "42000", - parameters = Map("inputString" -> "LATERAL can only be used with subquery.")) + parameters = Map("inputString" -> "LATERAL can only be used with subquery."), + context = ExpectedContext(fragment, start, stop)) } } test("UNSUPPORTED_FEATURE: NATURAL CROSS JOIN is not supported") { - validateParsingError( - sqlText = "SELECT * FROM a NATURAL CROSS JOIN b", + checkError( + exception = parseException("SELECT * FROM a NATURAL CROSS JOIN b"), errorClass = "UNSUPPORTED_FEATURE", - errorSubClass = Some("NATURAL_CROSS_JOIN"), - sqlState = "0A000") + errorSubClass = "NATURAL_CROSS_JOIN", + sqlState = "0A000", + context = ExpectedContext( + fragment = "NATURAL CROSS JOIN b", + start = 16, + stop = 35)) } test("INVALID_SQL_SYNTAX: redefine window") { - validateParsingError( - sqlText = "SELECT min(a) OVER win FROM t1 WINDOW win AS win, win AS win2", + checkError( + exception = parseException("SELECT min(a) OVER win FROM t1 WINDOW win AS win, win AS win2"), errorClass = "INVALID_SQL_SYNTAX", sqlState = "42000", - parameters = Map("inputString" -> "The definition of window `win` is repetitive.")) + parameters = Map("inputString" -> "The definition of window `win` is repetitive."), + context = ExpectedContext( + fragment = "WINDOW win AS win, win AS win2", + start = 31, + stop = 60)) } test("INVALID_SQL_SYNTAX: invalid window reference") { - validateParsingError( - sqlText = "SELECT min(a) OVER win FROM t1 WINDOW win AS win", + checkError( + exception = parseException("SELECT min(a) OVER win FROM t1 WINDOW win AS win"), errorClass = "INVALID_SQL_SYNTAX", sqlState = "42000", - parameters = Map("inputString" -> "Window reference `win` is not a window specification.")) + parameters = Map("inputString" -> "Window reference `win` is not a window specification."), + context = ExpectedContext( + fragment = "WINDOW win AS win", + start = 31, + stop = 47)) } test("INVALID_SQL_SYNTAX: window reference cannot be resolved") { - validateParsingError( - sqlText = "SELECT min(a) OVER win FROM t1 WINDOW win AS win2", + checkError( + exception = parseException("SELECT min(a) OVER win FROM t1 WINDOW win AS win2"), errorClass = "INVALID_SQL_SYNTAX", sqlState = "42000", - parameters = Map("inputString" -> "Cannot resolve window reference `win2`.")) + parameters = Map("inputString" -> "Cannot resolve window reference `win2`."), + context = ExpectedContext( + fragment = "WINDOW win AS win2", + start = 31, + stop = 48)) } test("UNSUPPORTED_FEATURE: TRANSFORM does not support DISTINCT/ALL") { - validateParsingError( - sqlText = "SELECT TRANSFORM(DISTINCT a) USING 'a' FROM t", + val sqlText = "SELECT TRANSFORM(DISTINCT a) USING 'a' FROM t" + checkError( + exception = parseException(sqlText), errorClass = "UNSUPPORTED_FEATURE", - errorSubClass = Some("TRANSFORM_DISTINCT_ALL"), - sqlState = "0A000") + errorSubClass = "TRANSFORM_DISTINCT_ALL", + sqlState = "0A000", + context = ExpectedContext( + fragment = sqlText, + start = 0, + stop = 44)) } test("UNSUPPORTED_FEATURE: In-memory mode does not support TRANSFORM with serde") { - validateParsingError( - sqlText = "SELECT TRANSFORM(a) ROW FORMAT SERDE " + - "'org.apache.hadoop.hive.serde2.OpenCSVSerde' USING 'a' FROM t", + val sqlText = "SELECT TRANSFORM(a) ROW FORMAT SERDE " + + "'org.apache.hadoop.hive.serde2.OpenCSVSerde' USING 'a' FROM t" + checkError( + exception = parseException(sqlText), errorClass = "UNSUPPORTED_FEATURE", - errorSubClass = Some("TRANSFORM_NON_HIVE"), - sqlState = "0A000") + errorSubClass = "TRANSFORM_NON_HIVE", + sqlState = "0A000", + context = ExpectedContext( + fragment = sqlText, + start = 0, + stop = 97)) } test("INVALID_SQL_SYNTAX: Too many arguments for transform") { - validateParsingError( - sqlText = "CREATE TABLE table(col int) PARTITIONED BY (years(col,col))", + checkError( + exception = parseException("CREATE TABLE table(col int) PARTITIONED BY (years(col,col))"), errorClass = "INVALID_SQL_SYNTAX", sqlState = "42000", - parameters = Map("inputString" -> "Too many arguments for transform `years`")) + parameters = Map("inputString" -> "Too many arguments for transform `years`"), + context = ExpectedContext( + fragment = "years(col,col)", + start = 44, + stop = 57)) } test("INVALID_SQL_SYNTAX: Invalid table value function name") { - validateParsingError( - sqlText = "SELECT * FROM db.func()", + checkError( + exception = parseException("SELECT * FROM db.func()"), errorClass = "INVALID_SQL_SYNTAX", sqlState = "42000", - parameters = Map("inputString" -> "table valued function cannot specify database name ")) - - validateParsingError( - sqlText = "SELECT * FROM ns.db.func()", + parameters = Map("inputString" -> "table valued function cannot specify database name "), + context = ExpectedContext( + fragment = "db.func()", + start = 14, + stop = 22)) + + checkError( + exception = parseException("SELECT * FROM ns.db.func()"), errorClass = "INVALID_SQL_SYNTAX", sqlState = "42000", - parameters = Map("inputString" -> "table valued function cannot specify database name ")) + parameters = Map("inputString" -> "table valued function cannot specify database name "), + context = ExpectedContext( + fragment = "ns.db.func()", + start = 14, + stop = 25)) } test("INVALID_SQL_SYNTAX: Invalid scope in show functions") { - validateParsingError( - sqlText = "SHOW sys FUNCTIONS", + val sqlText = "SHOW sys FUNCTIONS" + checkError( + exception = parseException(sqlText), errorClass = "INVALID_SQL_SYNTAX", sqlState = "42000", - parameters = Map("inputString" -> "SHOW `sys` FUNCTIONS not supported")) + parameters = Map("inputString" -> "SHOW `sys` FUNCTIONS not supported"), + context = ExpectedContext( + fragment = sqlText, + start = 0, + stop = 17)) } test("INVALID_SQL_SYNTAX: Invalid pattern in show functions") { - validateParsingError( - sqlText = "SHOW FUNCTIONS IN db f1", + val sqlText1 = "SHOW FUNCTIONS IN db f1" + checkError( + exception = parseException(sqlText1), errorClass = "INVALID_SQL_SYNTAX", sqlState = "42000", parameters = Map("inputString" -> - "Invalid pattern in SHOW FUNCTIONS: `f1`. It must be a \"STRING\" literal.")) - validateParsingError( - sqlText = "SHOW FUNCTIONS IN db LIKE f1", + "Invalid pattern in SHOW FUNCTIONS: `f1`. It must be a \"STRING\" literal."), + context = ExpectedContext( + fragment = sqlText1, + start = 0, + stop = 22)) + val sqlText2 = "SHOW FUNCTIONS IN db LIKE f1" + checkError( + exception = parseException(sqlText2), errorClass = "INVALID_SQL_SYNTAX", sqlState = "42000", parameters = Map("inputString" -> - "Invalid pattern in SHOW FUNCTIONS: `f1`. It must be a \"STRING\" literal.")) + "Invalid pattern in SHOW FUNCTIONS: `f1`. It must be a \"STRING\" literal."), + context = ExpectedContext( + fragment = sqlText2, + start = 0, + stop = 27)) } test("INVALID_SQL_SYNTAX: Create function with both if not exists and replace") { val sqlText = - """ - |CREATE OR REPLACE FUNCTION IF NOT EXISTS func1 as + """CREATE OR REPLACE FUNCTION IF NOT EXISTS func1 as |'com.matthewrathbone.example.SimpleUDFExample' USING JAR '/path/to/jar1', - |JAR '/path/to/jar2' - |""".stripMargin + |JAR '/path/to/jar2'""".stripMargin - validateParsingError( - sqlText = sqlText, + checkError( + exception = parseException(sqlText), errorClass = "INVALID_SQL_SYNTAX", sqlState = "42000", parameters = Map("inputString" -> - "CREATE FUNCTION with both IF NOT EXISTS and REPLACE is not allowed.")) + "CREATE FUNCTION with both IF NOT EXISTS and REPLACE is not allowed."), + context = ExpectedContext( + fragment = sqlText, + start = 0, + stop = 142)) } test("INVALID_SQL_SYNTAX: Create temporary function with if not exists") { val sqlText = - """ - |CREATE TEMPORARY FUNCTION IF NOT EXISTS func1 as + """CREATE TEMPORARY FUNCTION IF NOT EXISTS func1 as |'com.matthewrathbone.example.SimpleUDFExample' USING JAR '/path/to/jar1', - |JAR '/path/to/jar2' - |""".stripMargin + |JAR '/path/to/jar2'""".stripMargin - validateParsingError( - sqlText = sqlText, + checkError( + exception = parseException(sqlText), errorClass = "INVALID_SQL_SYNTAX", sqlState = "42000", parameters = Map("inputString" -> - "It is not allowed to define a TEMPORARY FUNCTION with IF NOT EXISTS.")) + "It is not allowed to define a TEMPORARY FUNCTION with IF NOT EXISTS."), + context = ExpectedContext( + fragment = sqlText, + start = 0, + stop = 141)) } test("INVALID_SQL_SYNTAX: Create temporary function with multi-part name") { val sqlText = - """ - |CREATE TEMPORARY FUNCTION ns.db.func as + """CREATE TEMPORARY FUNCTION ns.db.func as |'com.matthewrathbone.example.SimpleUDFExample' USING JAR '/path/to/jar1', - |JAR '/path/to/jar2' - |""".stripMargin + |JAR '/path/to/jar2'""".stripMargin - validateParsingError( - sqlText = sqlText, + checkError( + exception = parseException(sqlText), errorClass = "INVALID_SQL_SYNTAX", sqlState = "42000", - parameters = Map("inputString" -> "Unsupported function name `ns`.`db`.`func`")) + parameters = Map("inputString" -> "Unsupported function name `ns`.`db`.`func`"), + context = ExpectedContext( + fragment = sqlText, + start = 0, + stop = 132)) } test("INVALID_SQL_SYNTAX: Specifying database while creating temporary function") { val sqlText = - """ - |CREATE TEMPORARY FUNCTION db.func as + """CREATE TEMPORARY FUNCTION db.func as |'com.matthewrathbone.example.SimpleUDFExample' USING JAR '/path/to/jar1', - |JAR '/path/to/jar2' - |""".stripMargin + |JAR '/path/to/jar2'""".stripMargin - validateParsingError( - sqlText = sqlText, + checkError( + exception = parseException(sqlText), errorClass = "INVALID_SQL_SYNTAX", sqlState = "42000", parameters = Map("inputString" -> - "Specifying a database in CREATE TEMPORARY FUNCTION is not allowed: `db`")) + "Specifying a database in CREATE TEMPORARY FUNCTION is not allowed: `db`"), + context = ExpectedContext( + fragment = sqlText, + start = 0, + stop = 129)) } test("INVALID_SQL_SYNTAX: Drop temporary function requires a single part name") { - validateParsingError( - sqlText = "DROP TEMPORARY FUNCTION db.func", + val sqlText = "DROP TEMPORARY FUNCTION db.func" + checkError( + exception = parseException(sqlText), errorClass = "INVALID_SQL_SYNTAX", sqlState = "42000", parameters = Map("inputString" -> - "DROP TEMPORARY FUNCTION requires a single part name but got: `db`.`func`")) + "DROP TEMPORARY FUNCTION requires a single part name but got: `db`.`func`"), + context = ExpectedContext( + fragment = sqlText, + start = 0, + stop = 30)) } test("DUPLICATE_KEY: Found duplicate partition keys") { - validateParsingError( - sqlText = "INSERT OVERWRITE TABLE table PARTITION(p1='1', p1='1') SELECT 'col1', 'col2'", + checkError( + exception = parseException("INSERT OVERWRITE TABLE table PARTITION(p1='1', p1='1') SELECT 'col1', 'col2'"), errorClass = "DUPLICATE_KEY", sqlState = "23000", - parameters = Map("keyColumn" -> "`p1`")) + parameters = Map("keyColumn" -> "`p1`"), + context = ExpectedContext( + fragment = "PARTITION(p1='1', p1='1')", + start = 29, + stop = 53)) } test("DUPLICATE_KEY: in table properties") { - validateParsingError( - sqlText = "ALTER TABLE dbx.tab1 SET TBLPROPERTIES ('key1' = '1', 'key1' = '2')", + checkError( + exception = parseException("ALTER TABLE dbx.tab1 SET TBLPROPERTIES ('key1' = '1', 'key1' = '2')"), errorClass = "DUPLICATE_KEY", sqlState = "23000", - parameters = Map("keyColumn" -> "`key1`")) + parameters = Map("keyColumn" -> "`key1`"), + context = ExpectedContext( + fragment = "('key1' = '1', 'key1' = '2')", + start = 39, + stop = 66)) } test("PARSE_EMPTY_STATEMENT: empty input") { - validateParsingError( - sqlText = "", + checkError( + exception = parseException(""), errorClass = "PARSE_EMPTY_STATEMENT", - sqlState = "42000") + sqlState = Some("42000")) - validateParsingError( - sqlText = " ", + checkError( + exception = parseException(" "), errorClass = "PARSE_EMPTY_STATEMENT", - sqlState = "42000") + sqlState = Some("42000")) - validateParsingError( - sqlText = " \n", + checkError( + exception = parseException(" \n"), errorClass = "PARSE_EMPTY_STATEMENT", - sqlState = "42000") + sqlState = Some("42000")) } test("PARSE_SYNTAX_ERROR: no viable input") { - val sqlText = "select ((r + 1) " - validateParsingError( - sqlText = sqlText, + checkError( + exception = parseException("select ((r + 1) "), errorClass = "PARSE_SYNTAX_ERROR", sqlState = "42000", parameters = Map("error" -> "end of input", "hint" -> "")) } test("PARSE_SYNTAX_ERROR: extraneous input") { - validateParsingError( - sqlText = "select 1 1", + checkError( + exception = parseException("select 1 1"), errorClass = "PARSE_SYNTAX_ERROR", sqlState = "42000", parameters = Map("error" -> "'1'", "hint" -> ": extra input '1'")) - validateParsingError( - sqlText = "select *\nfrom r as q t", + checkError( + exception = parseException("select *\nfrom r as q t"), errorClass = "PARSE_SYNTAX_ERROR", sqlState = "42000", parameters = Map("error" -> "'t'", "hint" -> ": extra input 't'")) } test("PARSE_SYNTAX_ERROR: mismatched input") { - validateParsingError( - sqlText = "select * from r order by q from t", + checkError( + exception = parseException("select * from r order by q from t"), errorClass = "PARSE_SYNTAX_ERROR", sqlState = "42000", parameters = Map("error" -> "'from'", "hint" -> "")) - validateParsingError( - sqlText = "select *\nfrom r\norder by q\nfrom t", + checkError( + exception = parseException("select *\nfrom r\norder by q\nfrom t"), errorClass = "PARSE_SYNTAX_ERROR", sqlState = "42000", parameters = Map("error" -> "'from'", "hint" -> "")) @@ -305,14 +401,14 @@ class QueryParsingErrorsSuite extends QueryTest with QueryErrorsSuiteBase { test("PARSE_SYNTAX_ERROR: jargon token substitute to user-facing language") { // '<EOF>' -> end of input - validateParsingError( - sqlText = "select count(*", + checkError( + exception = parseException("select count(*"), errorClass = "PARSE_SYNTAX_ERROR", sqlState = "42000", parameters = Map("error" -> "end of input", "hint" -> "")) - validateParsingError( - sqlText = "select 1 as a from", + checkError( + exception = parseException("select 1 as a from"), errorClass = "PARSE_SYNTAX_ERROR", sqlState = "42000", parameters = Map("error" -> "end of input", "hint" -> "")) @@ -320,108 +416,144 @@ class QueryParsingErrorsSuite extends QueryTest with QueryErrorsSuiteBase { test("PARSE_SYNTAX_ERROR - SPARK-21136: " + "misleading error message due to problematic antlr grammar") { - validateParsingError( - sqlText = "select * from a left join_ b on a.id = b.id", + checkError( + exception = parseException("select * from a left join_ b on a.id = b.id"), errorClass = "PARSE_SYNTAX_ERROR", sqlState = "42000", parameters = Map("error" -> "'join_'", "hint" -> ": missing 'JOIN'")) - validateParsingError( - sqlText = "select * from test where test.t is like 'test'", + checkError( + exception = parseException("select * from test where test.t is like 'test'"), errorClass = "PARSE_SYNTAX_ERROR", sqlState = "42000", parameters = Map("error" -> "'is'", "hint" -> "")) - validateParsingError( - sqlText = "SELECT * FROM test WHERE x NOT NULL", + checkError( + exception = parseException("SELECT * FROM test WHERE x NOT NULL"), errorClass = "PARSE_SYNTAX_ERROR", sqlState = "42000", parameters = Map("error" -> "'NOT'", "hint" -> "")) } test("INVALID_SQL_SYNTAX: show table partition key must set value") { - validateParsingError( - sqlText = "SHOW TABLE EXTENDED IN default LIKE 'employee' PARTITION (grade)", + checkError( + exception = parseException("SHOW TABLE EXTENDED IN default LIKE 'employee' PARTITION (grade)"), errorClass = "INVALID_SQL_SYNTAX", sqlState = "42000", - parameters = Map("inputString" -> "Partition key `grade` must set value (can't be empty).")) + parameters = Map("inputString" -> "Partition key `grade` must set value (can't be empty)."), + context = ExpectedContext( + fragment = "PARTITION (grade)", + start = 47, + stop = 63)) } test("INVALID_SQL_SYNTAX: expected a column reference for transform bucket") { - validateParsingError( - sqlText = - "CREATE TABLE my_tab(a INT, b STRING) USING parquet PARTITIONED BY (bucket(32, a, 66))", + checkError( + exception = parseException("CREATE TABLE my_tab(a INT, b STRING) USING parquet PARTITIONED BY (bucket(32, a, 66))"), errorClass = "INVALID_SQL_SYNTAX", sqlState = "42000", - parameters = Map("inputString" -> "Expected a column reference for transform `bucket`: 66")) + parameters = Map("inputString" -> "Expected a column reference for transform `bucket`: 66"), + context = ExpectedContext( + fragment = "bucket(32, a, 66)", + start = 67, + stop = 83)) } test("UNSUPPORTED_FEATURE: DESC TABLE COLUMN for a specific partition") { - validateParsingError( - sqlText = "DESCRIBE TABLE EXTENDED customer PARTITION (grade = 'A') customer.age", + val sqlText = "DESCRIBE TABLE EXTENDED customer PARTITION (grade = 'A') customer.age" + checkError( + exception = parseException(sqlText), errorClass = "UNSUPPORTED_FEATURE", - errorSubClass = Some("DESC_TABLE_COLUMN_PARTITION"), - sqlState = "0A000") + errorSubClass = "DESC_TABLE_COLUMN_PARTITION", + sqlState = "0A000", + context = ExpectedContext( + fragment = sqlText, + start = 0, + stop = 68)) } test("INVALID_SQL_SYNTAX: PARTITION specification is incomplete") { - validateParsingError( - sqlText = "DESCRIBE TABLE EXTENDED customer PARTITION (grade)", + val sqlText = "DESCRIBE TABLE EXTENDED customer PARTITION (grade)" + checkError( + exception = parseException(sqlText), errorClass = "INVALID_SQL_SYNTAX", sqlState = "42000", - parameters = Map("inputString" -> "PARTITION specification is incomplete: `grade`")) + parameters = Map("inputString" -> "PARTITION specification is incomplete: `grade`"), + context = ExpectedContext( + fragment = sqlText, + start = 0, + stop = 49)) } test("UNSUPPORTED_FEATURE: cannot set reserved namespace property") { - val sql = "CREATE NAMESPACE IF NOT EXISTS a.b.c WITH PROPERTIES ('location'='/home/user/db')" - validateParsingError( - sqlText = sql, + val sqlText = "CREATE NAMESPACE IF NOT EXISTS a.b.c WITH PROPERTIES ('location'='/home/user/db')" + checkError( + exception = parseException(sqlText), errorClass = "UNSUPPORTED_FEATURE", - errorSubClass = Some("SET_NAMESPACE_PROPERTY"), + errorSubClass = "SET_NAMESPACE_PROPERTY", sqlState = "0A000", parameters = Map( "property" -> "location", - "msg" -> "please use the LOCATION clause to specify it")) + "msg" -> "please use the LOCATION clause to specify it"), + context = ExpectedContext( + fragment = sqlText, + start = 0, + stop = 80)) } test("UNSUPPORTED_FEATURE: cannot set reserved table property") { - val sql = "CREATE TABLE student (id INT, name STRING, age INT) " + + val sqlText = "CREATE TABLE student (id INT, name STRING, age INT) " + "USING PARQUET TBLPROPERTIES ('provider'='parquet')" - validateParsingError( - sqlText = sql, + checkError( + exception = parseException(sqlText), errorClass = "UNSUPPORTED_FEATURE", - errorSubClass = Some("SET_TABLE_PROPERTY"), + errorSubClass = "SET_TABLE_PROPERTY", sqlState = "0A000", parameters = Map( "property" -> "provider", - "msg" -> "please use the USING clause to specify it")) + "msg" -> "please use the USING clause to specify it"), + context = ExpectedContext( + fragment = sqlText, + start = 0, + stop = 101)) } test("INVALID_PROPERTY_KEY: invalid property key for set quoted configuration") { - val sql = "set =`value`" - validateParsingError( - sqlText = sql, + val sqlText = "set =`value`" + checkError( + exception = parseException(sqlText), errorClass = "INVALID_PROPERTY_KEY", sqlState = null, - parameters = Map("key" -> "\"\"", "value" -> "\"value\"")) + parameters = Map("key" -> "\"\"", "value" -> "\"value\""), + context = ExpectedContext( + fragment = sqlText, + start = 0, + stop = 11)) } test("INVALID_PROPERTY_VALUE: invalid property value for set quoted configuration") { - val sql = "set `key`=1;2;;" - validateParsingError( - sqlText = sql, + checkError( + exception = parseException("set `key`=1;2;;"), errorClass = "INVALID_PROPERTY_VALUE", sqlState = null, - parameters = Map("value" -> "\"1;2;;\"", "key" -> "\"key\"")) + parameters = Map("value" -> "\"1;2;;\"", "key" -> "\"key\""), + context = ExpectedContext( + fragment = "set `key`=1;2", + start = 0, + stop = 12)) } test("UNSUPPORTED_FEATURE: cannot set Properties and DbProperties at the same time") { - val sql = "CREATE NAMESPACE IF NOT EXISTS a.b.c WITH PROPERTIES ('a'='a', 'b'='b', 'c'='c') " + + val sqlText = "CREATE NAMESPACE IF NOT EXISTS a.b.c WITH PROPERTIES ('a'='a', 'b'='b', 'c'='c') " + "WITH DBPROPERTIES('a'='a', 'b'='b', 'c'='c')" - validateParsingError( - sqlText = sql, + checkError( + exception = parseException(sqlText), errorClass = "UNSUPPORTED_FEATURE", - errorSubClass = Some("SET_PROPERTIES_AND_DBPROPERTIES"), - sqlState = "0A000") + errorSubClass = "SET_PROPERTIES_AND_DBPROPERTIES", + sqlState = "0A000", + context = ExpectedContext( + fragment = sqlText, + start = 0, + stop = 124)) } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowPartitionsParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowPartitionsParserSuite.scala index c9cc5911484..eba9c92e3ca 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowPartitionsParserSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowPartitionsParserSuite.scala @@ -22,7 +22,6 @@ import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.catalyst.plans.logical.ShowPartitions import org.apache.spark.sql.errors.QueryErrorsSuiteBase -import org.apache.spark.sql.execution.SparkSqlParser class ShowPartitionsParserSuite extends AnalysisTest with QueryErrorsSuiteBase { test("SHOW PARTITIONS") { @@ -50,10 +49,14 @@ class ShowPartitionsParserSuite extends AnalysisTest with QueryErrorsSuiteBase { test("empty values in non-optional partition specs") { checkError( exception = intercept[ParseException] { - new SparkSqlParser().parsePlan("SHOW PARTITIONS dbx.tab1 PARTITION (a='1', b)") + parsePlan("SHOW PARTITIONS dbx.tab1 PARTITION (a='1', b)") }, errorClass = "INVALID_SQL_SYNTAX", sqlState = "42000", - parameters = Map("inputString" -> "Partition key `b` must set value (can't be empty).")) + parameters = Map("inputString" -> "Partition key `b` must set value (can't be empty)."), + context = ExpectedContext( + fragment = "PARTITION (a='1', b)", + start = 25, + stop = 44)) } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/TruncateTableParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/TruncateTableParserSuite.scala index ef45cdebbe8..4a38ed9e2b9 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/TruncateTableParserSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/TruncateTableParserSuite.scala @@ -53,6 +53,10 @@ class TruncateTableParserSuite extends AnalysisTest with QueryErrorsSuiteBase { }, errorClass = "INVALID_SQL_SYNTAX", sqlState = "42000", - parameters = Map("inputString" -> "Partition key `b` must set value (can't be empty).")) + parameters = Map("inputString" -> "Partition key `b` must set value (can't be empty)."), + context = ExpectedContext( + fragment = "PARTITION (a='1', b)", + start = 24, + stop = 43)) } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org