srielau commented on code in PR #58530:
URL: https://github.com/apache/spark/pull/58530#discussion_r3973077437
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementSplitter.scala:
##########
@@ -285,8 +370,155 @@ object SqlStatementSplitter {
val unclosed = lexer.has_unclosed_bracketed_comment
val partial =
- if (bufferHasContent || unclosed) buffer.toString.trim else ""
- SqlStatementSplitResult(completeStatements.toSeq, partial, unclosed &&
partial.nonEmpty)
+ if (bufferHasContent || unclosed) positionedStatement("") else None
+ PositionedSqlStatementSplitResult(
+ completeStatements.toSeq,
+ partial,
+ unclosed && partial.nonEmpty)
+ }
+
+ /**
+ * Returns the delimiter-array index and ending token index of a real outer
END for a malformed
+ * compound statement. Error recovery may repair the body, but a missing END
is synthetic and
+ * has token index -1.
+ */
+ private def findMalformedCompoundEnd(
+ sqlText: String,
+ toUtf16: Array[Int],
+ stream: CommonTokenStream,
+ startIdx: Int,
+ delimiterPositions: Array[Int],
+ fromDelimiter: Int,
+ validationPreprocess: String => String,
+ conf: SqlApiConf): Option[(Int, Int)] = {
+ if (stream.get(startIdx).getType != SqlBaseLexer.BEGIN) {
+ return None
+ }
+
+ var delimiter = fromDelimiter
+ while (delimiter <= delimiterPositions.length) {
+ val endIdx = if (delimiter < delimiterPositions.length) {
+ delimiterPositions(delimiter)
+ } else {
+ stream.size() - 1
+ }
+ val firstTok = stream.get(startIdx)
+ val lastTok = stream.get(endIdx)
+ val regionStart = toUtf16(firstTok.getStartIndex)
+ val regionEnd = if (lastTok.getType == Token.EOF) {
+ sqlText.length
+ } else {
+ toUtf16(lastTok.getStopIndex + 1)
+ }
+ val candidate = validationPreprocess(sqlText.substring(regionStart,
regionEnd))
Review Comment:
Fixed in 95b1cf5ebce. The parse_sql path no longer loops over growing
delimiter prefixes or repeatedly preprocesses, lexes, and parses substrings. It
tokenizes the input once and invokes the dedicated batch-boundary grammar once,
so boundary work is linear in the input size. The existing prefix parser
remains only on the unchanged generic splitter path.
##########
sql/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseSql.scala:
##########
@@ -27,38 +27,46 @@ import org.apache.spark.sql.types.{AbstractDataType,
DataType, StringType}
import org.apache.spark.unsafe.types.UTF8String
/**
- * Parses a SQL statement string and returns a compact JSON description of the
- * unresolved statement (identifier/code, lineage references, select-list
names,
- * parameters), or a STANDARD-format error object when the statement does not
- * parse.
+ * Parses a SQL batch string and returns a compact JSON array describing its
+ * unresolved statements (source position, identifier/code, lineage references,
+ * select-list names, parameters). A statement that does not parse is
represented
Review Comment:
Fixed in 95b1cf5ebce. The class Scaladoc now says that an unparsable
statement produces a result object containing a nested STANDARD-format error
object, matching `$[0].error.errorClass` and the generated output.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementSplitter.scala:
##########
@@ -285,8 +370,155 @@ object SqlStatementSplitter {
val unclosed = lexer.has_unclosed_bracketed_comment
val partial =
- if (bufferHasContent || unclosed) buffer.toString.trim else ""
- SqlStatementSplitResult(completeStatements.toSeq, partial, unclosed &&
partial.nonEmpty)
+ if (bufferHasContent || unclosed) positionedStatement("") else None
+ PositionedSqlStatementSplitResult(
+ completeStatements.toSeq,
+ partial,
+ unclosed && partial.nonEmpty)
+ }
+
+ /**
+ * Returns the delimiter-array index and ending token index of a real outer
END for a malformed
+ * compound statement. Error recovery may repair the body, but a missing END
is synthetic and
+ * has token index -1.
+ */
+ private def findMalformedCompoundEnd(
+ sqlText: String,
+ toUtf16: Array[Int],
+ stream: CommonTokenStream,
+ startIdx: Int,
+ delimiterPositions: Array[Int],
+ fromDelimiter: Int,
+ validationPreprocess: String => String,
+ conf: SqlApiConf): Option[(Int, Int)] = {
+ if (stream.get(startIdx).getType != SqlBaseLexer.BEGIN) {
+ return None
+ }
+
+ var delimiter = fromDelimiter
+ while (delimiter <= delimiterPositions.length) {
+ val endIdx = if (delimiter < delimiterPositions.length) {
+ delimiterPositions(delimiter)
+ } else {
+ stream.size() - 1
+ }
+ val firstTok = stream.get(startIdx)
+ val lastTok = stream.get(endIdx)
+ val regionStart = toUtf16(firstTok.getStartIndex)
+ val regionEnd = if (lastTok.getType == Token.EOF) {
+ sqlText.length
+ } else {
+ toUtf16(lastTok.getStopIndex + 1)
+ }
+ val candidate = validationPreprocess(sqlText.substring(regionStart,
regionEnd))
+ val lexer = new SqlBaseLexer(
+ new UpperCaseCharStream(CharStreams.fromString(candidate)))
+ lexer.removeErrorListeners()
+ val tokens = new CommonTokenStream(lexer)
+ tokens.fill()
+ val parser = new SqlBaseParser(tokens)
+ configureSplitterParser(parser, conf, bailOnError = false)
+ parser.getInterpreter.setPredictionMode(PredictionMode.LL)
+ try {
+ val context = parser.singleCompoundStatement()
+ val end = context.END()
+ if (end != null && isOuterCompoundEnd(tokens, end.getSymbol)) {
+ return Some((delimiter, endIdx))
+ }
+ } catch {
+ case _: StackOverflowError => return None
+ }
+ delimiter += 1
+ }
+ None
+ }
+
+ /** Returns true only when a real recovered END closes the candidate's outer
BEGIN. */
+ private def isOuterCompoundEnd(tokens: CommonTokenStream, recoveredEnd:
Token): Boolean = {
+ if (recoveredEnd.getTokenIndex < 0) return false
+ val suffixEnd = trailingEndToken(tokens)
+ suffixEnd != null && closesOuterBegin(tokens, suffixEnd)
+ }
+
+ private def closesOuterBegin(tokens: CommonTokenStream, suffixEnd: Token):
Boolean = {
+ var depth = 0
+ var index = 0
+ val limit = suffixEnd.getTokenIndex
+ while (index <= limit) {
+ val token = tokens.get(index)
+ if (token.getChannel != Token.HIDDEN_CHANNEL) {
+ token.getType match {
+ case SqlBaseLexer.BEGIN =>
Review Comment:
Fixed in 95b1cf5ebce. The malformed-compound prefix recovery and all
raw-token BEGIN/END classifiers have been removed. parse_sql now uses a
dedicated `parseSqlBatch` grammar entry point in one parser invocation. The
boundary grammar owns nested BEGIN blocks, IF/CASE/WHILE/REPEAT/LOOP/FOR
terminators, labels, and handler bodies; malformed leaf statements terminate
only at semicolons owned by those grammar contexts. A parser-only end token
marks a trailing statement without synthesizing structural delimiters. The
generic `splitStatements` path is unchanged. Added coverage for balanced and
unclosed compounds, nested/control/label/handler forms, malformed headers,
BEGIN/END identifiers in ANSI and default modes, SET/RESET, comments, and a
following SELECT.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementSplitter.scala:
##########
@@ -285,8 +370,155 @@ object SqlStatementSplitter {
val unclosed = lexer.has_unclosed_bracketed_comment
val partial =
- if (bufferHasContent || unclosed) buffer.toString.trim else ""
- SqlStatementSplitResult(completeStatements.toSeq, partial, unclosed &&
partial.nonEmpty)
+ if (bufferHasContent || unclosed) positionedStatement("") else None
+ PositionedSqlStatementSplitResult(
+ completeStatements.toSeq,
+ partial,
+ unclosed && partial.nonEmpty)
+ }
+
+ /**
+ * Returns the delimiter-array index and ending token index of a real outer
END for a malformed
Review Comment:
Addressed in 95b1cf5ebce by removing `findMalformedCompoundEnd` entirely
along with its tuple return and associated recovery heuristics. The new grammar
contexts directly expose each top-level statement and its real semicolon or
parser-only trailing delimiter.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
