srielau commented on code in PR #58530:
URL: https://github.com/apache/spark/pull/58530#discussion_r4000008869
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementSplitter.scala:
##########
@@ -285,8 +370,155 @@ object SqlStatementSplitter {
val unclosed = lexer.has_unclosed_bracketed_comment
val partial =
- if (bufferHasContent || unclosed) buffer.toString.trim else ""
- SqlStatementSplitResult(completeStatements.toSeq, partial, unclosed &&
partial.nonEmpty)
+ if (bufferHasContent || unclosed) positionedStatement("") else None
+ PositionedSqlStatementSplitResult(
+ completeStatements.toSeq,
+ partial,
+ unclosed && partial.nonEmpty)
+ }
+
+ /**
+ * Returns the delimiter-array index and ending token index of a real outer
END for a malformed
+ * compound statement. Error recovery may repair the body, but a missing END
is synthetic and
+ * has token index -1.
+ */
+ private def findMalformedCompoundEnd(
+ sqlText: String,
+ toUtf16: Array[Int],
+ stream: CommonTokenStream,
+ startIdx: Int,
+ delimiterPositions: Array[Int],
+ fromDelimiter: Int,
+ validationPreprocess: String => String,
+ conf: SqlApiConf): Option[(Int, Int)] = {
+ if (stream.get(startIdx).getType != SqlBaseLexer.BEGIN) {
+ return None
+ }
+
+ var delimiter = fromDelimiter
+ while (delimiter <= delimiterPositions.length) {
+ val endIdx = if (delimiter < delimiterPositions.length) {
+ delimiterPositions(delimiter)
+ } else {
+ stream.size() - 1
+ }
+ val firstTok = stream.get(startIdx)
+ val lastTok = stream.get(endIdx)
+ val regionStart = toUtf16(firstTok.getStartIndex)
+ val regionEnd = if (lastTok.getType == Token.EOF) {
+ sqlText.length
+ } else {
+ toUtf16(lastTok.getStopIndex + 1)
+ }
+ val candidate = validationPreprocess(sqlText.substring(regionStart,
regionEnd))
Review Comment:
Yes: inheriting the shared splitter's documented O(k^2) growing-prefix cost
for valid BEGIN ... END scripts is an accepted limitation for parse_sql in this
PR. parse_sql uses the same splitter as SparkSqlParser.splitStatements so there
is one boundary contract. If that cost becomes a problem in practice, we will
address it in SqlStatementSplitter itself rather than adding a parse_sql-only
path.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
