srielau commented on code in PR #59868:
URL: https://github.com/apache/spark/pull/59868#discussion_r4416508739
##########
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/SQLQueryContextSuite.scala:
##########
@@ -45,5 +45,31 @@ class SQLQueryContextSuite extends SparkFunSuite with
SQLHelper {
).foreach { case ((sqlText, start, stop), expectedFragment) =>
assert(getFragment(sqlText, start, stop) === expectedFragment)
}
+
+ val emoji = new String(Character.toChars(0x1F600))
+ val sql = s"SELECT '$emoji' AS a, DATE 'bad'"
+ val start = sql.codePointCount(0, sql.indexOf("DATE"))
+ val stop = sql.codePointCount(0, sql.length) - 1
+ assert(getFragment(sql, start, stop) === "DATE 'bad'")
+ // A UTF-16 slice of the same indexes would include the comma and drop the
last quote.
+ assert(sql.substring(start, stop + 1) !== "DATE 'bad'")
+ }
+
+ test("summary caret follows code-point indexes after a supplementary
character") {
Review Comment:
Added later-line cases in debac86ae83. A context on the line after the emoji
now expects 10 carets under `DATE 'bad'`, and a context that follows other text
on that line expects the caret to start under `DATE`. Those are the layouts
where the UTF-16 shift no longer cancels.
##########
sql/api/src/main/scala/org/apache/spark/sql/catalyst/trees/QueryContexts.scala:
##########
@@ -19,6 +19,25 @@ package org.apache.spark.sql.catalyst.trees
import org.apache.spark.{QueryContext, QueryContextType,
SparkUnsupportedOperationException}
+object SQLQueryContext {
+
+ /**
+ * Inclusive Unicode code-point slice `[from, to]` of `text`. Indexes are
clamped to the text.
+ * Parser origins store ANTLR code-point offsets, so this is the unit every
SQL-text recovery
Review Comment:
Documented in debac86ae83. `QueryContext.startIndex()` / `stopIndex()`
javadoc and the PySpark `QueryContext` docstrings now say these are 0-based
Unicode code-point offsets, with an inclusive stop. Connect proto comments left
for a follow-up as suggested.
##########
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/SQLQueryContextSuite.scala:
##########
@@ -45,5 +45,31 @@ class SQLQueryContextSuite extends SparkFunSuite with
SQLHelper {
).foreach { case ((sqlText, start, stop), expectedFragment) =>
assert(getFragment(sqlText, start, stop) === expectedFragment)
}
+
+ val emoji = new String(Character.toChars(0x1F600))
+ val sql = s"SELECT '$emoji' AS a, DATE 'bad'"
+ val start = sql.codePointCount(0, sql.indexOf("DATE"))
+ val stop = sql.codePointCount(0, sql.length) - 1
+ assert(getFragment(sql, start, stop) === "DATE 'bad'")
+ // A UTF-16 slice of the same indexes would include the comma and drop the
last quote.
Review Comment:
Applied in debac86ae83. The comment now matches the asserted UTF-16 slice
(leading space, dropped closing quote).
##########
sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResult.scala:
##########
@@ -123,9 +134,83 @@ object ParseSqlResult {
JObject(fields.toList)
}
+ /**
+ * 1-based Unicode code-point starts for `statements` in `batch`. Statements
+ * are in ascending UTF-16 start order, so one left-to-right pass is enough:
+ * each count covers only the gap since the previous start.
+ */
+ private def charStarts(
+ batch: String,
+ statements: Seq[PositionedSqlStatement]): Seq[Int] = {
+ var utf16 = 0
+ var codePoints = 0
+ statements.map { segment =>
+ val startUtf16 = math.max(0, math.min(segment.start, batch.length))
+ if (startUtf16 >= utf16) {
+ codePoints += batch.codePointCount(utf16, startUtf16)
+ } else {
+ codePoints = batch.codePointCount(0, startUtf16)
+ }
+ utf16 = startUtf16
+ codePoints + 1
+ }
+ }
+
+ /** Unicode code-point length of the trimmed statement text. */
+ private def charLength(segment: PositionedSqlStatement): Int = {
+ val text = segment.statement
+ text.codePointCount(0, text.length)
+ }
+
+ /**
+ * Recompute the fragment of each statement `queryContext` by code points.
+ * The indexes are parser (code-point) offsets, but [[SQLQueryContext]]
+ * slices its fragment by UTF-16 units, which is off after a supplementary
+ * character. A context that falls outside the statement covers all of it.
+ * Contexts of another object (a view body) are left as is.
+ */
+ private def rewriteErrorCharIndexes(errorObj: JObject, statement: String):
JObject = {
+ JObject(errorObj.obj.map {
+ case ("queryContext", JArray(contexts)) =>
+ "queryContext" -> JArray(contexts.map {
+ case ctx: JObject => rewriteQueryContextIndexes(ctx, statement)
+ case other => other
+ }.toList)
+ case other => other
+ })
+ }
+
+ private def rewriteQueryContextIndexes(ctx: JObject, statement: String):
JObject = {
+ val objectType = ctx.obj.collectFirst { case ("objectType", JString(t)) =>
t }
+ val start = ctx.obj.collectFirst { case ("startIndex", JInt(n)) => n.toInt
}
+ val stop = ctx.obj.collectFirst { case ("stopIndex", JInt(n)) => n.toInt }
+ (objectType, start, stop) match {
+ case (Some(""), Some(from), Some(to)) =>
+ val length = statement.codePointCount(0, statement.length)
+ val (first, last) =
+ if (from >= 1 && to >= from && to <= length) (from, to) else (1,
length)
+ JObject(ctx.obj.map {
+ case ("startIndex", _) => "startIndex" -> JInt(first)
+ case ("stopIndex", _) => "stopIndex" -> JInt(last)
+ case ("fragment", _) => "fragment" ->
JString(codePointSlice(statement, first - 1, last))
Review Comment:
In-range statement-level contexts now keep the shared renderer fragment in
debac86ae83. parse_sql still widens a context that falls outside the statement.
Carrying code-point start/length from the splitter is left as follow-up: the
remaining counts are linear in statement text and small next to the ANTLR
passes.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
