vinodkc commented on code in PR #58024:
URL: https://github.com/apache/spark/pull/58024#discussion_r4392482699
##########
sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4:
##########
@@ -80,10 +80,52 @@ import java.util.Deque;
public int complex_type_level_counter = 0;
/**
- * Increase the counter by one when hits KEYWORD 'ARRAY', 'MAP', 'STRUCT'.
+ * Increase the counter by one when hits KEYWORD 'ARRAY', 'MAP', 'STRUCT'
that starts an
+ * angle-bracketed complex type, such as ARRAY<INT>. Other uses of these
keywords, such as the
+ * ARRAY(1, 2) function or the `IS JSON ARRAY` predicate, have no matching
'>' and must leave
+ * the counter unchanged, otherwise a later '>>' could not be lexed as a
shift operator.
*/
public void incComplexTypeLevelCounter() {
- complex_type_level_counter++;
+ if (isFollowedByLessThan()) complex_type_level_counter++;
+ }
+
+ /**
+ * Returns whether the next character after the current token, skipping
whitespace and
+ * comments, is '<'. Called from a lexer action, where the input is
positioned right after the
+ * matched keyword.
+ */
+ private boolean isFollowedByLessThan() {
+ int i = 1;
+ while (true) {
+ int c = _input.LA(i);
+ if (c != CharStream.EOF && (Character.isWhitespace(c) ||
Character.isSpaceChar(c))) {
+ // Covers every character of the WS rule, including no-break spaces.
+ i++;
+ } else if (c == '-' && _input.LA(i + 1) == '-') {
+ // Simple comment: skip to the end of the line, honoring backslash
line continuations.
+ i += 2;
+ while (_input.LA(i) != '\n' && _input.LA(i) != CharStream.EOF) {
Review Comment:
fixed; the lookahead now stops at a bare CR the same way the lexer's comment
rule does.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
