This is an automated email from the ASF dual-hosted git repository.
duanzhengqiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new e3366999b1c Fix can not accept sql type 'TerminalNodeImpl' when
parsing /*! MySQL-specific code */ (#38032)
e3366999b1c is described below
commit e3366999b1c4bb634acdf6a3d488a4036dce4379
Author: Claire <[email protected]>
AuthorDate: Fri Feb 27 16:48:26 2026 +0800
Fix can not accept sql type 'TerminalNodeImpl' when parsing /*!
MySQL-specific code */ (#38032)
* update
* release
* update
* update test
* update checkstyle
* update
* update
* update
* release notes
---
RELEASE-NOTES.md | 1 +
.../sql/parser/engine/core/ParseASTNode.java | 50 ++++++-
.../sql/parser/engine/core/ParseASTNodeTest.java | 165 ++++++++++++++++++++-
.../src/main/antlr4/imports/mysql/Comments.g4 | 4 +-
test/it/parser/src/main/resources/case/dal/set.xml | 12 ++
.../src/main/resources/sql/supported/dal/set.xml | 2 +
6 files changed, 229 insertions(+), 5 deletions(-)
diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index 6c6843091d8..7191eacc30c 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -65,6 +65,7 @@
1. SQL Parser: Fix error parsing \l command SQL statement when front-end
protocol is og - [#37953](https://github.com/apache/shardingsphere/pull/37953)
1. SQL Parser: Fix SQLParsingException when using reserved word `order` in
ORDER BY clause - [#37958](https://github.com/apache/shardingsphere/pull/37958)
1. SQL Parser: Fix parsing error for SQLServer session `SET QUOTED_IDENTIFIER`
and `SET TEXTSIZE` statements -
[#38005](https://github.com/apache/shardingsphere/pull/38005)
+1. SQL Parser: Fix can not accept sql type 'TerminalNodeImpl' when parsing /*!
MySQL-specific code */ -
[#38032](https://github.com/apache/shardingsphere/pull/38032)
1. SQL Parser: Support '2'::int statement in PostgreSQL and openGauss -
[#37962](https://github.com/apache/shardingsphere/pull/37962)
1. SQL Parser: Support range type constructor functions in PostgreSQL without
quotes - [#37994](https://github.com/apache/shardingsphere/pull/37994)
1. SQL Parser: Support parsing MySQL stored procedure syntax-
[#38017](https://github.com/apache/shardingsphere/pull/38017)
diff --git
a/parser/sql/engine/core/src/main/java/org/apache/shardingsphere/sql/parser/engine/core/ParseASTNode.java
b/parser/sql/engine/core/src/main/java/org/apache/shardingsphere/sql/parser/engine/core/ParseASTNode.java
index 569038382a4..b35ab3d507e 100644
---
a/parser/sql/engine/core/src/main/java/org/apache/shardingsphere/sql/parser/engine/core/ParseASTNode.java
+++
b/parser/sql/engine/core/src/main/java/org/apache/shardingsphere/sql/parser/engine/core/ParseASTNode.java
@@ -18,13 +18,17 @@
package org.apache.shardingsphere.sql.parser.engine.core;
import lombok.RequiredArgsConstructor;
+import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.CommonToken;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.tree.ParseTree;
import org.apache.shardingsphere.sql.parser.api.ASTNode;
import java.util.Collection;
import java.util.LinkedList;
+import java.util.List;
/**
* Parse AST node.
@@ -52,11 +56,51 @@ public final class ParseASTNode implements ASTNode {
*/
public Collection<Token> getHiddenTokens() {
Collection<Token> result = new LinkedList<>();
- for (Token each : tokenStream.getTokens()) {
- if (Token.HIDDEN_CHANNEL == each.getChannel()) {
- result.add(each);
+ List<Token> allTokens = tokenStream.getTokens();
+ int mergedEndIndex = -1;
+ for (int i = 0; i < allTokens.size(); i++) {
+ if (i <= mergedEndIndex) {
+ continue;
}
+ Token each = allTokens.get(i);
+ if (Token.HIDDEN_CHANNEL != each.getChannel()) {
+ continue;
+ }
+ if (isExecutableCommentStart(each)) {
+ int endIndex = findExecutableCommentEnd(allTokens, i);
+ if (endIndex > i) {
+ result.add(buildMergedExecutableCommentToken(each,
allTokens.get(endIndex)));
+ mergedEndIndex = endIndex;
+ continue;
+ }
+ }
+ result.add(each);
}
return result;
}
+
+ private boolean isExecutableCommentStart(final Token token) {
+ return token.getText().startsWith("/*!");
+ }
+
+ private int findExecutableCommentEnd(final List<Token> tokens, final int
startIndex) {
+ for (int i = startIndex + 1; i < tokens.size(); i++) {
+ Token each = tokens.get(i);
+ if (Token.HIDDEN_CHANNEL == each.getChannel() &&
"*/".equals(each.getText())) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ private Token buildMergedExecutableCommentToken(final Token startToken,
final Token endToken) {
+ CharStream charStream = tokenStream.getTokenSource().getInputStream();
+ String fullText =
charStream.getText(Interval.of(startToken.getStartIndex(),
endToken.getStopIndex()));
+ CommonToken merged = new CommonToken(startToken.getType());
+ merged.setText(fullText);
+ merged.setStartIndex(startToken.getStartIndex());
+ merged.setStopIndex(endToken.getStopIndex());
+ merged.setChannel(Token.HIDDEN_CHANNEL);
+ return merged;
+ }
}
diff --git
a/parser/sql/engine/core/src/test/java/org/apache/shardingsphere/sql/parser/engine/core/ParseASTNodeTest.java
b/parser/sql/engine/core/src/test/java/org/apache/shardingsphere/sql/parser/engine/core/ParseASTNodeTest.java
index 1dc4c4fced0..f54c0b78713 100644
---
a/parser/sql/engine/core/src/test/java/org/apache/shardingsphere/sql/parser/engine/core/ParseASTNodeTest.java
+++
b/parser/sql/engine/core/src/test/java/org/apache/shardingsphere/sql/parser/engine/core/ParseASTNodeTest.java
@@ -17,12 +17,25 @@
package org.apache.shardingsphere.sql.parser.engine.core;
+import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.CommonToken;
import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.TokenSource;
+import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.tree.ParseTree;
import org.junit.jupiter.api.Test;
-import static org.hamcrest.Matchers.is;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -34,4 +47,154 @@ class ParseASTNodeTest {
when(parseTree.getChild(0)).thenReturn(parseTree);
assertThat(new ParseASTNode(parseTree,
mock(CommonTokenStream.class)).getRootNode(), is(parseTree));
}
+
+ @Test
+ void assertGetHiddenTokensWithEmptyTokenList() {
+ CommonTokenStream tokenStream = mock(CommonTokenStream.class);
+ when(tokenStream.getTokens()).thenReturn(Collections.emptyList());
+ assertTrue(new ParseASTNode(mock(ParseTree.class),
tokenStream).getHiddenTokens().isEmpty());
+ }
+
+ @Test
+ void assertGetHiddenTokensWithNonHiddenTokens() {
+ CommonTokenStream tokenStream = mock(CommonTokenStream.class);
+
when(tokenStream.getTokens()).thenReturn(Collections.singletonList(createToken("SELECT",
Token.DEFAULT_CHANNEL, 0, 5)));
+ assertTrue(new ParseASTNode(mock(ParseTree.class),
tokenStream).getHiddenTokens().isEmpty());
+ }
+
+ @Test
+ void assertGetHiddenTokensWithNonExecutableHiddenToken() {
+ CommonTokenStream tokenStream = mock(CommonTokenStream.class);
+
when(tokenStream.getTokens()).thenReturn(Collections.singletonList(createToken("/*
normal comment */", Token.HIDDEN_CHANNEL, 0, 18)));
+ Collection<Token> actual = new ParseASTNode(mock(ParseTree.class),
tokenStream).getHiddenTokens();
+ assertThat(actual.size(), is(1));
+ assertThat(actual.iterator().next().getText(), is("/* normal comment
*/"));
+ }
+
+ @Test
+ void assertGetHiddenTokensWithUnclosedExecutableComment() {
+ CommonTokenStream tokenStream = mock(CommonTokenStream.class);
+
when(tokenStream.getTokens()).thenReturn(Collections.singletonList(createToken("/*!
SET x=1", Token.HIDDEN_CHANNEL, 0, 10)));
+ Collection<Token> actual = new ParseASTNode(mock(ParseTree.class),
tokenStream).getHiddenTokens();
+ assertThat(actual.size(), is(1));
+ assertThat(actual.iterator().next().getText(), is("/*! SET x=1"));
+ }
+
+ @Test
+ void assertGetHiddenTokensWithEndCommentOnNonHiddenChannel() {
+ CommonTokenStream tokenStream = mock(CommonTokenStream.class);
+ when(tokenStream.getTokens()).thenReturn(Arrays.asList(
+ createToken("/*!", Token.HIDDEN_CHANNEL, 0, 2),
+ createToken("*/", Token.DEFAULT_CHANNEL, 10, 11)));
+ Collection<Token> actual = new ParseASTNode(mock(ParseTree.class),
tokenStream).getHiddenTokens();
+ assertThat(actual.size(), is(1));
+ assertThat(actual.iterator().next().getText(), is("/*!"));
+ }
+
+ @Test
+ void assertGetHiddenTokensWithMergedExecutableComment() {
+ String fullText = "/*! SET GLOBAL max_connections=123 */";
+ CommonTokenStream tokenStream = createMergeCapableTokenStream(
+ Arrays.asList(createToken("/*!", Token.HIDDEN_CHANNEL, 0, 2),
+ createToken("*/", Token.HIDDEN_CHANNEL, 34, 35)),
+ 0, 35, fullText);
+ Collection<Token> actual = new ParseASTNode(mock(ParseTree.class),
tokenStream).getHiddenTokens();
+ assertThat(actual.size(), is(1));
+ Token merged = actual.iterator().next();
+ assertThat(merged.getText(), is(fullText));
+ assertThat(merged.getStartIndex(), is(0));
+ assertThat(merged.getStopIndex(), is(35));
+ assertThat(merged.getChannel(), is(Token.HIDDEN_CHANNEL));
+ }
+
+ @Test
+ void
assertGetHiddenTokensWithIntermediateNonHiddenTokensBetweenExecutableComment() {
+ String fullText = "/*! SET max_connections=123 */";
+ CommonTokenStream tokenStream = createMergeCapableTokenStream(
+ Arrays.asList(createToken("/*!", Token.HIDDEN_CHANNEL, 0, 2),
+ createToken("SET", Token.DEFAULT_CHANNEL, 4, 6),
+ createToken("max_connections=123",
Token.DEFAULT_CHANNEL, 8, 26),
+ createToken("*/", Token.HIDDEN_CHANNEL, 27, 28)),
+ 0, 28, fullText);
+ Collection<Token> actual = new ParseASTNode(mock(ParseTree.class),
tokenStream).getHiddenTokens();
+ assertThat(actual.size(), is(1));
+ assertThat(actual.iterator().next().getText(), is(fullText));
+ }
+
+ @Test
+ void assertGetHiddenTokensWithMixedTokenTypes() {
+ String execFullText = "/*! SET x=1 */";
+ CommonTokenStream tokenStream = createMergeCapableTokenStream(
+ Arrays.asList(createToken("SELECT", Token.DEFAULT_CHANNEL, 0,
5),
+ createToken(" ", Token.HIDDEN_CHANNEL, 6, 6),
+ createToken("/*!", Token.HIDDEN_CHANNEL, 7, 9),
+ createToken("*/", Token.HIDDEN_CHANNEL, 18, 19),
+ createToken("FROM", Token.DEFAULT_CHANNEL, 21, 24)),
+ 7, 19, execFullText);
+ Collection<Token> actual = new ParseASTNode(mock(ParseTree.class),
tokenStream).getHiddenTokens();
+ assertThat(actual.size(), is(2));
+ Iterator<Token> iterator = actual.iterator();
+ assertThat(iterator.next().getText(), is(" "));
+ assertThat(iterator.next().getText(), is(execFullText));
+ }
+
+ @Test
+ void assertGetHiddenTokensWithTwoExecutableComments() {
+ String firstComment = "/*! SET x=1 */";
+ String secondComment = "/*!80029 SET y=2 */";
+ List<Token> tokens = Arrays.asList(
+ createToken("/*!", Token.HIDDEN_CHANNEL, 0, 2),
+ createToken("*/", Token.HIDDEN_CHANNEL, 13, 14),
+ createToken("SELECT", Token.DEFAULT_CHANNEL, 16, 21),
+ createToken("/*!", Token.HIDDEN_CHANNEL, 23, 28),
+ createToken("*/", Token.HIDDEN_CHANNEL, 44, 45));
+ CommonTokenStream tokenStream = mock(CommonTokenStream.class);
+ when(tokenStream.getTokens()).thenReturn(tokens);
+ TokenSource tokenSource = mock(TokenSource.class);
+ when(tokenStream.getTokenSource()).thenReturn(tokenSource);
+ CharStream charStream = mock(CharStream.class);
+ when(tokenSource.getInputStream()).thenReturn(charStream);
+ when(charStream.getText(any(Interval.class))).thenAnswer(invocation ->
{
+ Interval interval = invocation.getArgument(0);
+ if (interval.a == 0 && interval.b == 14) {
+ return firstComment;
+ }
+ if (interval.a == 23 && interval.b == 45) {
+ return secondComment;
+ }
+ return "";
+ });
+ Collection<Token> actual = new ParseASTNode(mock(ParseTree.class),
tokenStream).getHiddenTokens();
+ assertThat(actual.size(), is(2));
+ Iterator<Token> iterator = actual.iterator();
+ Token first = iterator.next();
+ assertThat(first.getText(), is(firstComment));
+ assertThat(first.getStartIndex(), is(0));
+ assertThat(first.getStopIndex(), is(14));
+ assertThat(first.getChannel(), is(Token.HIDDEN_CHANNEL));
+ Token second = iterator.next();
+ assertThat(second.getText(), is(secondComment));
+ assertThat(second.getStartIndex(), is(23));
+ assertThat(second.getStopIndex(), is(45));
+ assertThat(second.getChannel(), is(Token.HIDDEN_CHANNEL));
+ }
+
+ private static CommonToken createToken(final String text, final int
channel, final int startIndex, final int stopIndex) {
+ CommonToken result = new CommonToken(1, text);
+ result.setChannel(channel);
+ result.setStartIndex(startIndex);
+ result.setStopIndex(stopIndex);
+ return result;
+ }
+
+ private static CommonTokenStream createMergeCapableTokenStream(final
java.util.List<Token> tokens, final int intervalStart, final int intervalStop,
final String mergedText) {
+ CommonTokenStream result = mock(CommonTokenStream.class);
+ when(result.getTokens()).thenReturn(tokens);
+ TokenSource tokenSource = mock(TokenSource.class);
+ when(result.getTokenSource()).thenReturn(tokenSource);
+ CharStream charStream = mock(CharStream.class);
+ when(tokenSource.getInputStream()).thenReturn(charStream);
+ when(charStream.getText(Interval.of(intervalStart,
intervalStop))).thenReturn(mergedText);
+ return result;
+ }
}
diff --git
a/parser/sql/engine/dialect/mysql/src/main/antlr4/imports/mysql/Comments.g4
b/parser/sql/engine/dialect/mysql/src/main/antlr4/imports/mysql/Comments.g4
index 006f57e0047..4b132651f9c 100644
--- a/parser/sql/engine/dialect/mysql/src/main/antlr4/imports/mysql/Comments.g4
+++ b/parser/sql/engine/dialect/mysql/src/main/antlr4/imports/mysql/Comments.g4
@@ -19,5 +19,7 @@ lexer grammar Comments;
import Symbol;
-BLOCK_COMMENT: '/*' .*? '*/' -> channel(HIDDEN);
+BLOCK_COMMENT: '/*' { _input.LA(1) != '!' }? .*? '*/' -> channel(HIDDEN);
+EXECUTABLE_COMMENT_START: '/*!' [0-9]* -> channel(HIDDEN);
+EXECUTABLE_COMMENT_END: '*/' -> channel(HIDDEN);
INLINE_COMMENT: (('-- ' | '#') ~[\r\n]* ('\r'? '\n' | EOF) | '--' ('\r'? '\n'
| EOF)) -> channel(HIDDEN);
diff --git a/test/it/parser/src/main/resources/case/dal/set.xml
b/test/it/parser/src/main/resources/case/dal/set.xml
index 3b74d3ce3e4..0ef64b40479 100644
--- a/test/it/parser/src/main/resources/case/dal/set.xml
+++ b/test/it/parser/src/main/resources/case/dal/set.xml
@@ -144,4 +144,16 @@
<parameter name="time_zone" scope="PERSIST_ONLY" />
</parameter-assign>
</set-parameter>
+ <set-parameter sql-case-id="set_mysql_version_comment">
+ <parameter-assign value="123">
+ <parameter name="max_connections" scope="GLOBAL" />
+ </parameter-assign>
+ <comment start-index="0" stop-index="36" text="/*! SET GLOBAL
max_connections=123 */" />
+ </set-parameter>
+ <set-parameter sql-case-id="set_mysql_version_comment_with_version">
+ <parameter-assign value="456">
+ <parameter name="max_connections" scope="GLOBAL" />
+ </parameter-assign>
+ <comment start-index="0" stop-index="41" text="/*!80029 SET GLOBAL
max_connections=456 */" />
+ </set-parameter>
</sql-parser-test-cases>
diff --git a/test/it/parser/src/main/resources/sql/supported/dal/set.xml
b/test/it/parser/src/main/resources/sql/supported/dal/set.xml
index 28b3b750048..0509fc0f747 100644
--- a/test/it/parser/src/main/resources/sql/supported/dal/set.xml
+++ b/test/it/parser/src/main/resources/sql/supported/dal/set.xml
@@ -50,4 +50,6 @@
<sql-case id="set_session_authorization" value="SET SESSION AUTHORIZATION
user1 PASSWORD 'password'" db-types="openGauss" />
<sql-case id="set_persist_system_variable_doris" value="SET PERSIST
max_connections = 200" db-types="Doris" />
<sql-case id="set_persist_only_system_variable_doris" value="SET
PERSIST_ONLY time_zone = '+08:00'" db-types="Doris" />
+ <sql-case id="set_mysql_version_comment" value="/*! SET GLOBAL
max_connections=123 */" db-types="MySQL" />
+ <sql-case id="set_mysql_version_comment_with_version" value="/*!80029 SET
GLOBAL max_connections=456 */" db-types="MySQL" />
</sql-cases>
