This is an automated email from the ASF dual-hosted git repository. benweidig pushed a commit to branch property-expression-parser in repository https://gitbox.apache.org/repos/asf/tapestry-5.git
commit 578cef219792a19d45052b0c1a706faccc561139 Author: Ben Weidig <[email protected]> AuthorDate: Sun Oct 25 12:33:53 2026 +0100 TAP5-2806: Keep whitespace handling in the lexer grammar only BaseLexer skipped whitespace itself before scanning an identifier, because the generated lexer would otherwise skip it and then fail on the identifier behind it. Whitespace was therefore defined twice. Whitespace is now a hidden-channel token instead of being skipped. The generated lexer returns after it, BaseLexer checks for an identifier on every call, and the parser's token stream drops the hidden token. --- .../beanmodel/internal/antlr/PropertyExpressionLexer.g | 7 ++++++- .../tapestry5/beanmodel/internal/antlr/BaseLexer.java | 13 +------------ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/beanmodel/src/main/antlr/org/apache/tapestry5/beanmodel/internal/antlr/PropertyExpressionLexer.g b/beanmodel/src/main/antlr/org/apache/tapestry5/beanmodel/internal/antlr/PropertyExpressionLexer.g index 0d7cea78e..c21e1ed77 100644 --- a/beanmodel/src/main/antlr/org/apache/tapestry5/beanmodel/internal/antlr/PropertyExpressionLexer.g +++ b/beanmodel/src/main/antlr/org/apache/tapestry5/beanmodel/internal/antlr/PropertyExpressionLexer.g @@ -64,6 +64,8 @@ fragment QUOTE // Identifiers and the case-insensitive keywords are scanned by BaseLexer.nextToken(), // because their characters are defined by Character.isJavaIdentifierStart/Part. // These rules only provide the token types. +// BaseLexer takes every character that can start an identifier before the rules below see it. +// A new keyword, or any token starting with such a character, therefore belongs in BaseLexer. fragment NULL : {this.getClass(); /* Fix java.lang.VerifyError: Stack size too large */}; @@ -82,7 +84,10 @@ fragment IDENTIFIER SAFEDEREF : '?.'; -WS : (' '|'\t'|'\n'|'\r')+ { skip(); }; +// Whitespace is emitted on the hidden channel rather than skipped, so that the generated lexer returns +// after it and BaseLexer.nextToken() can check for an identifier behind it. The parser never sees it. + +WS : (' '|'\t'|'\n'|'\r')+ { $channel = HIDDEN; }; // Literal strings are always inside single quotes. diff --git a/beanmodel/src/main/java/org/apache/tapestry5/beanmodel/internal/antlr/BaseLexer.java b/beanmodel/src/main/java/org/apache/tapestry5/beanmodel/internal/antlr/BaseLexer.java index a8e0d1c0c..36e0c5665 100644 --- a/beanmodel/src/main/java/org/apache/tapestry5/beanmodel/internal/antlr/BaseLexer.java +++ b/beanmodel/src/main/java/org/apache/tapestry5/beanmodel/internal/antlr/BaseLexer.java @@ -49,17 +49,11 @@ public abstract class BaseLexer extends Lexer * Scans identifiers and keywords here, and leaves every other token to the generated lexer. * An identifier is a Java identifier without identifier-ignorable characters, read by code point. * A keyword is an identifier that matches NULL, TRUE, FALSE or THIS, ignoring ASCII case. + * Whitespace comes from the generated lexer as a hidden token, so each call starts on a token boundary. */ @Override public Token nextToken() { - // Whitespace has to go first, or the generated lexer would skip it and then read the identifier behind it. - - while (isWhitespace(input.LA(1))) - { - input.consume(); - } - if (!isIdentifierStart(codePointAt(1))) { return super.nextToken(); @@ -82,11 +76,6 @@ public abstract class BaseLexer extends Lexer return emit(); } - private static boolean isWhitespace(int c) - { - return c == ' ' || c == '\t' || c == '\n' || c == '\r'; - } - private static boolean isIdentifierStart(int codePoint) { return Character.isJavaIdentifierStart(codePoint) && !Character.isIdentifierIgnorable(codePoint);
