hokein updated this revision to Diff 415377.
hokein added a comment.
define a greatergreater nonterminal per discussion.
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D121678/new/
https://reviews.llvm.org/D121678
Files:
clang/include/clang/Tooling/Syntax/Pseudo/Token.h
clang/lib/Tooling/Syntax/Pseudo/Token.cpp
clang/lib/Tooling/Syntax/Pseudo/cxx.bnf
clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp
Index: clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp
===================================================================
--- clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp
+++ clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp
@@ -172,6 +172,30 @@
}));
}
+TEST(TokenTest, SplitGreaterGreater) {
+ LangOptions Opts;
+ std::string Code = R"cpp(
+>> // split
+// >> with an escaped newline in the middle, split
+>\
+>
+>>= // not split
+)cpp";
+ TokenStream Raw = stripComments(cook(lex(Code, Opts), Opts));
+ EXPECT_THAT(Raw.tokens(),
+ ElementsAreArray({token(">>", tok::greatergreater),
+ token(">>", tok::greatergreater),
+ token(">>=", tok::greatergreaterequal)}));
+ TokenStream Split = splitGreaterGreater(Raw);
+ EXPECT_THAT(Split.tokens(), ElementsAreArray({
+ token(">", tok::greater),
+ token(">", tok::greater),
+ token(">", tok::greater),
+ token(">", tok::greater),
+ token(">>=", tok::greatergreaterequal),
+ }));
+}
+
TEST(TokenTest, DropComments) {
LangOptions Opts;
std::string Code = R"cpp(
Index: clang/lib/Tooling/Syntax/Pseudo/cxx.bnf
===================================================================
--- clang/lib/Tooling/Syntax/Pseudo/cxx.bnf
+++ clang/lib/Tooling/Syntax/Pseudo/cxx.bnf
@@ -13,6 +13,9 @@
# - the file merely describes the core C++ grammar. Preprocessor directives and
# lexical conversions are omitted as we reuse clang's lexer and run a fake
# preprocessor;
+# - grammar rules with the >> token are adjusted, the greatergreater token is
+# split into two > tokens, to make the GLR parser aware of nested templates
+# and right shift operator.
#
# Guidelines:
# - non-terminals are lower_case; terminals (aka tokens) correspond to
@@ -96,7 +99,7 @@
fold-operator := ^
fold-operator := |
fold-operator := <<
-fold-operator := >>
+fold-operator := greatergreater
fold-operator := +=
fold-operator := -=
fold-operator := *=
@@ -202,7 +205,7 @@
# expr.shift
shift-expression := additive-expression
shift-expression := shift-expression << additive-expression
-shift-expression := shift-expression >> additive-expression
+shift-expression := shift-expression greatergreater additive-expression
# expr.spaceship
compare-expression := shift-expression
compare-expression := compare-expression <=> shift-expression
@@ -615,7 +618,7 @@
operator-name := ^^
operator-name := ||
operator-name := <<
-operator-name := >>
+operator-name := greatergreater
operator-name := <<=
operator-name := >>=
operator-name := ++
@@ -737,3 +740,8 @@
module-keyword := IDENTIFIER
import-keyword := IDENTIFIER
export-keyword := IDENTIFIER
+
+#! Greatergreater token -- clang lexer always lexes it as a single token, we
+#! split it into two tokens to make the GLR parser aware of the nested-template
+#! case.
+greatergreater := > >
Index: clang/lib/Tooling/Syntax/Pseudo/Token.cpp
===================================================================
--- clang/lib/Tooling/Syntax/Pseudo/Token.cpp
+++ clang/lib/Tooling/Syntax/Pseudo/Token.cpp
@@ -7,6 +7,7 @@
//===----------------------------------------------------------------------===//
#include "clang/Tooling/Syntax/Pseudo/Token.h"
+#include "clang/Basic/TokenKinds.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/Support/Format.h"
#include "llvm/Support/FormatVariadic.h"
@@ -93,6 +94,29 @@
OS << '\n';
}
+TokenStream splitGreaterGreater(const TokenStream &Input) {
+ TokenStream Out;
+ for (Token T : Input.tokens()) {
+ // FIXME: split lessless token to support Cuda's triple angle brackets <<<.
+ if (T.Kind == tok::greatergreater) {
+ if (T.text() == ">>") {
+ T.Kind = tok::greater;
+ T.Length = 1;
+ Out.push(T);
+ T.Data = T.text().data() + 1;
+ // FIXME: Line is wrong if the first greater is followed by an escaped
+ // newline.
+ Out.push(T);
+ continue;
+ }
+ assert(false && "split an uncook token stream!");
+ }
+ Out.push(std::move(T));
+ }
+ Out.finalize();
+ return Out;
+}
+
TokenStream stripComments(const TokenStream &Input) {
TokenStream Out;
for (const Token &T : Input.tokens()) {
Index: clang/include/clang/Tooling/Syntax/Pseudo/Token.h
===================================================================
--- clang/include/clang/Tooling/Syntax/Pseudo/Token.h
+++ clang/include/clang/Tooling/Syntax/Pseudo/Token.h
@@ -195,6 +195,11 @@
/// (And having cooked token kinds in PP-disabled sections is useful for us).
TokenStream cook(const TokenStream &, const clang::LangOptions &);
+/// Derives a token stream by splitting the greatergreater token.
+///
+/// The greatergreater >> token is split into two greater tokens.
+TokenStream splitGreaterGreater(const TokenStream &);
+
/// Drops comment tokens.
TokenStream stripComments(const TokenStream &);
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
