Author: rsmith Date: Tue Mar 19 15:09:55 2019 New Revision: 356530 URL: http://llvm.org/viewvc/llvm-project?rev=356530&view=rev Log: Replace tok::angle_string_literal with new tok::header_name.
Use the new kind for both angled header-name tokens and for double-quoted header-name tokens. This is in preparation for C++20's context-sensitive header-name token formation rules. Modified: cfe/trunk/include/clang/Basic/TokenKinds.def cfe/trunk/include/clang/Basic/TokenKinds.h cfe/trunk/include/clang/Lex/Preprocessor.h cfe/trunk/include/clang/Lex/PreprocessorLexer.h cfe/trunk/lib/Lex/Lexer.cpp cfe/trunk/lib/Lex/PPDirectives.cpp cfe/trunk/lib/Lex/PPMacroExpansion.cpp cfe/trunk/lib/Lex/Pragma.cpp cfe/trunk/lib/Lex/Preprocessor.cpp cfe/trunk/test/Preprocessor/_Pragma-dependency.c Modified: cfe/trunk/include/clang/Basic/TokenKinds.def URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Basic/TokenKinds.def?rev=356530&r1=356529&r2=356530&view=diff ============================================================================== --- cfe/trunk/include/clang/Basic/TokenKinds.def (original) +++ cfe/trunk/include/clang/Basic/TokenKinds.def Tue Mar 19 15:09:55 2019 @@ -154,7 +154,9 @@ TOK(utf32_char_constant) // U'a' // C99 6.4.5: String Literals. TOK(string_literal) // "foo" TOK(wide_string_literal) // L"foo" -TOK(angle_string_literal)// <foo> + +// C11 6.4.7: Header Names +TOK(header_name) // <foo>, or "foo" lexed as a header-name // C++11 String Literals. TOK(utf8_string_literal) // u8"foo" Modified: cfe/trunk/include/clang/Basic/TokenKinds.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Basic/TokenKinds.h?rev=356530&r1=356529&r2=356530&view=diff ============================================================================== --- cfe/trunk/include/clang/Basic/TokenKinds.h (original) +++ cfe/trunk/include/clang/Basic/TokenKinds.h Tue Mar 19 15:09:55 2019 @@ -86,7 +86,7 @@ inline bool isLiteral(TokenKind K) { return K == tok::numeric_constant || K == tok::char_constant || K == tok::wide_char_constant || K == tok::utf8_char_constant || K == tok::utf16_char_constant || K == tok::utf32_char_constant || - isStringLiteral(K) || K == tok::angle_string_literal; + isStringLiteral(K) || K == tok::header_name; } /// Return true if this is any of tok::annot_* kinds. Modified: cfe/trunk/include/clang/Lex/Preprocessor.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/Preprocessor.h?rev=356530&r1=356529&r2=356530&view=diff ============================================================================== --- cfe/trunk/include/clang/Lex/Preprocessor.h (original) +++ cfe/trunk/include/clang/Lex/Preprocessor.h Tue Mar 19 15:09:55 2019 @@ -1274,7 +1274,7 @@ public: void Lex(Token &Result); /// Lex a token, forming a header-name token if possible. - bool LexHeaderName(Token &Result, bool AllowConcatenation = true); + bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true); void LexAfterModuleImport(Token &Result); Modified: cfe/trunk/include/clang/Lex/PreprocessorLexer.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/PreprocessorLexer.h?rev=356530&r1=356529&r2=356530&view=diff ============================================================================== --- cfe/trunk/include/clang/Lex/PreprocessorLexer.h (original) +++ cfe/trunk/include/clang/Lex/PreprocessorLexer.h Tue Mar 19 15:09:55 2019 @@ -48,8 +48,7 @@ protected: /// True when parsing \#XXX; turns '\\n' into a tok::eod token. bool ParsingPreprocessorDirective = false; - /// True after \#include; turns \<xx> into a tok::angle_string_literal - /// token. + /// True after \#include; turns \<xx> or "xxx" into a tok::header_name token. bool ParsingFilename = false; /// True if in raw mode. Modified: cfe/trunk/lib/Lex/Lexer.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Lexer.cpp?rev=356530&r1=356529&r2=356530&view=diff ============================================================================== --- cfe/trunk/lib/Lex/Lexer.cpp (original) +++ cfe/trunk/lib/Lex/Lexer.cpp Tue Mar 19 15:09:55 2019 @@ -2072,7 +2072,7 @@ bool Lexer::LexAngledStringLiteral(Token // Update the location of token as well as BufferPtr. const char *TokStart = BufferPtr; - FormTokenWithChars(Result, CurPtr, tok::angle_string_literal); + FormTokenWithChars(Result, CurPtr, tok::header_name); Result.setLiteralData(TokStart); return true; } @@ -3465,7 +3465,9 @@ LexNextToken: case '"': // Notify MIOpt that we read a non-whitespace/non-comment token. MIOpt.ReadToken(); - return LexStringLiteral(Result, CurPtr, tok::string_literal); + return LexStringLiteral(Result, CurPtr, + ParsingFilename ? tok::header_name + : tok::string_literal); // C99 6.4.6: Punctuators. case '?': Modified: cfe/trunk/lib/Lex/PPDirectives.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/PPDirectives.cpp?rev=356530&r1=356529&r2=356530&view=diff ============================================================================== --- cfe/trunk/lib/Lex/PPDirectives.cpp (original) +++ cfe/trunk/lib/Lex/PPDirectives.cpp Tue Mar 19 15:09:55 2019 @@ -1446,6 +1446,14 @@ bool Preprocessor::GetIncludeFilenameSpe // Get the text form of the filename. assert(!Buffer.empty() && "Can't have tokens with empty spellings!"); + // FIXME: Consider warning on some of the cases described in C11 6.4.7/3 and + // C++20 [lex.header]/2: + // + // If `"`, `'`, `\`, `/*`, or `//` appears in a header-name, then + // in C: behavior is undefined + // in C++: program is conditionally-supported with implementation-defined + // semantics + // Make sure the filename is <x> or "x". bool isAngled; if (Buffer[0] == '<') { @@ -1613,7 +1621,7 @@ void Preprocessor::HandleIncludeDirectiv if (LexHeaderName(FilenameTok)) return; - if (!FilenameTok.isOneOf(tok::angle_string_literal, tok::string_literal)) { + if (FilenameTok.isNot(tok::header_name)) { Diag(FilenameTok.getLocation(), diag::err_pp_expects_filename); if (FilenameTok.isNot(tok::eod)) DiscardUntilEndOfDirective(); Modified: cfe/trunk/lib/Lex/PPMacroExpansion.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/PPMacroExpansion.cpp?rev=356530&r1=356529&r2=356530&view=diff ============================================================================== --- cfe/trunk/lib/Lex/PPMacroExpansion.cpp (original) +++ cfe/trunk/lib/Lex/PPMacroExpansion.cpp Tue Mar 19 15:09:55 2019 @@ -1166,7 +1166,7 @@ static bool EvaluateHasIncludeCommon(Tok PP.Diag(LParenLoc, diag::err_pp_expected_after) << II << tok::l_paren; // If the next token looks like a filename or the start of one, // assume it is and process it as such. - if (!Tok.is(tok::angle_string_literal) && !Tok.is(tok::string_literal)) + if (Tok.isNot(tok::header_name)) return false; } else { // Save '(' location for possible missing ')' message. @@ -1175,7 +1175,7 @@ static bool EvaluateHasIncludeCommon(Tok return false; } - if (!Tok.isOneOf(tok::angle_string_literal, tok::string_literal)) { + if (Tok.isNot(tok::header_name)) { PP.Diag(Tok.getLocation(), diag::err_pp_expects_filename); return false; } Modified: cfe/trunk/lib/Lex/Pragma.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Pragma.cpp?rev=356530&r1=356529&r2=356530&view=diff ============================================================================== --- cfe/trunk/lib/Lex/Pragma.cpp (original) +++ cfe/trunk/lib/Lex/Pragma.cpp Tue Mar 19 15:09:55 2019 @@ -486,7 +486,7 @@ void Preprocessor::HandlePragmaDependenc return; // If the next token wasn't a header-name, diagnose the error. - if (!FilenameTok.isOneOf(tok::angle_string_literal, tok::string_literal)) { + if (FilenameTok.isNot(tok::header_name)) { Diag(FilenameTok.getLocation(), diag::err_pp_expects_filename); return; } @@ -670,8 +670,7 @@ void Preprocessor::HandlePragmaIncludeAl StringRef SourceFileName; SmallString<128> FileNameBuffer; - if (SourceFilenameTok.is(tok::string_literal) || - SourceFilenameTok.is(tok::angle_string_literal)) { + if (SourceFilenameTok.is(tok::header_name)) { SourceFileName = getSpelling(SourceFilenameTok, FileNameBuffer); } else { Diag(Tok, diag::warn_pragma_include_alias_expected_filename); @@ -691,8 +690,7 @@ void Preprocessor::HandlePragmaIncludeAl return; StringRef ReplaceFileName; - if (ReplaceFilenameTok.is(tok::string_literal) || - ReplaceFilenameTok.is(tok::angle_string_literal)) { + if (ReplaceFilenameTok.is(tok::header_name)) { ReplaceFileName = getSpelling(ReplaceFilenameTok, FileNameBuffer); } else { Diag(Tok, diag::warn_pragma_include_alias_expected_filename); Modified: cfe/trunk/lib/Lex/Preprocessor.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Preprocessor.cpp?rev=356530&r1=356529&r2=356530&view=diff ============================================================================== --- cfe/trunk/lib/Lex/Preprocessor.cpp (original) +++ cfe/trunk/lib/Lex/Preprocessor.cpp Tue Mar 19 15:09:55 2019 @@ -901,11 +901,12 @@ void Preprocessor::Lex(Token &Result) { /// \param FilenameTok Filled in with the next token. On success, this will /// be either an angle_header_name or a string_literal token. On /// failure, it will be whatever other token was found instead. -/// \param AllowConcatenation If \c true, allow a < token, followed by other -/// tokens and finally a > token, to form a single header-name token. +/// \param AllowMacroExpansion If \c true, allow the header name to be formed +/// by macro expansion (concatenating tokens as necessary if the first +/// token is a '<'). /// \return \c true if we reached EOD or EOF while looking for a > token in /// a concatenated header name and diagnosed it. \c false otherwise. -bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowConcatenation) { +bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) { // Lex using header-name tokenization rules if tokens are being lexed from // a file. Just grab a token normally if we're in a macro expansion. if (CurPPLexer) @@ -915,13 +916,16 @@ bool Preprocessor::LexHeaderName(Token & // This could be a <foo/bar.h> file coming from a macro expansion. In this // case, glue the tokens together into an angle_string_literal token. - if (FilenameTok.is(tok::less) && AllowConcatenation) { - SmallString<128> FilenameBuffer; + SmallString<128> FilenameBuffer; + if (FilenameTok.is(tok::less) && AllowMacroExpansion) { SourceLocation Start = FilenameTok.getLocation(); SourceLocation End; FilenameBuffer.push_back('<'); // Consume tokens until we find a '>'. + // FIXME: A header-name could be formed starting or ending with an + // alternative token. It's not clear whether that's ill-formed in all + // cases. while (FilenameTok.isNot(tok::greater)) { Lex(FilenameTok); if (FilenameTok.isOneOf(tok::eod, tok::eof)) { @@ -962,8 +966,22 @@ bool Preprocessor::LexHeaderName(Token & } FilenameTok.startToken(); - FilenameTok.setKind(tok::angle_string_literal); + FilenameTok.setKind(tok::header_name); CreateString(FilenameBuffer, FilenameTok, Start, End); + } else if (FilenameTok.is(tok::string_literal) && AllowMacroExpansion) { + // Convert a string-literal token of the form " h-char-sequence " + // (produced by macro expansion) into a header-name token. + // + // The rules for header-names don't quite match the rules for + // string-literals, but all the places where they differ result in + // undefined behavior, so we can and do treat them the same. + // + // A string-literal with a prefix or suffix is not translated into a + // header-name. This could theoretically be observable via the C++20 + // context-sensitive header-name formation rules. + StringRef Str = getSpelling(FilenameTok, FilenameBuffer); + if (Str.size() >= 2 && Str.front() == '"' && Str.back() == '"') + FilenameTok.setKind(tok::header_name); } return false; Modified: cfe/trunk/test/Preprocessor/_Pragma-dependency.c URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Preprocessor/_Pragma-dependency.c?rev=356530&r1=356529&r2=356530&view=diff ============================================================================== --- cfe/trunk/test/Preprocessor/_Pragma-dependency.c (original) +++ cfe/trunk/test/Preprocessor/_Pragma-dependency.c Tue Mar 19 15:09:55 2019 @@ -1,5 +1,11 @@ // RUN: %clang_cc1 -E -verify %s +#pragma GCC dependency "./_Pragma-dependency.c" + +#define self "./_Pragma-dependency.c" +// expected-error@+1 {{expected "FILENAME" or <FILENAME>}} +#pragma GCC dependency self + #define DO_PRAGMA _Pragma #define STR "GCC dependency \"parse.y\"") // expected-error@+1 {{'parse.y' file not found}} _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits