Author: Manuel Klimek Date: 2023-01-31T16:06:46Z New Revision: 04ed86ff1b7272faf8e62fa32da4acaa2d3c6add
URL: https://github.com/llvm/llvm-project/commit/04ed86ff1b7272faf8e62fa32da4acaa2d3c6add DIFF: https://github.com/llvm/llvm-project/commit/04ed86ff1b7272faf8e62fa32da4acaa2d3c6add.diff LOG: [clang-format][NFC] Bring FormatTokenSource under test. Add tests for FormatTokenSource and harden it against edge cases. Added: clang/unittests/Format/FormatTokenSourceTest.cpp Modified: clang/lib/Format/FormatTokenSource.h clang/unittests/Format/CMakeLists.txt Removed: ################################################################################ diff --git a/clang/lib/Format/FormatTokenSource.h b/clang/lib/Format/FormatTokenSource.h index e14f72b7d1f44..359e0e52878bb 100644 --- a/clang/lib/Format/FormatTokenSource.h +++ b/clang/lib/Format/FormatTokenSource.h @@ -77,7 +77,9 @@ class IndexedTokenSource : public FormatTokenSource { return Position > 0 ? Tokens[Position - 1] : nullptr; } - FormatToken *peekNextToken(bool SkipComment) override { + FormatToken *peekNextToken(bool SkipComment = false) override { + if (isEOF()) + return Tokens[Position]; int Next = Position + 1; if (SkipComment) while (Tokens[Next]->is(tok::comment)) @@ -89,7 +91,9 @@ class IndexedTokenSource : public FormatTokenSource { return Tokens[Next]; } - bool isEOF() override { return Tokens[Position]->is(tok::eof); } + bool isEOF() override { + return Position == -1 ? false : Tokens[Position]->is(tok::eof); + } unsigned getPosition() override { LLVM_DEBUG(llvm::dbgs() << "Getting Position: " << Position << "\n"); diff --git a/clang/unittests/Format/CMakeLists.txt b/clang/unittests/Format/CMakeLists.txt index 09457c105ca30..1db1f602415bb 100644 --- a/clang/unittests/Format/CMakeLists.txt +++ b/clang/unittests/Format/CMakeLists.txt @@ -21,6 +21,7 @@ add_clang_unittest(FormatTests FormatTestTableGen.cpp FormatTestTextProto.cpp FormatTestVerilog.cpp + FormatTokenSourceTest.cpp IntegerLiteralSeparatorTest.cpp MacroCallReconstructorTest.cpp MacroExpanderTest.cpp diff --git a/clang/unittests/Format/FormatTokenSourceTest.cpp b/clang/unittests/Format/FormatTokenSourceTest.cpp new file mode 100644 index 0000000000000..3c2f317adf5c3 --- /dev/null +++ b/clang/unittests/Format/FormatTokenSourceTest.cpp @@ -0,0 +1,75 @@ +//===- unittest/Format/FormatTokenSourceTest.cpp --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "../../lib/Format/FormatTokenSource.h" +#include "TestLexer.h" +#include "clang/Basic/TokenKinds.h" +#include "gtest/gtest.h" + +namespace clang { +namespace format { +namespace { + +class IndexedTokenSourceTest : public ::testing::Test { +protected: + TokenList lex(llvm::StringRef Code, + const FormatStyle &Style = getLLVMStyle()) { + return TestLexer(Allocator, Buffers, Style).lex(Code); + } + llvm::SpecificBumpPtrAllocator<FormatToken> Allocator; + std::vector<std::unique_ptr<llvm::MemoryBuffer>> Buffers; +}; + +#define EXPECT_TOKEN_KIND(FormatTok, Kind) \ + do { \ + FormatToken *Tok = FormatTok; \ + EXPECT_EQ((Tok)->Tok.getKind(), Kind) << *(Tok); \ + } while (false); + +TEST_F(IndexedTokenSourceTest, EmptyInput) { + IndexedTokenSource Source(lex("")); + EXPECT_FALSE(Source.isEOF()); + EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof); + EXPECT_TRUE(Source.isEOF()); + EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof); + EXPECT_TRUE(Source.isEOF()); + EXPECT_TOKEN_KIND(Source.peekNextToken(/*SkipComment=*/false), tok::eof); + EXPECT_TOKEN_KIND(Source.peekNextToken(/*SkipComment=*/true), tok::eof); + EXPECT_EQ(Source.getPreviousToken(), nullptr); + EXPECT_TRUE(Source.isEOF()); +} + +TEST_F(IndexedTokenSourceTest, NavigateTokenStream) { + IndexedTokenSource Source(lex("int a;")); + EXPECT_TOKEN_KIND(Source.peekNextToken(), tok::kw_int); + EXPECT_TOKEN_KIND(Source.getNextToken(), tok::kw_int); + EXPECT_EQ(Source.getPreviousToken(), nullptr); + EXPECT_TOKEN_KIND(Source.peekNextToken(), tok::identifier); + EXPECT_TOKEN_KIND(Source.getNextToken(), tok::identifier); + EXPECT_TOKEN_KIND(Source.getPreviousToken(), tok::kw_int); + EXPECT_TOKEN_KIND(Source.peekNextToken(), tok::semi); + EXPECT_TOKEN_KIND(Source.getNextToken(), tok::semi); + EXPECT_TOKEN_KIND(Source.getPreviousToken(), tok::identifier); + EXPECT_TOKEN_KIND(Source.peekNextToken(), tok::eof); + EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof); + EXPECT_TOKEN_KIND(Source.getPreviousToken(), tok::semi); +} + +TEST_F(IndexedTokenSourceTest, ResetPosition) { + IndexedTokenSource Source(lex("int a;")); + Source.getNextToken(); + unsigned Position = Source.getPosition(); + Source.getNextToken(); + Source.getNextToken(); + EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof); + EXPECT_TOKEN_KIND(Source.setPosition(Position), tok::kw_int); +} + +} // namespace +} // namespace format +} // namespace clang \ No newline at end of file _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits