sammccall created this revision. sammccall added reviewers: kadircet, adamcz. Herald added subscribers: cfe-commits, usaxena95, arphaman, jkorous, MaskRay, ilya-biryukov, mgorny. Herald added a project: clang.
This is a piece from the design in https://tinyurl.com/clangd-config https://reviews.llvm.org/D82335 is a draft with all the bits linked together. It doesn't do anything yet, it's just a library + tests. It deliberately implements a minimal set of actual configuration options. Repository: rG LLVM Github Monorepo https://reviews.llvm.org/D82386 Files: clang-tools-extra/clangd/CMakeLists.txt clang-tools-extra/clangd/ConfigFragment.h clang-tools-extra/clangd/ConfigYAML.cpp clang-tools-extra/clangd/unittests/CMakeLists.txt clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp
Index: clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp =================================================================== --- /dev/null +++ clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp @@ -0,0 +1,154 @@ +//===-- ConfigYAMLTests.cpp -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "Annotations.h" +#include "ConfigFragment.h" +#include "Matchers.h" +#include "Protocol.h" +#include "llvm/Support/SMLoc.h" +#include "llvm/Support/SourceMgr.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "gtest/internal/gtest-internal.h" + +namespace clang { +namespace clangd { +namespace config { +template <typename T> void PrintTo(const Located<T> &V, std::ostream *OS) { + *OS << ::testing::PrintToString(*V); +} + +namespace { +using ::testing::ElementsAre; +using ::testing::IsEmpty; + +MATCHER_P(Val, Value, "") { + if (*arg == Value) + return true; + *result_listener << "value is " << *arg; + return false; +} + +Position toPosition(llvm::SMLoc L, const llvm::SourceMgr &SM) { + auto LineCol = SM.getLineAndColumn(L); + Position P; + P.line = LineCol.first - 1; + P.character = LineCol.second - 1; + return P; +} + +Range toRange(llvm::SMRange R, const llvm::SourceMgr &SM) { + return Range{toPosition(R.Start, SM), toPosition(R.End, SM)}; +} + +struct CapturedDiags { + std::function<void(const llvm::SMDiagnostic &)> callback() { + return [this](const llvm::SMDiagnostic &D) { + Diagnostics.emplace_back(); + Diag &Out = Diagnostics.back(); + Out.Message = D.getMessage().str(); + Out.Kind = D.getKind(); + Out.Pos.line = D.getLineNo() - 1; + Out.Pos.character = D.getColumnNo(); // Zero-based - bug in SourceMgr? + if (!D.getRanges().empty()) { + const auto &R = D.getRanges().front(); + Out.Range.emplace(); + Out.Range->start.line = Out.Range->end.line = Out.Pos.line; + Out.Range->start.character = R.first; + Out.Range->end.character = R.second; + } + }; + } + struct Diag { + std::string Message; + llvm::SourceMgr::DiagKind Kind; + Position Pos; + llvm::Optional<Range> Range; + + friend void PrintTo(const Diag &D, std::ostream *OS) { *OS << D.Message; } + }; + std::vector<Diag> Diagnostics; +}; + +MATCHER_P(DiagMessage, M, "") { return arg.Message == M; } + +TEST(ParseYAML, SemanticForms) { + CapturedDiags Diags; + const char *YAML = R"yaml( +If: + PathMatch: + - 'abc' +CompileFlags: { Add: [foo, bar] } +--- +CompileFlags: + Add: baz + )yaml"; + auto Results = Fragment::parseYAML(YAML, "config.yaml", Diags.callback()); + EXPECT_THAT(Diags.Diagnostics, IsEmpty()); + ASSERT_EQ(Results.size(), 2u); + EXPECT_FALSE(Results.front().Condition->UnrecognizedCondition); + EXPECT_THAT(Results.front().Condition->PathMatch, ElementsAre(Val("abc"))); + EXPECT_THAT(Results.front().CompileFlags.Add, + ElementsAre(Val("foo"), Val("bar"))); + + EXPECT_FALSE(Results.back().Condition); + EXPECT_THAT(Results.back().CompileFlags.Add, ElementsAre(Val("baz"))); +} + +TEST(ParseYAML, Locations) { + CapturedDiags Diags; + Annotations YAML(R"yaml( +If: + PathMatch: [['???bad***regex(((']] + )yaml"); + auto Results = + Fragment::parseYAML(YAML.code(), "config.yaml", Diags.callback()); + EXPECT_THAT(Diags.Diagnostics, IsEmpty()); + ASSERT_EQ(Results.size(), 1u); + ASSERT_NE(Results.front().Source.Manager, nullptr); + EXPECT_EQ(toRange(Results.front().Condition->PathMatch.front().Range, + *Results.front().Source.Manager), + YAML.range()); +} + +TEST(ParseYAML, Diagnostics) { + CapturedDiags Diags; + Annotations YAML(R"yaml( +If: + [[UnknownCondition]]: "foo" +CompileFlags: + Add: 'first' +--- +CompileFlags: {^ +)yaml"); + auto Results = + Fragment::parseYAML(YAML.code(), "config.yaml", Diags.callback()); + + ASSERT_THAT(Diags.Diagnostics, + ElementsAre(DiagMessage("Unknown Condition key UnknownCondition"), + DiagMessage("Unexpected token. Expected Key, Flow " + "Entry, or Flow Mapping End."))); + + EXPECT_EQ(Diags.Diagnostics.front().Kind, llvm::SourceMgr::DK_Warning); + EXPECT_EQ(Diags.Diagnostics.front().Pos, YAML.range().start); + EXPECT_THAT(Diags.Diagnostics.front().Range, HasValue(YAML.range())); + + EXPECT_EQ(Diags.Diagnostics.back().Kind, llvm::SourceMgr::DK_Error); + EXPECT_EQ(Diags.Diagnostics.back().Pos, YAML.point()); + EXPECT_EQ(Diags.Diagnostics.back().Range, llvm::None); + + ASSERT_EQ(Results.size(), 2u); + EXPECT_THAT(Results.front().CompileFlags.Add, ElementsAre(Val("first"))); + EXPECT_TRUE(Results.front().Condition->UnrecognizedCondition); + EXPECT_THAT(Results.back().CompileFlags.Add, IsEmpty()); +} + +} // namespace +} // namespace config +} // namespace clangd +} // namespace clang Index: clang-tools-extra/clangd/unittests/CMakeLists.txt =================================================================== --- clang-tools-extra/clangd/unittests/CMakeLists.txt +++ clang-tools-extra/clangd/unittests/CMakeLists.txt @@ -41,6 +41,7 @@ CollectMacrosTests.cpp CompileCommandsTests.cpp CompilerTests.cpp + ConfigYAMLTests.cpp DexTests.cpp DiagnosticsTests.cpp DraftStoreTests.cpp Index: clang-tools-extra/clangd/ConfigYAML.cpp =================================================================== --- /dev/null +++ clang-tools-extra/clangd/ConfigYAML.cpp @@ -0,0 +1,208 @@ +//===--- ConfigYAML.cpp - Loading configuration fragments from YAML files -===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "ConfigFragment.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/YAMLParser.h" +#include <system_error> + +namespace clang { +namespace clangd { +namespace config { +namespace { +using llvm::yaml::BlockScalarNode; +using llvm::yaml::MappingNode; +using llvm::yaml::Node; +using llvm::yaml::ScalarNode; +using llvm::yaml::SequenceNode; + +class Parser { + llvm::SourceMgr &SM; + llvm::SmallString<256> Buf; + +public: + Parser(llvm::SourceMgr &SM) : SM(SM) {} + + // Tries to parse N into F, returning false if it failed and we couldn't + // meaningfully recover (e.g. YAML syntax error broke the stream). + // The private parse() helpers follow the same pattern. + bool parse(Fragment &F, Node &N) { + DictParser Dict("Config", this); + Dict.handle("If", [&](Node &N) { + F.Condition.emplace(); + return parse(*F.Condition, N); + }); + Dict.handle("CompileFlags", + [&](Node &N) { return parse(F.CompileFlags, N); }); + return Dict.parse(N); + } + +private: + bool parse(Fragment::ConditionFragment &F, Node &N) { + DictParser Dict("Condition", this); + Dict.unrecognized([&](llvm::StringRef) { F.UnrecognizedCondition = true; }); + Dict.handle("PathMatch", [&](Node &N) { + if (auto Values = scalarValues(N)) + F.PathMatch = std::move(*Values); + return !N.failed(); + }); + return Dict.parse(N); + } + + bool parse(Fragment::CompileFlagsFragment &F, Node &N) { + DictParser Dict("CompileFlags", this); + Dict.handle("Add", [&](Node &N) { + if (auto Values = scalarValues(N)) + F.Add = std::move(*Values); + return !N.failed(); + }); + return Dict.parse(N); + } + + // Helper for parsing mapping nodes (dictionaries). + // We don't use YamlIO as we want to control over unknown keys. + class DictParser { + llvm::StringRef Description; + std::vector<std::pair<llvm::StringRef, std::function<bool(Node &)>>> Keys; + std::function<void(llvm::StringRef)> Unknown; + Parser *Outer; + + public: + DictParser(llvm::StringRef Description, Parser *Outer) + : Description(Description), Outer(Outer) {} + + // Parse is called when Key is encountered. + void handle(llvm::StringLiteral Key, std::function<bool(Node &)> Parse) { + Keys.emplace_back(Key, std::move(Parse)); + } + + // Handle is called when a Key is not matched by any handle(). + void unrecognized(std::function<void(llvm::StringRef)> Fallback) { + Unknown = std::move(Fallback); + } + + // Process a mapping node and call handlers for each key/value pair. + bool parse(Node &N) const { + if (N.getType() != Node::NK_Mapping) + return Outer->error(Description + " should be a dictionary", N); + llvm::SmallSet<std::string, 8> Seen; + for (auto &KV : llvm::cast<MappingNode>(N)) { + auto *K = KV.getKey(); + if (!K) + return false; + auto Key = Outer->scalarValue(*K, "Dictionary key"); + if (!Key) + continue; + if (!Seen.insert(**Key).second) { + Outer->warning("Duplicate key " + **Key, *K); + continue; + } + auto *Value = KV.getValue(); + if (!Value) + return false; + bool Matched = false; + for (const auto &Handler : Keys) { + if (Handler.first == **Key) { + if (!Handler.second(*Value)) + return false; + Matched = true; + break; + } + } + if (!Matched) { + Outer->warning("Unknown " + Description + " key " + **Key, *K); + if (Unknown) + Unknown(**Key); + } + } + return true; + } + }; + + // Try to parse a single scalar value from the node, warn on failure. + llvm::Optional<Located<std::string>> scalarValue(Node &N, + llvm::StringRef Desc) { + if (auto *S = llvm::dyn_cast<ScalarNode>(&N)) + return Located<std::string>(S->getValue(Buf).str(), N.getSourceRange()); + else if (auto *BS = llvm::dyn_cast<BlockScalarNode>(&N)) + return Located<std::string>(S->getValue(Buf).str(), N.getSourceRange()); + warning(Desc + " should be scalar", N); + return llvm::None; + } + + // Try to parse a list of single scalar values, or just a single value. + llvm::Optional<std::vector<Located<std::string>>> scalarValues(Node &N) { + std::vector<Located<std::string>> Result; + if (auto *S = llvm::dyn_cast<ScalarNode>(&N)) { + Result.emplace_back(S->getValue(Buf).str(), N.getSourceRange()); + } else if (auto *S = llvm::dyn_cast<BlockScalarNode>(&N)) { + Result.emplace_back(S->getValue().str(), N.getSourceRange()); + } else if (auto *S = llvm::dyn_cast<SequenceNode>(&N)) { + for (auto &Child : *S) { + if (auto Value = scalarValue(Child, "List item")) + Result.push_back(std::move(*Value)); + } + } else { + warning("Expected scalar or list of scalars", N); + return llvm::None; + } + return Result; + } + + // Report a "hard" error, reflecting a config file that can never be valid. + bool error(const llvm::Twine &Msg, const Node &N) { + SM.PrintMessage(N.getSourceRange().Start, llvm::SourceMgr::DK_Error, Msg, + N.getSourceRange()); + return false; + } + + // Report a "soft" error that could be caused by e.g. version skew. + void warning(const llvm::Twine &Msg, const Node &N) { + SM.PrintMessage(N.getSourceRange().Start, llvm::SourceMgr::DK_Warning, Msg, + N.getSourceRange()); + } +}; + +} // namespace + +std::vector<Fragment> Fragment::parseYAML(llvm::StringRef YAML, + llvm::StringRef BufferName, + DiagnosticCallback Diags) { + // The YAML document may contain multiple conditional fragments. + // The SourceManager is shared for all of them. + auto SM = std::make_shared<llvm::SourceMgr>(); + auto Buf = llvm::MemoryBuffer::getMemBufferCopy(YAML, BufferName); + // Adapt DiagnosticCallback to function-pointer interface. + // Callback receives both errors we emit and those from the YAML parser. + SM->setDiagHandler( + [](const llvm::SMDiagnostic &Diag, void *Ctx) { + (*reinterpret_cast<DiagnosticCallback *>(Ctx))(Diag); + }, + &Diags); + std::vector<Fragment> Result; + for (auto &Doc : llvm::yaml::Stream(*Buf, *SM)) { + if (Node *N = Doc.parseBlockNode()) { + Fragment Fragment; + Fragment.Source.Manager = SM; + Fragment.Source.Location = N->getSourceRange().Start; + if (Parser(*SM).parse(Fragment, *N)) + Result.push_back(std::move(Fragment)); + } + } + // Hack: stash the buffer in the SourceMgr to keep it alive. + // SM has two entries: "main" non-owning buffer, and ignored owning buffer. + SM->AddNewSourceBuffer(std::move(Buf), llvm::SMLoc()); + return Result; +} + +} // namespace config +} // namespace clangd +} // namespace clang Index: clang-tools-extra/clangd/ConfigFragment.h =================================================================== --- /dev/null +++ clang-tools-extra/clangd/ConfigFragment.h @@ -0,0 +1,94 @@ +//===--- ConfigFragment.h - Unit of user-specified configuration -*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Various clangd features have configurable behaviour (or can be disabled). +// The configuration system allows users to control this: +// - in a user config file, a project config file, via LSP, or via flags +// - specifying different settings for different files +// +// This file defines the config::Fragment structure which is models one piece of +// configuration as obtained from a source like a file. +// This is distinct from how the config is interpreted (CompiledFragment), +// combined (ConfigProvider) and exposed to the rest of clangd (Config). +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIG_FRAGMENT_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CONFIG_FRAGMENT_H + +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/SMLoc.h" +#include "llvm/Support/SourceMgr.h" +#include <string> +#include <vector> + +namespace clang { +namespace clangd { +namespace config { + +/// An entity written in config along, with its optional location in the file. +template <typename T> struct Located { + Located(T Value, llvm::SMRange Range = {}) + : Range(Range), Value(std::move(Value)) {} + + llvm::SMRange Range; + T &operator->() { return Value; } + const T &operator->() const { return Value; } + T &operator*() { return Value; } + const T &operator*() const { return Value; } + +private: + T Value; +}; + +/// Used to report problems in parsing or interpreting a config. +/// Errors reflect structurally invalid config that should be user-visible. +/// Warnings reflect e.g. unknown properties that are recoverable. +using DiagnosticCallback = llvm::function_ref<void(const llvm::SMDiagnostic &)>; + +/// A chunk of configuration obtained from a config file, LSP, or elsewhere. +struct Fragment { + /// Parses fragments from a YAML file (one from each --- delimited document). + /// Documents that contained fatal errors are omitted from the results. + /// BufferName is used for the SourceMgr and diagnostics. + static std::vector<Fragment> parseYAML(llvm::StringRef YAML, + llvm::StringRef BufferName, + DiagnosticCallback); + + struct SourceInfo { + /// Retains a buffer of the original source this fragment was parsed from. + /// Locations within Located<T> objects point into this SourceMgr. + /// Shared because multiple fragments are often parsed from one (YAML) file. + /// May be null, then all locations are ignored. + std::shared_ptr<llvm::SourceMgr> Manager; + /// The start of the original source for this fragment. + /// Only valid if SourceManager is set. + llvm::SMLoc Location; + }; + SourceInfo Source; + + struct ConditionFragment { + std::vector<Located<std::string>> PathMatch; + /// An unrecognized key was found while parsing the condition. + /// The condition will evaluate to false. + bool UnrecognizedCondition; + }; + llvm::Optional<ConditionFragment> Condition; + + struct CompileFlagsFragment { + std::vector<Located<std::string>> Add; + } CompileFlags; +}; + +} // namespace config +} // namespace clangd +} // namespace clang + +#endif Index: clang-tools-extra/clangd/CMakeLists.txt =================================================================== --- clang-tools-extra/clangd/CMakeLists.txt +++ clang-tools-extra/clangd/CMakeLists.txt @@ -36,6 +36,7 @@ CollectMacros.cpp CompileCommands.cpp Compiler.cpp + ConfigYAML.cpp Diagnostics.cpp DraftStore.cpp ExpectedTypes.cpp
_______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
