https://github.com/dzbarsky updated https://github.com/llvm/llvm-project/pull/202663
>From 7bd79992f8744d6055a3257757f3476416551b55 Mon Sep 17 00:00:00 2001 From: David Zbarsky <[email protected]> Date: Tue, 9 Jun 2026 04:54:39 -0400 Subject: [PATCH] [clang][Tooling] Reuse standard-library symbol descriptors StandardLibrary.cpp expands the generated C and C++ symbol mappings once to initialize SymbolHeaderMapping and a second time to count unique symbols. The generated SymbolMapping arrays also store two pointers and an unsigned integer for every symbol-to-header mapping. Store each qualified name and header name as one QName\0Header name in EnumStringDef<uint8_t, 1>, with the namespace length in value(). BUILD_ENUM_STRINGS creates pointer-free EnumStringsStorage tables. Split StdSymbolMap.inc when the four-byte EnumString records and string bytes approach the uint16_t offset limit. Count and initialize SymbolHeaderMapping from the same EnumStrings tables. In an arm64 Release build with assertions, StandardLibrary.cpp.o decreases from 542,568 to 212,344 bytes (-330,224), and its loadable contents decrease from 272,928 to 190,176 bytes (-82,752). __DATA,__const decreases by 184,416 bytes, __TEXT,__cstring decreases by 70,113 bytes, __TEXT,__const increases by 172,302 bytes, and object relocations decrease from 18,041 to 784 (-17,257). Linked clangd decreases from 122,442,440 to 122,375,944 bytes (-66,496), and stripped clangd decreases from 69,416,496 to 69,350,440 bytes (-66,056). Linked fixups decrease from 176,944 to 159,655 (-17,289). An exhaustive dump of all public C and C++ symbol-to-header mappings is byte-identical before and after the change (4,463 lines). clangd --check also completes with zero errors. --- .../Inclusions/Stdlib/StandardLibrary.cpp | 168 ++++++++++++------ .../Inclusions/Stdlib/StdSymbolMap.inc | 4 + clang/tools/include-mapping/gen_std.py | 56 +++++- 3 files changed, 172 insertions(+), 56 deletions(-) diff --git a/clang/lib/Tooling/Inclusions/Stdlib/StandardLibrary.cpp b/clang/lib/Tooling/Inclusions/Stdlib/StandardLibrary.cpp index 807a8d8a34ad7..ee44dadefc8dc 100644 --- a/clang/lib/Tooling/Inclusions/Stdlib/StandardLibrary.cpp +++ b/clang/lib/Tooling/Inclusions/Stdlib/StandardLibrary.cpp @@ -10,11 +10,15 @@ #include "clang/AST/Decl.h" #include "clang/Basic/LangOptions.h" #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/Enum.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/Casting.h" +#include "llvm/Support/ErrorHandling.h" +#include <cstdint> +#include <limits> #include <optional> +#include <utility> namespace clang { namespace tooling { @@ -24,6 +28,72 @@ namespace { // Symbol name -> Symbol::ID, within a namespace. using NSSymbolMap = llvm::DenseMap<llvm::StringRef, unsigned>; +using SymbolNamespaceLength = uint8_t; +using SymbolMappingDef = llvm::EnumStringDef<SymbolNamespaceLength>; +using SymbolMapping = llvm::EnumString<SymbolNamespaceLength>; +using SymbolMappingTable = llvm::EnumStrings<SymbolNamespaceLength>; +static_assert(sizeof(SymbolMapping) == 4, + "update the standard symbol table partition size"); + +template <size_t Size> +constexpr SymbolNamespaceLength getSymbolNamespaceLength(const char (&)[Size]) { + static_assert(Size - 1 <= std::numeric_limits<SymbolNamespaceLength>::max(), + "symbol namespace length does not fit in one byte"); + return static_cast<SymbolNamespaceLength>(Size - 1); +} + +static constexpr SymbolMappingDef CSymbolDefs[] = { +#define SYMBOL(Name, NS, Header) \ + {{llvm::StringLiteral::withInnerNUL(#NS #Name "\0" #Header)}, \ + getSymbolNamespaceLength(#NS)}, +#include "CSpecialSymbolMap.inc" +#include "CSymbolMap.inc" +#undef SYMBOL +}; +static constexpr auto CSymbols = BUILD_ENUM_STRINGS(CSymbolDefs); + +static constexpr SymbolMappingDef CXXSpecialSymbolDefs[] = { +#define SYMBOL(Name, NS, Header) \ + {{llvm::StringLiteral::withInnerNUL(#NS #Name "\0" #Header)}, \ + getSymbolNamespaceLength(#NS)}, +#include "StdSpecialSymbolMap.inc" +#undef SYMBOL +}; +static constexpr auto CXXSpecialSymbols = + BUILD_ENUM_STRINGS(CXXSpecialSymbolDefs); + +#define SYMBOL_MAP_BEGIN(Index) \ + static constexpr SymbolMappingDef CXXSymbolDefs##Index[] = { +#define SYMBOL(Name, NS, Header) \ + {{llvm::StringLiteral::withInnerNUL(#NS #Name "\0" #Header)}, \ + getSymbolNamespaceLength(#NS)}, +#define SYMBOL_MAP_END(Index) \ + } \ + ; \ + static constexpr auto CXXSymbols##Index = \ + BUILD_ENUM_STRINGS(CXXSymbolDefs##Index); +#include "StdSymbolMap.inc" +#undef SYMBOL_MAP_END +#undef SYMBOL +#undef SYMBOL_MAP_BEGIN + +static constexpr SymbolMappingDef CXXTsSymbolDefs[] = { +#define SYMBOL(Name, NS, Header) \ + {{llvm::StringLiteral::withInnerNUL(#NS #Name "\0" #Header)}, \ + getSymbolNamespaceLength(#NS)}, +#include "StdTsSymbolMap.inc" +#undef SYMBOL +}; +static constexpr auto CXXTsSymbols = BUILD_ENUM_STRINGS(CXXTsSymbolDefs); + +static std::pair<StringRef, StringRef> +splitSymbolMapping(const SymbolMapping &Mapping) { + StringRef Name = Mapping.name(); + size_t Delimiter = Name.find('\0'); + assert(Delimiter != StringRef::npos && "missing symbol mapping delimiter"); + return {Name.take_front(Delimiter), Name.drop_front(Delimiter + 1)}; +} + // A Mapping per language. struct SymbolHeaderMapping { llvm::StringRef *HeaderNames = nullptr; @@ -54,37 +124,27 @@ static const SymbolHeaderMapping *getMappingPerLang(Lang L) { return LanguageMappings[static_cast<unsigned>(L)]; } -static int countSymbols(Lang Language) { - ArrayRef<const char *> Symbols; -#define SYMBOL(Name, NS, Header) #NS #Name, - switch (Language) { - case Lang::C: { - static constexpr const char *CSymbols[] = { -#include "CSpecialSymbolMap.inc" -#include "CSymbolMap.inc" - }; - Symbols = CSymbols; - break; - } - case Lang::CXX: { - static constexpr const char *CXXSymbols[] = { -#include "StdSpecialSymbolMap.inc" -#include "StdSymbolMap.inc" -#include "StdTsSymbolMap.inc" - }; - Symbols = CXXSymbols; - break; - } +static unsigned countSymbols(ArrayRef<SymbolMappingTable> SymbolMappingTables) { + unsigned Count = 0; + StringRef Previous; + for (const SymbolMappingTable &Table : SymbolMappingTables) { + for (const SymbolMapping &Mapping : Table) { + StringRef QName = splitSymbolMapping(Mapping).first; + if (Previous != QName) { + ++Count; + Previous = QName; + } + } } -#undef SYMBOL - return llvm::DenseSet<StringRef>(llvm::from_range, Symbols).size(); + return Count; } -static int initialize(Lang Language) { +static int initialize(Lang Language, + ArrayRef<SymbolMappingTable> SymbolMappingTables) { SymbolHeaderMapping *Mapping = new SymbolHeaderMapping(); LanguageMappings[static_cast<unsigned>(Language)] = Mapping; - unsigned SymCount = countSymbols(Language); + unsigned SymCount = countSymbols(SymbolMappingTables); Mapping->SymbolCount = SymCount; Mapping->SymbolNames = new std::remove_reference_t<decltype(*Mapping->SymbolNames)>[SymCount]; @@ -137,36 +197,12 @@ static int initialize(Lang Language) { NSSymbols.try_emplace(QName.drop_front(NSLen), SymIndex); }; - struct Symbol { - const char *QName; - unsigned NSLen; - const char *HeaderName; - }; -#define SYMBOL(Name, NS, Header) \ - {#NS #Name, static_cast<decltype(Symbol::NSLen)>(StringRef(#NS).size()), \ - #Header}, - switch (Language) { - case Lang::C: { - static constexpr Symbol CSymbols[] = { -#include "CSpecialSymbolMap.inc" -#include "CSymbolMap.inc" - }; - for (const Symbol &S : CSymbols) - Add(S.QName, S.NSLen, S.HeaderName); - break; - } - case Lang::CXX: { - static constexpr Symbol CXXSymbols[] = { -#include "StdSpecialSymbolMap.inc" -#include "StdSymbolMap.inc" -#include "StdTsSymbolMap.inc" - }; - for (const Symbol &S : CXXSymbols) - Add(S.QName, S.NSLen, S.HeaderName); - break; - } + for (const SymbolMappingTable &Table : SymbolMappingTables) { + for (const SymbolMapping &Mapping : Table) { + auto [QName, HeaderName] = splitSymbolMapping(Mapping); + Add(QName, Mapping.value(), HeaderName); + } } -#undef SYMBOL Mapping->HeaderNames = new llvm::StringRef[Mapping->HeaderIDs->size()]; for (const auto &E : *Mapping->HeaderIDs) @@ -175,6 +211,28 @@ static int initialize(Lang Language) { return 0; } +static int initialize(Lang Language) { + switch (Language) { + case Lang::C: { + const SymbolMappingTable SymbolMappingTables[] = {CSymbols}; + return initialize(Language, SymbolMappingTables); + } + case Lang::CXX: { + const SymbolMappingTable SymbolMappingTables[] = {CXXSpecialSymbols, +#define SYMBOL(Name, NS, Header) +#define SYMBOL_MAP_BEGIN(Index) CXXSymbols##Index, +#define SYMBOL_MAP_END(Index) +#include "StdSymbolMap.inc" +#undef SYMBOL_MAP_END +#undef SYMBOL_MAP_BEGIN +#undef SYMBOL + CXXTsSymbols}; + return initialize(Language, SymbolMappingTables); + } + } + llvm_unreachable("unknown language"); +} + static void ensureInitialized() { static int Dummy = []() { for (unsigned L = 0; L <= static_cast<unsigned>(Lang::LastValue); ++L) diff --git a/clang/lib/Tooling/Inclusions/Stdlib/StdSymbolMap.inc b/clang/lib/Tooling/Inclusions/Stdlib/StdSymbolMap.inc index c1927180d3397..45329d83458d4 100644 --- a/clang/lib/Tooling/Inclusions/Stdlib/StdSymbolMap.inc +++ b/clang/lib/Tooling/Inclusions/Stdlib/StdSymbolMap.inc @@ -9,6 +9,7 @@ // Generated from cppreference offline HTML book (modified on 2024-11-10). //===----------------------------------------------------------------------===// +SYMBOL_MAP_BEGIN(0) SYMBOL(ATOMIC_BOOL_LOCK_FREE, None, <atomic>) SYMBOL(ATOMIC_CHAR16_T_LOCK_FREE, None, <atomic>) SYMBOL(ATOMIC_CHAR32_T_LOCK_FREE, None, <atomic>) @@ -2289,6 +2290,8 @@ SYMBOL(nextafterf, std::, <cmath>) SYMBOL(nextafterf, None, <cmath>) SYMBOL(nextafterf, None, <math.h>) SYMBOL(nextafterl, std::, <cmath>) +SYMBOL_MAP_END(0) +SYMBOL_MAP_BEGIN(1) SYMBOL(nextafterl, None, <cmath>) SYMBOL(nextafterl, None, <math.h>) SYMBOL(nexttoward, std::, <cmath>) @@ -3985,3 +3988,4 @@ SYMBOL(transform, std::views::, <ranges>) SYMBOL(values, std::views::, <ranges>) SYMBOL(zip, std::views::, <ranges>) SYMBOL(zip_transform, std::views::, <ranges>) +SYMBOL_MAP_END(1) diff --git a/clang/tools/include-mapping/gen_std.py b/clang/tools/include-mapping/gen_std.py index f362227bc6aab..aac87a436b372 100755 --- a/clang/tools/include-mapping/gen_std.py +++ b/clang/tools/include-mapping/gen_std.py @@ -55,6 +55,58 @@ //===----------------------------------------------------------------------===// """ +# EnumString<uint8_t, 1> stores a 4-byte record for each string. Its 16-bit +# offset must address the string data from the record. +MAX_SYMBOL_TABLE_STORAGE_BYTES = (1 << 16) - 1 +SYMBOL_TABLE_RECORD_BYTES = 4 + + +def SymbolTableRowSize(symbol_row): + name, namespace, header = symbol_row + string_size = len(("%s%s\0%s" % (namespace, name, header)).encode("utf-8")) + return SYMBOL_TABLE_RECORD_BYTES + string_size + + +def EmitSymbolRows(symbol_rows, partition_symbol_tables): + if not partition_symbol_tables: + for name, namespace, header in symbol_rows: + print("SYMBOL(%s, %s, %s)" % (name, namespace, header)) + return + + symbol_groups = [] + for symbol_row in symbol_rows: + qualified_name = symbol_row[1], symbol_row[0] + if not symbol_groups or symbol_groups[-1][0] != qualified_name: + symbol_groups.append((qualified_name, [])) + symbol_groups[-1][1].append(symbol_row) + + symbol_tables = [] + current_symbol_table = [] + current_symbol_table_size = 0 + for _, symbol_group in symbol_groups: + group_size = sum(SymbolTableRowSize(row) for row in symbol_group) + if group_size > MAX_SYMBOL_TABLE_STORAGE_BYTES: + raise ValueError("symbol mapping group exceeds symbol table limit") + if ( + current_symbol_table + and current_symbol_table_size + group_size > MAX_SYMBOL_TABLE_STORAGE_BYTES + ): + symbol_tables.append(current_symbol_table) + current_symbol_table = [] + current_symbol_table_size = 0 + current_symbol_table.extend(symbol_group) + current_symbol_table_size += group_size + if current_symbol_table: + symbol_tables.append(current_symbol_table) + if not symbol_tables: + symbol_tables.append([]) + + for table_index, symbol_table in enumerate(symbol_tables): + print("SYMBOL_MAP_BEGIN(%d)" % table_index) + for name, namespace, header in symbol_table: + print("SYMBOL(%s, %s, %s)" % (name, namespace, header)) + print("SYMBOL_MAP_END(%d)" % table_index) + def ParseArg(): parser = argparse.ArgumentParser(description="Generate StdGen file") @@ -263,6 +315,7 @@ def main(): exit("Path %s doesn't exist!" % symbol_index_root) symbols = cppreference_parser.GetSymbols(parse_pages) + symbol_rows = [] # We don't have version information from the unzipped offline HTML files. # so we use the modified time of the symbol_index.html as the version. @@ -279,7 +332,7 @@ def main(): s.headers.extend(AdditionalHeadersForIOSymbols(s)) for header in s.headers: # SYMBOL(unqualified_name, namespace, header) - print("SYMBOL(%s, %s, %s)" % (s.name, s.namespace, header)) + symbol_rows.append((s.name, s.namespace, header)) elif len(symbol.headers) == 0: sys.stderr.write("No header found for symbol %s\n" % symbol.name) else: @@ -288,6 +341,7 @@ def main(): "Ambiguous header for symbol %s: %s\n" % (symbol.name, ", ".join(symbol.headers)) ) + EmitSymbolRows(symbol_rows, args.symbols == "cpp") if __name__ == "__main__": _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
