This is an automated email from the ASF dual-hosted git repository. xiaoxiang781216 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/nuttx.git
commit 1760e846ab9ee0d47316df339ee3a6d9e9beba3a Author: wangmingrong1 <[email protected]> AuthorDate: Mon Nov 2 16:08:35 2026 +0800 tools/gcov: auto-detect toolchain from ELF, rework CLI The toolchain used to build the firmware is recorded in the ELF, so detect it from there instead of asking the user to pick one. * `-e ELF` is now required and is the source of truth: clang vs gcc is read from `.comment`, target arch from `e_machine` (via a small pure-Python ELF inspector, no external readelf). * `-t` and `--toolchain` removed. gcov/lcov/genhtml (GCC) and llvm-profdata/llvm-cov/llvm-readelf/ld.lld (Clang) are resolved from PATH; the cross gcov is matched by arch (e.g. aarch64-none-elf-gcov). * `-b` changes from "base dir" to a repeatable `OLD=NEW` path remap, wired to `lcov --substitute` (GCC) and `-path-equivalence` (Clang). * `-x/--only-copy` removed. * `-s` (clang) may now point directly at a pre-built covmap object (nuttx_llvm_covmap.o / coverage_all.o), so a report can be generated after the build tree is gone; the relocatable-object link is cached via a fingerprint sidecar. `gcov-tool` is resolved by mirroring the resolved gcov binary path (e.g. aarch64-none-elf-gcov -> aarch64-none-elf-gcov-tool), fixing the Ubuntu case where /usr/bin/gcov is gcov-13 but gcov-tool is gcov-tool-11, which broke multi-`-a` merges. The script is also split into labelled sections with small dataclasses (ElfInfo, GccTools, LlvmTools, PathMap). Verified end-to-end on aarch64+clang, aarch64+gcc and x86+gcc datasets, including multi-`-a` merge. Signed-off-by: wangmingrong1 <[email protected]> --- tools/gcov.py | 1556 ++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 983 insertions(+), 573 deletions(-) diff --git a/tools/gcov.py b/tools/gcov.py index a13e4fcba0c..5818031e6d8 100755 --- a/tools/gcov.py +++ b/tools/gcov.py @@ -18,13 +18,42 @@ # License for the specific language governing permissions and limitations # under the License. +"""Code coverage report generator for NuttX. + +Supports both GCC gcov and Clang source-based coverage. The toolchain is +inferred from the ELF binary's `.comment` section, and the cross-compile +prefix (if any) is inferred from the ELF machine type, so users no longer +have to specify `-t`/`--toolchain`. + +Usage example: + + gcov.py -e build/nuttx -s build -a coverage_data.bin -o report_dir + +The required tools (gcov/lcov/genhtml for GCC; llvm-profdata/llvm-cov/ +llvm-readelf/ld.lld for Clang) are resolved from PATH; the cross-compile +gcov is matched by arch (e.g. aarch64-none-elf-gcov for EM_AARCH64). + +Use `-b OLD=NEW` (repeatable) to rewrite source paths recorded by the +compiler at build time to local paths, mirroring `-fdebug-prefix-map`: + + gcov.py -b /old/build/path=/new/local/path ... +""" + import argparse +import concurrent.futures +import json import os import re import shutil import struct import subprocess import sys +from dataclasses import dataclass, field +from typing import List, Optional, Tuple + +# ============================================================================= +# 1. CONSTANTS +# ============================================================================= # LLVM raw profile magic bytes (first 8 bytes of .profraw). # @@ -37,79 +66,292 @@ import sys # # 64-bit BE: \xff lprofr \x81 / 64-bit LE: \x81 rforpl \xff # 32-bit BE: \xff lprofR \x81 / 32-bit LE: \x81 Rforpl \xff - PROFRAW_MAGIC_64 = b"\xff\x6c\x70\x72\x6f\x66\x72\x81" PROFRAW_MAGIC_32 = b"\xff\x6c\x70\x72\x6f\x66\x52\x81" - -# gcov binary pack magic heuristic: +PROFRAW_MAGICS = ( + PROFRAW_MAGIC_64, + PROFRAW_MAGIC_64[::-1], + PROFRAW_MAGIC_32, + PROFRAW_MAGIC_32[::-1], +) + +# gcov binary pack heuristic: # First 4 bytes are path_len (LE uint32), typically 30~200 for gcda paths. # Followed by a path string starting with a printable char (not \x00 or \xff). - GCOV_BINPACK_PATH_LEN_MIN = 10 GCOV_BINPACK_PATH_LEN_MAX = 4096 +# Suffix appended to a -a input file to name the directory holding the +# coverage data extracted from it (hex dump / binpack -> individual files). +CONVERT_SUFFIX = "_convert" + +# ELF e_machine values we care about for cross-prefix inference. +EM_386 = 3 +EM_X86_64 = 62 +EM_ARM = 40 +EM_AARCH64 = 183 +EM_RISCV = 243 +EM_XTENSA = 94 + +# Candidate cross-compile gcov binary names per architecture, in priority +# order. The first one found on PATH is used. +GCC_CROSS_PREFIXES = { + EM_AARCH64: ["aarch64-none-elf-gcov", "aarch64-linux-gnu-gcov"], + EM_ARM: ["arm-none-eabi-gcov", "arm-linux-gnueabi-gcov"], + EM_RISCV: ["riscv64-unknown-elf-gcov", "riscv32-unknown-elf-gcov"], + EM_XTENSA: ["xtensa-esp32-elf-gcov", "xtensa-esp32s3-elf-gcov"], + EM_X86_64: ["gcov"], + EM_386: ["gcov"], +} + + +# ============================================================================= +# 2. LOGGING / SUBPROCESS HELPERS +# ============================================================================= + + +def log(msg: str) -> None: + print(msg, flush=True) + + +def warn(msg: str) -> None: + print(f"warning: {msg}", flush=True) + + +def die(msg: str, code: int = 1) -> None: + print(f"error: {msg}", file=sys.stderr, flush=True) + sys.exit(code) -def is_profraw_file(path): - """Check if a file is an LLVM profraw file by magic number. - Accepts both native and byte-swapped magic so that profraw files - produced on either little- or big-endian targets are recognised. +def print_report_link(report_dir: str) -> None: + """Print the file:// link to the generated HTML report's index page. + + Only call after confirming the report was produced; the index is the + common entry point for both the lcov/genhtml and llvm-cov outputs. """ + log( + "Copy the following link and open it in the browser to view the " + "coverage report:" + ) + log(f"file://{os.path.join(report_dir, 'index.html')}") + + +def run_cmd(cmd: List[str], check: bool = True) -> int: + """Run a subprocess, streaming output, returning its exit code.""" + log(" ".join(cmd)) + return subprocess.run( + cmd, + check=check, + stdout=sys.stdout, + stderr=sys.stdout, + ).returncode + + +# ============================================================================= +# 3. FORMAT DETECTION +# ============================================================================= + + +def is_profraw_file(path: str) -> bool: + """Check if a file is an LLVM profraw file by magic number.""" try: with open(path, "rb") as f: - magic = f.read(8) - return magic in ( - PROFRAW_MAGIC_64, - PROFRAW_MAGIC_64[::-1], - PROFRAW_MAGIC_32, - PROFRAW_MAGIC_32[::-1], - ) + return f.read(8) in PROFRAW_MAGICS + except (IOError, OSError): + return False + + +def is_gcov_binpack_file(path: str) -> bool: + """Check if a file is a gcov binary pack file. + + Binary format: [path_len(4B LE)][path+NUL][data_size(4B LE)][data]... + Heuristic: first 4 bytes are a small uint32, followed by a printable path. + """ + try: + with open(path, "rb") as f: + header = f.read(8) + if len(header) < 8: + return False + path_len = struct.unpack_from("<I", header, 0)[0] + if path_len < GCOV_BINPACK_PATH_LEN_MIN: + return False + if path_len > GCOV_BINPACK_PATH_LEN_MAX: + return False + first_char = header[4] + return 0x20 <= first_char < 0x7F except (IOError, OSError): return False -def detect_toolchain(gcda_dirs): - """Detect whether coverage data is from gcc or clang. +# ============================================================================= +# 4. ELF INSPECTOR (toolchain + machine, no external readelf) +# ============================================================================= - Returns 'clang' if any profraw file is found, otherwise 'gcc'. + +@dataclass +class ElfInfo: + toolchain: str # "gcc" | "clang" + e_machine: int + + @property + def arch_name(self) -> str: + return { + EM_386: "x86", + EM_X86_64: "x86_64", + EM_ARM: "arm", + EM_AARCH64: "aarch64", + EM_RISCV: "riscv", + EM_XTENSA: "xtensa", + }.get(self.e_machine, f"em{self.e_machine}") + + +def _read_section(f, header: bytes, name: str) -> bytes: + """Read a named section's raw contents from an open ELF file. + + Pure-Python ELF parsing; supports both ELF32 and ELF64, both endians. + Reads only the section-header table, the section-name string table, and + the matched section — never the whole file. ``header`` must hold at least + the first 64 bytes of the file (the ELF header). Returns empty bytes if + the section is missing or the ELF is malformed. """ - for d in gcda_dirs: - if os.path.isfile(d): - if is_profraw_file(d): - return "clang" + if len(header) < 0x40 or header[:4] != b"\x7fELF": + return b"" + + elf_class = header[4] # 1 = ELF32, 2 = ELF64 + little = header[5] == 1 + end = "<" if little else ">" + + if elf_class == 1: + # ELF32 header; section header: name(0), offset(16), size(20) + e_shoff = struct.unpack_from(end + "I", header, 0x20)[0] + e_shentsize = struct.unpack_from(end + "H", header, 0x2E)[0] + e_shnum = struct.unpack_from(end + "H", header, 0x30)[0] + e_shstrndx = struct.unpack_from(end + "H", header, 0x32)[0] + off_off, size_off = 16, 20 + elif elf_class == 2: + # ELF64 header; section header: name(0), offset(24), size(32) + e_shoff = struct.unpack_from(end + "Q", header, 0x28)[0] + e_shentsize = struct.unpack_from(end + "H", header, 0x3A)[0] + e_shnum = struct.unpack_from(end + "H", header, 0x3C)[0] + e_shstrndx = struct.unpack_from(end + "H", header, 0x3E)[0] + off_off, size_off = 24, 32 + else: + return b"" + + if e_shnum == 0 or e_shoff == 0: + return b"" + + # Width of the sh_offset / sh_size fields: 4 bytes on ELF32, 8 on ELF64. + addr_fmt = end + ("I" if elf_class == 1 else "Q") + + def pread(offset: int, size: int) -> bytes: + f.seek(offset) + return f.read(size) + + # Read the whole section-header table (small: e_shnum * e_shentsize). + sht = pread(e_shoff, e_shnum * e_shentsize) + + def section_header(idx: int) -> bytes: + start = idx * e_shentsize + return sht[start : start + e_shentsize] + + # Read .shstrtab (section name string table) first. + shstr_hdr = section_header(e_shstrndx) + shstr_off = struct.unpack_from(addr_fmt, shstr_hdr, off_off)[0] + shstr_size = struct.unpack_from(addr_fmt, shstr_hdr, size_off)[0] + shstrtab = pread(shstr_off, shstr_size) + + target = name.encode() + b"\x00" + + for i in range(e_shnum): + hdr = section_header(i) + if len(hdr) < e_shentsize: + continue + sh_name = struct.unpack_from(end + "I", hdr, 0)[0] + # Match the NUL-terminated name from shstrtab + end_pos = shstrtab.find(b"\x00", sh_name) + if end_pos < 0: continue - for root, dirs, files in os.walk(d): - for f in files: - fpath = os.path.join(root, f) - if f.endswith(".profraw") or is_profraw_file(fpath): - return "clang" - return "gcc" + sec_name = shstrtab[sh_name:end_pos] + if sec_name + b"\x00" == target: + sh_off = struct.unpack_from(addr_fmt, hdr, off_off)[0] + sh_sz = struct.unpack_from(addr_fmt, hdr, size_off)[0] + return pread(sh_off, sh_sz) + return b"" + + +def inspect_elf(path: str) -> ElfInfo: + """Inspect ELF for toolchain and machine type. + + Toolchain is detected from the `.comment` section: presence of "clang" + means clang, otherwise gcc. e_machine is read from the ELF header. Only + the ELF header, section-header table, .shstrtab and .comment are read, + so this is cheap even for very large (e.g. ~700MB clang) binaries. + """ + with open(path, "rb") as f: + header = f.read(64) + if len(header) < 0x40 or header[:4] != b"\x7fELF": + die(f"not an ELF file: {path}") + + elf_class = header[4] + if elf_class not in (1, 2): + die(f"unknown ELF class {elf_class}") + little = header[5] == 1 + end = "<" if little else ">" + # e_machine sits at offset 0x12 in both ELF32 and ELF64 headers. + e_machine = struct.unpack_from(end + "H", header, 0x12)[0] -def parse_gcda_data(path): + comment = _read_section(f, header, ".comment") + + toolchain = "clang" if b"clang" in comment.lower() else "gcc" + return ElfInfo(toolchain=toolchain, e_machine=e_machine) + + +# ============================================================================= +# 5. DATA EXTRACTION (gcda hex dump / gcov binpack / profraw) +# ============================================================================= + + +def _safe_join(out_dir: str, rel: str) -> Optional[str]: + """Join ``rel`` (from untrusted dump content) under ``out_dir`` safely. + + Strips leading separators and rejects any path that escapes ``out_dir`` + via ``..`` traversal. Returns the absolute path, or None if it would + escape (caller should warn and skip). + """ + rel = rel.lstrip("/").lstrip("\\") + out_abs = os.path.realpath(out_dir) + dst = os.path.realpath(os.path.join(out_abs, rel)) + if dst != out_abs and not dst.startswith(out_abs + os.sep): + return None + return dst + + +def parse_gcda_hex_dump(path: str) -> str: """Parse hex dump log containing 'gcov start/end' blocks. Works for both gcc (.gcda) and clang (.profraw) hex dumps since - the clang coverage.c now uses the same format. + the clang coverage.c uses the same format. """ - with open(path, "r") as file: - lines = file.read().strip().splitlines() + with open(path, "r") as fp: + lines = fp.read().strip().splitlines() - gcda_path = path + "_covert" - os.makedirs(gcda_path, exist_ok=True) + out_dir = path + CONVERT_SUFFIX + os.makedirs(out_dir, exist_ok=True) started = False filename = "" - output = "" + payload = "" size = 0 for line in lines: if line.startswith("gcov start"): started = True - match = re.search(r"filename:(.*?)\s+size:\s*(\d+)Byte", line) - if match: - filename = match.group(1) - size = int(match.group(2)) + m = re.search(r"filename:(.*?)\s+size:\s*(\d+)Byte", line) + if m: + filename = m.group(1) + size = int(m.group(2)) continue if not started: @@ -118,72 +360,59 @@ def parse_gcda_data(path): try: if line.startswith("gcov end"): started = False - if size != len(output) // 2: + if size != len(payload) // 2: raise ValueError( - f"Size mismatch for {filename}: expected {size} bytes, got {len(output) // 2} bytes" + f"size mismatch for {filename}: " + f"expected {size}, got {len(payload) // 2}" ) - match = re.search(r"checksum:\s*(0x[0-9a-fA-F]+)", line) - if match: - checksum = int(match.group(1), 16) - output = bytearray.fromhex(output) - expected = sum(output) % 65536 + blob = bytearray.fromhex(payload) + + m = re.search(r"checksum:\s*(0x[0-9a-fA-F]+)", line) + if m: + checksum = int(m.group(1), 16) + expected = sum(blob) % 65536 if checksum != expected: raise ValueError( - f"Checksum mismatch for {filename}: expected {checksum}, got {expected}" + f"checksum mismatch for {filename}: " + f"expected {checksum}, got {expected}" ) - - outfile = os.path.join(gcda_path, "./" + filename) - os.makedirs(os.path.dirname(outfile), exist_ok=True) - - with open(outfile, "wb") as fp: - fp.write(output) - print(f"write {outfile} success") - output = "" + else: + # No checksum token: still write the data (it passed the + # size check) but flag that it could not be verified. + warn(f"no checksum for {filename}; writing unverified data") + + out_path = _safe_join(out_dir, filename) + if out_path is None: + warn(f"skipping unsafe path in dump: {filename}") + else: + os.makedirs(os.path.dirname(out_path), exist_ok=True) + with open(out_path, "wb") as out_fp: + out_fp.write(blob) + log(f"write {out_path} success") + payload = "" else: - output += line.strip() + payload += line.strip() except Exception as e: - print(f"Error processing {path}: {e}") - print(f"gcov start filename:{filename} size:{size}") - print(output[:200]) - - return gcda_path - - -def is_gcov_binpack_file(path): - """Check if a file is a gcov binary pack file. + warn(f"error processing {path}: {e}") + log(f" gcov start filename:{filename} size:{size}") + log(f" payload[:200]={payload[:200]}") + payload = "" + started = False - Binary format: [path_len(4B LE)][path+NUL][data_size(4B LE)][data]... - Heuristic: first 4 bytes are a small uint32, followed by a printable path. - """ - - try: - with open(path, "rb") as f: - header = f.read(8) - if len(header) < 8: - return False - path_len = struct.unpack_from("<I", header, 0)[0] - if path_len < GCOV_BINPACK_PATH_LEN_MIN: - return False - if path_len > GCOV_BINPACK_PATH_LEN_MAX: - return False - first_char = header[4] - return first_char >= 0x20 and first_char < 0x7F - except (IOError, OSError): - return False + return out_dir -def parse_gcov_binpack(path): +def parse_gcov_binpack(path: str) -> str: """Parse gcov binary pack file into individual gcda files. Binary format: [path_len(4B LE)][path+NUL][data_size(4B LE)][data]... """ - with open(path, "rb") as f: data = f.read() - gcda_path = path + "_covert" - os.makedirs(gcda_path, exist_ok=True) + out_dir = path + CONVERT_SUFFIX + os.makedirs(out_dir, exist_ok=True) offset = 0 count = 0 @@ -201,7 +430,7 @@ def parse_gcov_binpack(path): or path_len > GCOV_BINPACK_PATH_LEN_MAX or offset + path_len > len(data) ): - print(f"Invalid path_len {path_len} at offset {offset - 4}") + warn(f"invalid path_len {path_len} at offset {offset - 4}") errors += 1 break @@ -209,7 +438,7 @@ def parse_gcov_binpack(path): offset += path_len if offset + 4 > len(data): - print(f"Truncated at data_size for {filepath}") + warn(f"truncated at data_size for {filepath}") errors += 1 break @@ -217,7 +446,7 @@ def parse_gcov_binpack(path): offset += 4 if offset + data_size > len(data): - print(f"Truncated data for {filepath}") + warn(f"truncated data for {filepath}") gcda_data = data[offset:] offset = len(data) errors += 1 @@ -225,605 +454,786 @@ def parse_gcov_binpack(path): gcda_data = data[offset : offset + data_size] offset += data_size - out_path = os.path.join(gcda_path, filepath.lstrip("/")) + out_path = _safe_join(out_dir, filepath) + if out_path is None: + warn(f"skipping unsafe path in binpack: {filepath}") + errors += 1 + continue os.makedirs(os.path.dirname(out_path), exist_ok=True) + with open(out_path, "wb") as out_fp: + out_fp.write(gcda_data) + count += 1 - with open(out_path, "wb") as out: - out.write(gcda_data) + log(f"parsed binary pack: {count} files ({errors} errors)") + return out_dir - count += 1 - print(f"Parsed binary pack: {count} files ({errors} errors)") - return gcda_path +def find_profraw_files(directory: str) -> List[str]: + """Recursively find all .profraw files in a directory.""" + found = [] + for root, _, files in os.walk(directory): + for f in files: + fpath = os.path.join(root, f) + if f.endswith(".profraw") or is_profraw_file(fpath): + found.append(fpath) + return found -def correct_content_path(file, shield: list, newpath): - with open(file, "r", encoding="utf-8") as f: - content = f.read() +def resolve_data_inputs(inputs: List[str], delete_after: bool) -> List[str]: + """Normalize -a inputs to a list of directories or profraw files. - for i in shield: - content = content.replace(i, "") + Files are auto-classified: + * .profraw (binary) -> kept as-is + * gcov binpack (binary) -> parsed to a directory + * hex-dump text log -> parsed to a directory + Directories are passed through. + """ + out: List[str] = [] + for entry in inputs: + if not os.path.exists(entry): + die(f"input not found: {entry}") + if os.path.isfile(entry): + abs_path = os.path.abspath(entry) + if is_profraw_file(abs_path): + out.append(abs_path) + elif is_gcov_binpack_file(abs_path): + out.append(parse_gcov_binpack(abs_path)) + if delete_after: + os.remove(entry) + else: + out.append(parse_gcda_hex_dump(abs_path)) + if delete_after: + os.remove(entry) + else: + out.append(os.path.abspath(entry)) + return out - new_content = content - if newpath is not None: - pattern = r"SF:([^\s]*?)/nuttx/include/nuttx" - matches = re.findall(pattern, content) - if matches: - new_content = content.replace(matches[0], newpath) +# ============================================================================= +# 6. TOOL RESOLUTION +# ============================================================================= - with open(file, "w", encoding="utf-8") as f: - f.write(new_content) +@dataclass +class GccTools: + gcov: str + lcov: str + genhtml: str -def copy_file_endswith(endswith, source, target): - for root, dirs, files in os.walk(source, topdown=True): - if target in root: - continue - for file in files: - if file.endswith(endswith): - src_file = os.path.join(root, file) - dst_file = os.path.join(target, os.path.relpath(src_file, source)) - os.makedirs(os.path.dirname(dst_file), exist_ok=True) - shutil.copy2(src_file, dst_file) +@dataclass +class LlvmTools: + profdata: str + cov: str + readelf: Optional[str] = None + lld: Optional[str] = None -def find_profraw_files(directory): - """Recursively find all .profraw files in a directory.""" - profraw_files = [] - for root, dirs, files in os.walk(directory): +def _which_first(candidates: List[str]) -> Optional[str]: + """Return the first candidate tool found on PATH, or None.""" + for cand in candidates: + found = shutil.which(cand) + if found: + return found + return None + + +def resolve_gcc_tools(elf: ElfInfo) -> GccTools: + """Resolve gcov/lcov/genhtml from PATH based on the ELF arch.""" + candidates = GCC_CROSS_PREFIXES.get(elf.e_machine, ["gcov"]) + gcov = _which_first(candidates) + if gcov is None: + die( + f"could not find gcov for {elf.arch_name} on PATH. " f"Tried: {candidates}." + ) + + lcov = _which_first(["lcov"]) + if lcov is None: + die("lcov not found on PATH. Install lcov.") + + genhtml = _which_first(["genhtml"]) + if genhtml is None: + die("genhtml not found on PATH. Install lcov (provides genhtml).") + + log(f"gcov: {gcov}") + log(f"lcov: {lcov}") + log(f"genhtml: {genhtml}") + return GccTools(gcov=gcov, lcov=lcov, genhtml=genhtml) + + +def resolve_llvm_tools() -> LlvmTools: + """Resolve llvm-profdata/llvm-cov/llvm-readelf/ld.lld from PATH.""" + profdata = _which_first(["llvm-profdata"]) + if profdata is None: + die("llvm-profdata not found on PATH. Install LLVM.") + cov = _which_first(["llvm-cov"]) + if cov is None: + die("llvm-cov not found on PATH. Install LLVM.") + readelf = _which_first(["llvm-readelf"]) + lld = _which_first(["ld.lld"]) + + log(f"llvm-profdata: {profdata}") + log(f"llvm-cov: {cov}") + log(f"llvm-readelf: {readelf or '(not found, covmap fix disabled)'}") + log(f"ld.lld: {lld or '(not found, covmap fix disabled)'}") + return LlvmTools(profdata=profdata, cov=cov, readelf=readelf, lld=lld) + + +# ============================================================================= +# 7. PATH-MAPPING (-b OLD=NEW) +# ============================================================================= + + +@dataclass +class PathMap: + pairs: List[Tuple[str, str]] = field(default_factory=list) + + @classmethod + def parse(cls, raw_args: Optional[List[str]]) -> "PathMap": + pairs: List[Tuple[str, str]] = [] + for raw in raw_args or []: + if "=" not in raw: + die(f"-b expects OLD=NEW form, got: {raw!r}") + old, new = raw.split("=", 1) + if not old: + die(f"-b OLD part must be non-empty: {raw!r}") + pairs.append((old, new)) + return cls(pairs=pairs) + + def lcov_substitute_args(self) -> List[str]: + """Render --substitute args for lcov.""" + out: List[str] = [] + for old, new in self.pairs: + # Use '#' as the sed delimiter to avoid clashing with '/' in paths + out.extend(["--substitute", f"s#{old}#{new}#g"]) + return out + + def llvm_path_equivalence_args(self) -> List[str]: + """Render -path-equivalence args for llvm-cov subcommands.""" + out: List[str] = [] + for old, new in self.pairs: + out.append(f"-path-equivalence={old},{new}") + return out + + +# ============================================================================= +# 8. UTILITIES +# ============================================================================= + + +def copy_files_with_suffix(suffix: str, source: str, target: str) -> None: + """Mirror files matching `*<suffix>` from source into target. + + Skips paths inside `target` to avoid recursive copies when target is a + descendant of source. + """ + target_abs = os.path.abspath(target) + for root, _, files in os.walk(source): + if os.path.abspath(root).startswith(target_abs): + continue for f in files: - fpath = os.path.join(root, f) - if f.endswith(".profraw") or is_profraw_file(fpath): - profraw_files.append(fpath) - return profraw_files + if f.endswith(suffix): + src = os.path.join(root, f) + rel = os.path.relpath(src, source) + dst = os.path.join(target, rel) + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copy2(src, dst) + + +def get_lcov_major_version(lcov_path: str) -> int: + """Return major version of installed lcov, or 1 if undetermined.""" + try: + result = subprocess.run( + [lcov_path, "--version"], + capture_output=True, + text=True, + timeout=5, + ) + m = re.search(r"LCOV version (\d+)", result.stdout) + if m: + return int(m.group(1)) + except (subprocess.TimeoutExpired, OSError, FileNotFoundError): + pass + return 1 + + +def lcov_info_has_records(info_file: str) -> bool: + """True if an lcov tracefile has at least one SF: record.""" + try: + with open(info_file, "r", encoding="utf-8", errors="replace") as fp: + for line in fp: + if line.startswith("SF:"): + return True + except (IOError, OSError): + pass + return False + + +# ============================================================================= +# 9. GCC PIPELINE: gcda -> .info -> HTML +# ============================================================================= + + +def _resolve_gcov_tool(gcov_path: str) -> Optional[str]: + """Find the gcov-tool binary that matches a given gcov binary. + + Strategy (first hit wins): + 1. Sibling next to the gcov binary, e.g. /opt/.../bin/gcov-13 → + /opt/.../bin/gcov-tool-13, or aarch64-none-elf-gcov → + aarch64-none-elf-gcov-tool. This avoids the Ubuntu pitfall where + /usr/bin/gcov is gcov-13 but /usr/bin/gcov-tool is the old gcov-tool-11. + 2. Plain `gcov-tool` on PATH (last resort). + """ + real = os.path.realpath(gcov_path) + base = os.path.basename(real) # e.g. x86_64-linux-gnu-gcov-13 + directory = os.path.dirname(real) + candidates: List[str] = [] + # Mirror "gcov[-suffix]" → "gcov-tool[-suffix]" + if "gcov" in base: + head, _, tail = base.partition("gcov") + candidates.append(os.path.join(directory, head + "gcov-tool" + tail)) -# ---- GCC pipeline (lcov + genhtml) ---- + # Also try the literal sibling next to the user-supplied path + candidates.append(gcov_path + "-tool") + + for c in candidates: + if os.path.isfile(c): + return c + + return shutil.which("gcov-tool") + + +def gcc_merge_gcda(tools: GccTools, gcda_dirs: List[str], out_dir: str) -> None: + """Merge multiple gcda directories into out_dir using `<gcov>-tool merge`. + + The first directory is copied to ``out_dir``; each subsequent directory + is merged in via ``gcov-tool merge``. ``out_dir`` must not exist (we make + a fresh copy every time so reruns are repeatable). + """ + if os.path.exists(out_dir): + shutil.rmtree(out_dir) + + merge_tool: Optional[str] = None + if len(gcda_dirs) > 1: + merge_tool = _resolve_gcov_tool(tools.gcov) + if merge_tool is None: + die( + f"need gcov-tool to merge {len(gcda_dirs)} -a inputs but " + "no matching gcov-tool was found on PATH. Install the " + "matching package (e.g. apt install gcc-13)." + ) + log(f"gcov-tool: {merge_tool}") + + shutil.copytree(gcda_dirs[0], out_dir) + for extra in gcda_dirs[1:]: + try: + run_cmd([merge_tool, "merge", out_dir, extra, "-o", out_dir]) + except subprocess.CalledProcessError: + die( + f"gcov-tool merge failed for {extra}. Common causes: " + "(1) version mismatch between gcov-tool and the .gcda files " + "('incorrect gcov version' messages above) — install the " + "matching gcov-tool so it is found on PATH, " + "(2) the input was already merged in. " + f"merge_tool={merge_tool}" + ) -def run_lcov(data_dir, gcov_tool): - output = data_dir + ".info" - command = [ - "lcov", +def gcc_run_lcov(tools: GccTools, data_dir: str, path_map: PathMap) -> str: + """Run lcov capture, returning path to the produced .info file.""" + info_file = data_dir + ".info" + cmd = [ + tools.lcov, "-c", "-o", - output, + info_file, "--rc", "lcov_branch_coverage=1", "--gcov-tool", - gcov_tool, + tools.gcov, "--ignore-errors", "gcov", "--directory", data_dir, ] - print(command) - subprocess.run( - command, - check=True, - stdout=sys.stdout, - stderr=sys.stdout, + cmd.extend(path_map.lcov_substitute_args()) + run_cmd(cmd) + return info_file + + +def gcc_run_genhtml(tools: GccTools, info_file: str, report_dir: str) -> None: + ignore_errors = "source" + if get_lcov_major_version(tools.lcov) >= 2: + ignore_errors = "source,category" + + run_cmd( + [ + tools.genhtml, + "--branch-coverage", + "-o", + report_dir, + "--ignore-errors", + ignore_errors, + info_file, + ] ) - return output + +def run_gcc_pipeline( + gcda_inputs: List[str], + gcno_dir: str, + result_dir: str, + elf: ElfInfo, + path_map: PathMap, +) -> None: + tools = resolve_gcc_tools(elf) + + data_dir = os.path.join(result_dir, "data") + report_dir = os.path.join(result_dir, "report") + + gcc_merge_gcda(tools, gcda_inputs, data_dir) + copy_files_with_suffix(".gcno", gcno_dir, data_dir) + + info_file = gcc_run_lcov(tools, data_dir, path_map) + + if not lcov_info_has_records(info_file): + warn( + f"lcov produced no coverage records in {info_file}. This usually " + "means the local gcov tool cannot decode the collected .gcda " + "files (toolchain version mismatch — see 'version ... prefer ...' " + "warnings above). Skipping HTML report generation." + ) + return + + try: + gcc_run_genhtml(tools, info_file, report_dir) + except subprocess.CalledProcessError: + die("failed to generate coverage report.") + + print_report_link(report_dir) + + +# ============================================================================= +# 10. CLANG PIPELINE: profraw -> profdata -> HTML +# ============================================================================= -def get_lcov_major_version(): - """Get the major version number of the installed lcov/genhtml.""" +def clang_merge_profraw( + tools: LlvmTools, profraw_files: List[str], out_profdata: str +) -> str: + cmd = [tools.profdata, "merge", "-sparse", "-o", out_profdata] + cmd.extend(profraw_files) + run_cmd(cmd) + return out_profdata + + +def _check_covmap_one(readelf: str, path: str) -> Optional[str]: + """Return ``path`` if its ELF has a ``__llvm_covmap`` section, else None.""" try: result = subprocess.run( - ["lcov", "--version"], capture_output=True, text=True, timeout=5 + [readelf, "-S", path], + capture_output=True, + text=True, + timeout=10, ) - match = re.search(r"LCOV version (\d+)", result.stdout) - if match: - return int(match.group(1)) - except (subprocess.TimeoutExpired, OSError, FileNotFoundError): - pass - return 1 + except (subprocess.TimeoutExpired, OSError): + return None + return path if "__llvm_covmap" in result.stdout else None -def lcov_info_has_records(info_file): - """Return True if an lcov .info tracefile contains at least one 'SF:' record. +def clang_find_covmap_objects(build_dir: str, readelf: str) -> List[str]: + """Find all .o files containing ``__llvm_covmap`` sections. - When the local gcov tool cannot decode any .gcda file (e.g. because the - build toolchain produced a newer gcov version than the host tool supports), - lcov still exits 0 (due to --ignore-errors gcov) but writes an info file - with zero source records. Running genhtml on such a file always fails with - 'no valid records found in tracefile', which is a pipeline-level symptom - of an environment issue, not a real coverage regression. + Each check spawns ``llvm-readelf -S``, so the work is I/O / subprocess + bound — a thread pool parallelises it without the fork/pickle overhead of + processes. Tiny trees run inline to skip pool spin-up. Output order is + not preserved — safe because ``ld.lld -r`` is order-insensitive. """ - try: - with open(info_file, "r", encoding="utf-8", errors="replace") as f: - for line in f: - if line.startswith("SF:"): - return True - except (IOError, OSError): - return False - return False + paths: List[str] = [] + for root, _, files in os.walk(build_dir): + for f in files: + if f.endswith(".o"): + paths.append(os.path.join(root, f)) + if not paths: + return [] -def run_genhtml(info, report): - # 'category' is only supported by lcov >= 2 - ignore_errors = "source" - if get_lcov_major_version() >= 2: - ignore_errors = "source,category" + if len(paths) < 32: + objects: List[str] = [] + for p in paths: + hit = _check_covmap_one(readelf, p) + if hit is not None: + objects.append(hit) + return objects - cmd = [ - "genhtml", - "--branch-coverage", - "-o", - report, - "--ignore-errors", - ignore_errors, - info, - ] - print(cmd) - subprocess.run( - cmd, - check=True, - stdout=sys.stdout, - stderr=sys.stdout, - ) + workers = min(32, (os.cpu_count() or 4) * 4) + objects = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as ex: + for hit in ex.map(lambda p: _check_covmap_one(readelf, p), paths): + if hit is not None: + objects.append(hit) + return objects -def run_merge(gcda_dir1, gcda_dir2, output, merge_tool): - command = [ - merge_tool, - "merge", - gcda_dir1, - gcda_dir2, - "-o", - output, - ] - print(command) - subprocess.run( - command, - check=True, - stdout=sys.stdout, - stderr=sys.stdout, - ) +def clang_link_relocatable(objects: List[str], out_path: str, lld: str) -> str: + log(f"linking {len(objects)} objects into {out_path} ...") + run_cmd([lld, "-r", "-o", out_path] + objects) + return out_path -# ---- Clang pipeline (llvm-profdata + llvm-cov) ---- +def _fingerprint_paths(paths: List[str]) -> List[List]: + """(path, mtime_ns, size) records keying the relocatable cache.""" + records = [] + for p in paths: + try: + st = os.stat(p) + except OSError: + continue + records.append([p, st.st_mtime_ns, st.st_size]) + records.sort() + return records + + +def _load_cache_fingerprint(cache_path: str) -> Optional[dict]: + try: + with open(cache_path) as fp: + return json.load(fp) + except (OSError, ValueError): + return None -def find_llvm_tools(gcov_tool): - """Derive llvm-profdata, llvm-cov, llvm-readelf, and ld.lld paths from the -t argument. +def _save_cache_fingerprint(cache_path: str, payload: dict) -> None: + try: + with open(cache_path, "w") as fp: + json.dump(payload, fp) + except OSError as e: + warn(f"failed to write cache fingerprint {cache_path}: {e}") + + +def _find_prebuilt_reloc(build_dir: str) -> Optional[str]: + """Return a pre-built relocatable covmap object if -s points at one. - The user passes -t with a path like: - /path/to/bin/clang -> look for llvm-profdata, llvm-cov - /path/to/bin/aarch64-none-elf-gcov -> try llvm-profdata in same dir - llvm-cov -> use as-is + Two accepted forms, both produced once and persisted so reports can + be generated long after the build tree (the per-source ``.o`` files) + is gone: - Strategy: - 1. If gcov_tool dir contains llvm-profdata, use it - 2. Try replacing the tool basename with llvm-profdata / llvm-cov - 3. Fall back to PATH lookup + * ``-s path/to/<obj>.o`` -> the file itself + * ``-s path/to/dir`` -> ``dir/nuttx_llvm_covmap.o`` or + ``dir/coverage_all.o`` if present + + ``nuttx_llvm_covmap.o`` is the slim LLVM coverage object emitted by the + build (see tools/gen_llvm_covmap.py); ``coverage_all.o`` is the full + relocatable that tools/gcov.py itself caches. Returning a path here lets + ``clang_resolve_cov_binary`` skip the readelf scan + ld.lld link. """ - tool_dir = os.path.dirname(os.path.abspath(gcov_tool)) - - profdata = None - llvm_cov = None - llvm_readelf = None - lld = None - - # Check same directory - for candidate in os.listdir(tool_dir) if os.path.isdir(tool_dir) else []: - full = os.path.join(tool_dir, candidate) - if candidate == "llvm-profdata" or candidate.endswith("-llvm-profdata"): - profdata = full - if candidate == "llvm-cov" or candidate.endswith("-llvm-cov"): - llvm_cov = full - if candidate == "llvm-readelf" or candidate.endswith("-llvm-readelf"): - llvm_readelf = full - if candidate == "ld.lld": - lld = full - - # Fall back to PATH - if profdata is None: - profdata = shutil.which("llvm-profdata") - if llvm_cov is None: - llvm_cov = shutil.which("llvm-cov") - if llvm_readelf is None: - llvm_readelf = shutil.which("llvm-readelf") - if lld is None: - lld = shutil.which("ld.lld") + if build_dir.endswith(".o") and os.path.isfile(build_dir): + return build_dir + if os.path.isdir(build_dir): + for name in ("nuttx_llvm_covmap.o", "coverage_all.o"): + candidate = os.path.join(build_dir, name) + if os.path.isfile(candidate): + return candidate + return None + + +def clang_resolve_cov_binary( + tools: LlvmTools, elf_path: str, build_dir: str, result_dir: str +) -> str: + """Pick the binary llvm-cov should consult. + + On lld-linked NuttX binaries, __llvm_covfun/__llvm_covmap may sit at + VMA=0, so llvm-cov refuses to read them from the final ELF; we link + a relocatable .o from all source objects carrying covmap sections. + + Resolution order: + 1. A pre-built ``coverage_all.o`` reachable via ``-s`` (file or dir). + Used as-is -- no build tree required. This is how persisted + coverage data is reported after the ``out/`` build dir is deleted. + 2. Scan ``build_dir`` for ``.o`` files with covmap sections and link + them on the fly (result cached via a ``.fingerprint`` sidecar). + 3. Fall back to the ELF binary directly (usually fails on lld VMA=0). + """ + prebuilt = _find_prebuilt_reloc(build_dir) + if prebuilt is not None: + log(f"using pre-built relocatable object for report: {prebuilt}") + return prebuilt + + if not (tools.readelf and tools.lld): + if not tools.readelf: + log("note: llvm-readelf not found, skipping relocatable object build.") + if not tools.lld: + log("note: ld.lld not found, skipping relocatable object build.") + return elf_path + + reloc = os.path.join(result_dir, "coverage_all.o") + cache_path = reloc + ".fingerprint" + + # Cache hit: skip both readelf scan and ld.lld link. + if os.path.isfile(reloc) and os.path.isfile(cache_path): + cached = _load_cache_fingerprint(cache_path) + if cached and cached.get("build_dir") == os.path.abspath(build_dir): + cached_records = cached.get("objects") or [] + cached_paths = [r[0] for r in cached_records] + current_records = _fingerprint_paths(cached_paths) + if current_records == cached_records: + log( + f"cache hit: reusing {reloc} " + f"({len(cached_records)} objects unchanged)" + ) + return reloc + log("cache miss: covmap object set changed, rebuilding.") - return profdata, llvm_cov, llvm_readelf, lld + log("scanning for .o files with __llvm_covmap sections ...") + objects = clang_find_covmap_objects(build_dir, tools.readelf) + if not objects: + log("no .o files with __llvm_covmap found, using ELF binary directly.") + return elf_path + try: + clang_link_relocatable(objects, reloc, tools.lld) + log(f"using relocatable object for report: {reloc}") + except subprocess.CalledProcessError as e: + warn(f"failed to build relocatable object: {e}; falling back to ELF.") + return elf_path + + _save_cache_fingerprint( + cache_path, + { + "build_dir": os.path.abspath(build_dir), + "objects": _fingerprint_paths(objects), + }, + ) -def run_llvm_profdata_merge(profraw_files, output_profdata, profdata_tool): - """Merge one or more .profraw files into a single .profdata.""" - cmd = [profdata_tool, "merge", "-sparse", "-o", output_profdata] + profraw_files - print(cmd) - subprocess.run(cmd, check=True, stdout=sys.stdout, stderr=sys.stdout) - return output_profdata + return reloc -def run_llvm_cov_export(profdata, binary, llvm_cov_tool, output_info): - """Export coverage data in lcov format for genhtml consumption.""" - cmd = [ - llvm_cov_tool, - "export", - binary, - "-instr-profile=" + profdata, - "-format=lcov", - ] - print(cmd) - with open(output_info, "w") as f: - subprocess.run(cmd, check=True, stdout=f, stderr=sys.stdout) - return output_info +def clang_run_report( + tools: LlvmTools, profdata: str, binary: str, path_map: PathMap +) -> None: + cmd = [tools.cov, "report", binary, f"-instr-profile={profdata}"] + cmd.extend(path_map.llvm_path_equivalence_args()) + run_cmd(cmd) -def run_llvm_cov_show_html(profdata, binary, llvm_cov_tool, report_dir): - """Generate HTML report directly using llvm-cov show.""" +def clang_run_show_html( + tools: LlvmTools, profdata: str, binary: str, report_dir: str, path_map: PathMap +) -> None: cmd = [ - llvm_cov_tool, + tools.cov, "show", binary, - "-instr-profile=" + profdata, + f"-instr-profile={profdata}", "-format=html", - "-output-dir=" + report_dir, + f"-output-dir={report_dir}", "-show-line-counts-or-regions", "-show-branches=count", "-show-directory-coverage", ] - print(cmd) - subprocess.run(cmd, check=True, stdout=sys.stdout, stderr=sys.stdout) + cmd.extend(path_map.llvm_path_equivalence_args()) + run_cmd(cmd) -def run_llvm_cov_report(profdata, binary, llvm_cov_tool): - """Print coverage summary to stdout.""" +def clang_run_export_lcov( + tools: LlvmTools, profdata: str, binary: str, out_info: str, path_map: PathMap +) -> str: cmd = [ - llvm_cov_tool, - "report", + tools.cov, + "export", binary, - "-instr-profile=" + profdata, + f"-instr-profile={profdata}", + "-format=lcov", ] - print(cmd) - subprocess.run(cmd, check=True, stdout=sys.stdout, stderr=sys.stdout) + cmd.extend(path_map.llvm_path_equivalence_args()) + log(" ".join(cmd)) + with open(out_info, "w") as fp: + subprocess.run(cmd, check=True, stdout=fp, stderr=sys.stdout) + return out_info + + +def run_clang_pipeline( + data_inputs: List[str], + gcno_dir: str, + result_dir: str, + elf_path: str, + path_map: PathMap, +) -> None: + tools = resolve_llvm_tools() + data_dir = os.path.join(result_dir, "data") + report_dir = os.path.join(result_dir, "report") + os.makedirs(data_dir, exist_ok=True) -def find_covmap_objects(build_dir, readelf_tool): - """Find all .o files containing __llvm_covmap sections. + # Collect all profraw files + profraw_files: List[str] = [] + for entry in data_inputs: + if os.path.isfile(entry): + profraw_files.append(entry) + elif os.path.isdir(entry): + profraw_files.extend(find_profraw_files(entry)) + if not profraw_files: + die("no .profraw files found in the specified -a paths.") + log(f"found {len(profraw_files)} profraw file(s)") - After final linking with lld, covfun/covmap sections may have VMA=0, - making llvm-cov unable to read the final ELF directly. Building a - relocatable object from the original .o files is the workaround. - """ - objects = [] - for root, dirs, files in os.walk(build_dir): - for f in files: - if not f.endswith(".o"): - continue - fpath = os.path.join(root, f) - try: - result = subprocess.run( - [readelf_tool, "-S", fpath], - capture_output=True, - text=True, - timeout=10, - ) - if "__llvm_covmap" in result.stdout: - objects.append(fpath) - except (subprocess.TimeoutExpired, OSError): - continue - return objects + # profraw -> profdata + profdata = os.path.join(data_dir, "merged.profdata") + clang_merge_profraw(tools, profraw_files, profdata) + + cov_binary = clang_resolve_cov_binary(tools, elf_path, gcno_dir, result_dir) + log("\n=== Coverage Summary ===") + clang_run_report(tools, profdata, cov_binary, path_map) + + # HTML: prefer llvm-cov show; fall back to genhtml(lcov-format export). + try: + clang_run_show_html(tools, profdata, cov_binary, report_dir, path_map) + ok = True + except subprocess.CalledProcessError as e: + warn(f"llvm-cov show HTML failed: {e}; trying genhtml fallback.") + ok = False -def build_relocatable_object(objects, output_path, lld_tool): - """Link .o files into a single relocatable object with ld.lld -r.""" - cmd = [lld_tool, "-r", "-o", output_path] + objects - print(f"Linking {len(objects)} objects into {output_path} ...") - subprocess.run(cmd, check=True, stdout=sys.stdout, stderr=sys.stdout) - return output_path + if not ok: + genhtml = shutil.which("genhtml") + if genhtml is None: + die("HTML generation failed and genhtml is not available.") + try: + cov_info = os.path.join(data_dir, "coverage.info") + clang_run_export_lcov(tools, profdata, cov_binary, cov_info, path_map) + run_cmd( + [ + genhtml, + "--branch-coverage", + "-o", + report_dir, + "--ignore-errors", + "source", + cov_info, + ] + ) + ok = True + except subprocess.CalledProcessError as e: + die(f"genhtml fallback also failed: {e}") + + # Only advertise the report if it actually produced an index page. + if ok and os.path.isfile(os.path.join(report_dir, "index.html")): + print_report_link(report_dir) + else: + warn("HTML report was not generated; no report link to show.") -# ---- Main ---- +# ============================================================================= +# 11. CLI + MAIN +# ============================================================================= -def arg_parser(): +def build_arg_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description="Code coverage generation tool (supports GCC gcov and Clang source-based coverage).", - add_help=False, + description=( + "Code coverage report generator (auto-detects GCC vs Clang from " + "the ELF binary)." + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog="""\ +Examples: + # Clang (LLVM source-based coverage) + gcov.py -e build/nuttx -s build -a default.profraw -o report + + # GCC (lcov + genhtml) + gcov.py -e build/nuttx -s build -a coverage_dir -o report + + # With path remap (works for both backends) + gcov.py -b /old/build/path=/new/local/path \\ + -e build/nuttx -s build -a default.profraw -o report + +Tools are resolved from PATH (gcov/lcov/genhtml for GCC; llvm-profdata/ +llvm-cov/llvm-readelf/ld.lld for Clang); make sure the toolchain is on PATH. +""", ) parser.add_argument( - "-t", - dest="gcov_tool", - help="Path to gcov tool (gcc) or clang binary / llvm-cov (clang)", - ) - parser.add_argument("-b", dest="base_dir", help="Compile base directory") - parser.add_argument("--debug", action="store_true", help="Enable debug mode") - parser.add_argument( - "--delete", - action="store_true", - help="Delete gcda/profraw files after processing", + "-e", + "--elf", + dest="elf_binary", + required=True, + help="Path to the ELF binary that produced the coverage data. " + "The toolchain (gcc/clang) and target architecture are inferred " + "from this file.", ) parser.add_argument( "-s", dest="gcno_dir", - default=None, - help="Directory containing gcno files (gcc) or build output directory (clang). " - "For clang, defaults to the directory of the ELF binary (-e).", + required=True, + help="Directory with .gcno files (gcc) or build output containing " + ".o files for the covmap relocatable workaround (clang). For " + "clang it may also point directly at a pre-built LLVM coverage object " + "(nuttx_llvm_covmap.o or coverage_all.o, as produced by the build), " + "or a directory containing one, to report from persisted coverage " + "data without the build tree.", ) parser.add_argument( "-a", - dest="gcda_dir", - default=".", + dest="data_inputs", + default=["."], nargs="+", - help="Directory/file containing gcda (gcc) or profraw (clang) data", - ) - parser.add_argument( - "-x", - dest="only_copy", - action="store_true", - help="Only copy/extract data files, don't generate report", + help="Directories or files holding .gcda (gcc) or .profraw (clang) " + "data. Hex-dump text logs and gcov binary packs are supported.", ) parser.add_argument( "-o", dest="result_dir", default="gcov", - help="Directory to store coverage data and report", + help="Output directory for intermediate data and the HTML report.", ) parser.add_argument( - "-e", - "--elf", - dest="elf_binary", - default=None, - help="Path to ELF binary (required for clang coverage report generation)", + "-b", + dest="path_map", + action="append", + metavar="OLD=NEW", + help="Rewrite paths recorded in coverage data: replace OLD with NEW. " + "Maps to lcov --substitute / llvm-cov -path-equivalence. " + "Repeatable.", ) parser.add_argument( - "--toolchain", - dest="toolchain", - choices=["gcc", "clang", "auto"], - default="auto", - help="Force toolchain type (default: auto-detect from data files)", + "--debug", + action="store_true", + help="Redirect stdout to <result_dir>/debug.log", ) - - return parser.parse_args() - - -def main_gcc(args, gcda_dirs, gcno_dir, result_dir): - """GCC coverage pipeline: lcov + genhtml.""" - - merge_tool = args.gcov_tool + "-tool" - data_dir = os.path.join(result_dir, "data") - report_dir = os.path.join(result_dir, "report") - - # lcov tool is required - if shutil.which("lcov") is None: - print("Error: lcov is not detected, please install lcov.") - sys.exit(1) - - # Merge all gcda directories - shutil.copytree(gcda_dirs[0], data_dir) - for gcda_dir in gcda_dirs[1:]: - run_merge(data_dir, gcda_dir, data_dir, merge_tool) - - # Copy gcno files and run lcov - copy_file_endswith(".gcno", gcno_dir, data_dir) - coverage_file = run_lcov(data_dir, args.gcov_tool) - - if args.only_copy: - sys.exit(0) - - # If lcov produced an .info tracefile with no source records, the local - # gcov tool could not decode the collected .gcda files - usually a - # toolchain version mismatch (look for 'version Bxx*, prefer Byy*' - # warnings from lcov above). Running genhtml on an empty tracefile always - # fails with 'no valid records found in tracefile' and aborts the whole - # pipeline even though there is nothing actionable in the coverage data. - # Print a clear diagnostic and skip the HTML step instead of failing. - if not lcov_info_has_records(coverage_file): - print( - "Warning: lcov produced no coverage records in " - f"{coverage_file}. This usually means the local gcov tool " - "cannot decode the collected .gcda files (toolchain version " - "mismatch - see 'version ... prefer ...' warnings above). " - "Skipping HTML report generation." - ) - return - - try: - run_genhtml(coverage_file, report_dir) - print( - "Copy the following link and open it in the browser to view the coverage report:" - ) - print(f"file://{os.path.join(report_dir, 'index.html')}") - except subprocess.CalledProcessError: - print("Failed to generate coverage report.") - sys.exit(1) - - -def main_clang(args, gcda_dirs, gcno_dir, result_dir): - """Clang coverage pipeline: llvm-profdata merge + llvm-cov. - - When the build output directory (-s) contains .o files with __llvm_covmap - sections, automatically builds a relocatable object (ld.lld -r) and uses - it for report generation. This works around the issue where lld-linked - final ELF has covfun/covmap sections with VMA=0. - """ - - data_dir = os.path.join(result_dir, "data") - report_dir = os.path.join(result_dir, "report") - os.makedirs(data_dir, exist_ok=True) - - # Find LLVM tools - profdata_tool, llvm_cov_tool, llvm_readelf, lld_tool = find_llvm_tools( - args.gcov_tool + parser.add_argument( + "--delete", + action="store_true", + help="Delete dump files (hex/binpack) after they have been parsed.", ) + return parser - if profdata_tool is None: - print("Error: llvm-profdata not found. Install LLVM or specify path via -t.") - sys.exit(1) - if llvm_cov_tool is None: - print("Error: llvm-cov not found. Install LLVM or specify path via -t.") - sys.exit(1) - - print(f"Using llvm-profdata: {profdata_tool}") - print(f"Using llvm-cov: {llvm_cov_tool}") - - # Collect all profraw files - profraw_files = [] - for d in gcda_dirs: - if os.path.isfile(d): - profraw_files.append(d) - elif os.path.isdir(d): - profraw_files.extend(find_profraw_files(d)) - - if not profraw_files: - print("Error: No .profraw files found in the specified paths.") - sys.exit(1) - - print(f"Found {len(profraw_files)} profraw file(s)") - - # Merge profraw -> profdata - profdata_file = os.path.join(data_dir, "merged.profdata") - run_llvm_profdata_merge(profraw_files, profdata_file, profdata_tool) - - if args.only_copy: - sys.exit(0) - - # Determine ELF binary path - elf_binary = args.elf_binary - if elf_binary is None: - # Try to find nuttx ELF in gcno_dir (build output directory) - candidates = [ - os.path.join(gcno_dir, "nuttx"), - os.path.join(gcno_dir, "nuttx.elf"), - ] - # Also search for vela_*.elf - if os.path.isdir(gcno_dir): - for f in os.listdir(gcno_dir): - if f.endswith(".elf"): - candidates.append(os.path.join(gcno_dir, f)) - - for c in candidates: - if os.path.isfile(c): - elf_binary = c - break - - if elf_binary is None: - print( - "Error: ELF binary not found. Specify with -e/--elf or ensure " - "'nuttx' exists in the -s directory." - ) - sys.exit(1) - - elf_binary = os.path.abspath(elf_binary) - print(f"Using ELF binary: {elf_binary}") - - # Build relocatable object from .o files if tools are available. - # This is needed because lld-linked ELF may have covfun/covmap at VMA=0. - cov_binary = elf_binary - if llvm_readelf and lld_tool: - print("\nScanning for .o files with __llvm_covmap sections ...") - covmap_objects = find_covmap_objects(gcno_dir, llvm_readelf) - if covmap_objects: - reloc_obj = os.path.join(result_dir, "coverage_all.o") - try: - build_relocatable_object(covmap_objects, reloc_obj, lld_tool) - cov_binary = reloc_obj - print(f"Using relocatable object for report: {cov_binary}") - except subprocess.CalledProcessError as e: - print(f"Warning: failed to build relocatable object: {e}") - print("Falling back to ELF binary.") - else: - print("No .o files with __llvm_covmap found, using ELF binary directly.") - else: - if llvm_readelf is None: - print("Note: llvm-readelf not found, skipping relocatable object build.") - if lld_tool is None: - print("Note: ld.lld not found, skipping relocatable object build.") - # Print summary report - print("\n=== Coverage Summary ===") - run_llvm_cov_report(profdata_file, cov_binary, llvm_cov_tool) +def main() -> None: + args = build_arg_parser().parse_args() - # Generate HTML report: prefer llvm-cov show (supports directory grouping), - # fall back to genhtml (lcov) if llvm-cov show fails. - html_ok = False - - try: - run_llvm_cov_show_html(profdata_file, cov_binary, llvm_cov_tool, report_dir) - html_ok = True - except subprocess.CalledProcessError as e: - print(f"Warning: llvm-cov show HTML failed: {e}, falling back to genhtml...") - - if not html_ok and shutil.which("genhtml") is not None: - try: - coverage_info = os.path.join(data_dir, "coverage.info") - run_llvm_cov_export(profdata_file, cov_binary, llvm_cov_tool, coverage_info) - run_genhtml(coverage_info, report_dir) - html_ok = True - except subprocess.CalledProcessError as e: - print(f"Warning: genhtml also failed: {e}") - - if html_ok: - print( - "\nCopy the following link and open it in the browser to view the coverage report:" - ) - print(f"file://{os.path.join(report_dir, 'index.html')}") - else: - print("HTML report generation failed. Coverage summary was printed above.") - sys.exit(1) - - -def main(): - args = arg_parser() - - root_dir = os.getcwd() result_dir = os.path.abspath(args.result_dir) - os.makedirs(result_dir, exist_ok=True) if args.debug: - debug_file = os.path.join(result_dir, "debug.log") - sys.stdout = open(debug_file, "w+") - - # Parse dump files (hex dump, binary pack, or profraw) - gcda_dirs = [] - for i in args.gcda_dir: - if os.path.isfile(i): - abs_path = os.path.join(root_dir, i) if not os.path.isabs(i) else i - if is_profraw_file(abs_path): - # Binary profraw file (clang), use directly - gcda_dirs.append(os.path.abspath(i)) - elif is_gcov_binpack_file(abs_path): - # Binary pack file from COVERAGE_MINI gcov_binfile_dump - gcda_dirs.append(parse_gcov_binpack(abs_path)) - if args.delete: - os.remove(i) - else: - # Text file: try to parse as hex dump log - gcda_dirs.append(parse_gcda_data(abs_path)) - if args.delete: - os.remove(i) - else: - gcda_dirs.append(os.path.abspath(i)) + sys.stdout = open(os.path.join(result_dir, "debug.log"), "w+") - # Detect toolchain - if args.toolchain == "auto": - toolchain = detect_toolchain(gcda_dirs) - else: - toolchain = args.toolchain + elf_path = os.path.abspath(args.elf_binary) + if not os.path.isfile(elf_path): + die(f"ELF binary not found: {elf_path}") + elf = inspect_elf(elf_path) + log(f"ELF: {elf_path}") + log(f" toolchain: {elf.toolchain}") + log(f" arch: {elf.arch_name} (e_machine={elf.e_machine})") - print(f"Detected toolchain: {toolchain}") + # Resolve gcno_dir / build_dir (always provided via required -s). + gcno_dir = os.path.abspath(args.gcno_dir) - # Resolve gcno_dir: for clang, default to ELF binary's directory - if args.gcno_dir is not None: - gcno_dir = os.path.abspath(args.gcno_dir) - elif toolchain == "clang" and args.elf_binary is not None: - gcno_dir = os.path.abspath(os.path.dirname(args.elf_binary)) - else: - gcno_dir = os.path.abspath(".") + path_map = PathMap.parse(args.path_map) + + # Normalize -a inputs (parse hex dumps / binpacks if needed) + data_inputs = resolve_data_inputs(args.data_inputs, args.delete) - if toolchain == "clang": - main_clang(args, gcda_dirs, gcno_dir, result_dir) + if elf.toolchain == "clang": + run_clang_pipeline(data_inputs, gcno_dir, result_dir, elf_path, path_map) else: - main_gcc(args, gcda_dirs, gcno_dir, result_dir) + run_gcc_pipeline(data_inputs, gcno_dir, result_dir, elf, path_map) if __name__ == "__main__":
