https://github.com/python/cpython/commit/5ec03cf3b086fd01614cbd97bb99d45aac3668fe
commit: 5ec03cf3b086fd01614cbd97bb99d45aac3668fe
branch: main
author: dgpb <[email protected]>
committer: gpshead <[email protected]>
date: 2025-11-27T22:22:21Z
summary:
gh-133228: c-analyzer clang preprocessor (GH-133229)
* impl
* included 2 failures to tsvs next to similar entries
* added fix/hack for curses.h fails
* fix leftover from debug
files:
A Tools/c-analyzer/c_parser/preprocessor/clang.py
M Tools/c-analyzer/c_parser/preprocessor/__init__.py
M Tools/c-analyzer/c_parser/preprocessor/gcc.py
M Tools/c-analyzer/cpython/_parser.py
M Tools/c-analyzer/cpython/globals-to-fix.tsv
M Tools/c-analyzer/cpython/ignored.tsv
diff --git a/Tools/c-analyzer/c_parser/preprocessor/__init__.py
b/Tools/c-analyzer/c_parser/preprocessor/__init__.py
index 30a86cbd7dc494..f8d2f805cb1b19 100644
--- a/Tools/c-analyzer/c_parser/preprocessor/__init__.py
+++ b/Tools/c-analyzer/c_parser/preprocessor/__init__.py
@@ -16,6 +16,7 @@
from . import (
pure as _pure,
gcc as _gcc,
+ clang as _clang,
)
@@ -234,7 +235,7 @@ def handling_errors(ignore_exc=None, *, log_err=None):
'bcpp': None,
# aliases/extras:
'gcc': _gcc.preprocess,
- 'clang': None,
+ 'clang': _clang.preprocess,
}
diff --git a/Tools/c-analyzer/c_parser/preprocessor/clang.py
b/Tools/c-analyzer/c_parser/preprocessor/clang.py
new file mode 100644
index 00000000000000..574a23f8f6d6f9
--- /dev/null
+++ b/Tools/c-analyzer/c_parser/preprocessor/clang.py
@@ -0,0 +1,104 @@
+import os.path
+import re, sys
+
+from . import common as _common
+from . import gcc as _gcc
+
+_normpath = _gcc._normpath
+
+TOOL = 'clang'
+
+META_FILES = {
+ '<built-in>',
+ '<command line>',
+}
+
+
+def preprocess(filename,
+ incldirs=None,
+ includes=None,
+ macros=None,
+ samefiles=None,
+ cwd=None,
+ ):
+ if not cwd or not os.path.isabs(cwd):
+ cwd = os.path.abspath(cwd or '.')
+ filename = _normpath(filename, cwd)
+
+ postargs = _gcc.POST_ARGS
+ basename = os.path.basename(filename)
+ dirname = os.path.basename(os.path.dirname(filename))
+ if (basename not in _gcc.FILES_WITHOUT_INTERNAL_CAPI
+ and dirname not in _gcc.DIRS_WITHOUT_INTERNAL_CAPI):
+ postargs += ('-DPy_BUILD_CORE=1',)
+
+ text = _common.preprocess(
+ TOOL,
+ filename,
+ incldirs=incldirs,
+ includes=includes,
+ macros=macros,
+ #preargs=PRE_ARGS,
+ postargs=postargs,
+ executable=['clang'],
+ compiler='unix',
+ cwd=cwd,
+ )
+ return _iter_lines(text, filename, samefiles, cwd)
+
+
+EXIT_MARKERS = {'# 2 "<built-in>" 2', '# 3 "<built-in>" 2', '# 4 "<built-in>"
2'}
+
+
+def _iter_lines(text, reqfile, samefiles, cwd, raw=False):
+ lines = iter(text.splitlines())
+
+ # The first line is special.
+ # The subsequent lines are consistent.
+ firstlines = [
+ f'# 1 "{reqfile}"',
+ '# 1 "<built-in>" 1',
+ '# 1 "<built-in>" 3',
+ '# 370 "<built-in>" 3',
+ '# 1 "<command line>" 1',
+ '# 1 "<built-in>" 2',
+ ]
+ for expected in firstlines:
+ line = next(lines)
+ if line != expected:
+ raise NotImplementedError((line, expected))
+
+ # Do all the CLI-provided includes.
+ filter_reqfile = (lambda f: _gcc._filter_reqfile(f, reqfile, samefiles))
+ make_info = (lambda lno: _common.FileInfo(reqfile, lno))
+ last = None
+ for line in lines:
+ assert last != reqfile, (last,)
+ # NOTE:condition is clang specific
+ if not line:
+ continue
+ lno, included, flags = _gcc._parse_marker_line(line, reqfile)
+ if not included:
+ raise NotImplementedError((line,))
+ if included == reqfile:
+ # This will be the last one.
+ assert 2 in flags, (line, flags)
+ else:
+ # NOTE:first condition is specific to clang
+ if _normpath(included, cwd) == reqfile:
+ assert 1 in flags or 2 in flags, (line, flags, included,
reqfile)
+ else:
+ assert 1 in flags, (line, flags, included, reqfile)
+ yield from _gcc._iter_top_include_lines(
+ lines,
+ _normpath(included, cwd),
+ cwd,
+ filter_reqfile,
+ make_info,
+ raw,
+ EXIT_MARKERS
+ )
+ last = included
+ # The last one is always the requested file.
+ # NOTE:_normpath is clang specific
+ assert _normpath(included, cwd) == reqfile, (line,)
diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py
b/Tools/c-analyzer/c_parser/preprocessor/gcc.py
index d20cd19f6e6d5e..4a55a1a24ee1be 100644
--- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py
+++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py
@@ -65,6 +65,8 @@
'-E',
)
+EXIT_MARKERS = {'# 0 "<command-line>" 2', '# 1 "<command-line>" 2'}
+
def preprocess(filename,
incldirs=None,
@@ -138,6 +140,7 @@ def _iter_lines(text, reqfile, samefiles, cwd, raw=False):
filter_reqfile,
make_info,
raw,
+ EXIT_MARKERS
)
last = included
# The last one is always the requested file.
@@ -146,7 +149,7 @@ def _iter_lines(text, reqfile, samefiles, cwd, raw=False):
def _iter_top_include_lines(lines, topfile, cwd,
filter_reqfile, make_info,
- raw):
+ raw, exit_markers):
partial = 0 # depth
files = [topfile]
# We start at 1 in case there are source lines (including blank ones)
@@ -154,12 +157,20 @@ def _iter_top_include_lines(lines, topfile, cwd,
# _parse_marker_line() that the preprocessor reported lno as 1.
lno = 1
for line in lines:
- if line == '# 0 "<command-line>" 2' or line == '# 1 "<command-line>"
2':
+ if line in exit_markers:
# We're done with this top-level include.
return
_lno, included, flags = _parse_marker_line(line)
if included:
+ # HACK:
+ # Mixes curses.h and ncurses.h marker lines
+ # gcc silently passes this, while clang fails
+ # See: /Include/py_curses.h #if-elif directives
+ # And compare with preprocessor output
+ if os.path.basename(included) == 'curses.h':
+ included = os.path.join(os.path.dirname(included), 'ncurses.h')
+
lno = _lno
included = _normpath(included, cwd)
# We hit a marker line.
diff --git a/Tools/c-analyzer/cpython/_parser.py
b/Tools/c-analyzer/cpython/_parser.py
index fd198d7d06c96f..d348a99fff7a11 100644
--- a/Tools/c-analyzer/cpython/_parser.py
+++ b/Tools/c-analyzer/cpython/_parser.py
@@ -340,6 +340,10 @@ def format_tsv_lines(lines):
# Catch-alls:
_abs('Include/**/*.h'): (5_000, 500),
+
+ # Specific to clang
+ _abs('Modules/selectmodule.c'): (40_000, 3000),
+ _abs('Modules/_testcapi/pyatomic.c'): (30_000, 1000),
}
diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv
b/Tools/c-analyzer/cpython/globals-to-fix.tsv
index 3c3cb2f9c86f16..301784f773d31f 100644
--- a/Tools/c-analyzer/cpython/globals-to-fix.tsv
+++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv
@@ -400,6 +400,7 @@ Modules/_tkinter.c - tcl_lock -
Modules/_tkinter.c - excInCmd -
Modules/_tkinter.c - valInCmd -
Modules/_tkinter.c - trbInCmd -
+Modules/socketmodule.c - netdb_lock -
##################################
diff --git a/Tools/c-analyzer/cpython/ignored.tsv
b/Tools/c-analyzer/cpython/ignored.tsv
index bd4a8cf0d3e65c..adb183000deeff 100644
--- a/Tools/c-analyzer/cpython/ignored.tsv
+++ b/Tools/c-analyzer/cpython/ignored.tsv
@@ -16,6 +16,7 @@ filename funcname name reason
## indicators for resource availability/capability
# (set during first init)
Python/bootstrap_hash.c py_getrandom getrandom_works -
+Python/bootstrap_hash.c py_getentropy getentropy_works -
Python/fileutils.c - _Py_open_cloexec_works -
Python/fileutils.c set_inheritable ioctl_works -
# (set lazily, *after* first init)
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
