https://gcc.gnu.org/g:688e32e919c8a645de680f91be6feb63215bb6f4
commit r17-3922-g688e32e919c8a645de680f91be6feb63215bb6f4 Author: Pedro Alves <[email protected]> Date: Thu Nov 27 16:50:57 2025 +0000 dg-extract-results.{sh,py}: Optionally read file list from stdin/file dg-extract-results.sh and dg-extract-results.py take the summary/log files to combine as command-line arguments, one argument per file. That is fine for GCC, whose parallel harness produces a handful of files. GDB uses these scripts too, but its parallel harness writes one .sum (and one .log) per test script, of which there are 2657 at the moment. On Windows, with native Windows python, passing that many sum or log files to dg-extract-results overflows the command-line length limit (~32 KB), and the combine step dies. The .sh wrapper re-execs the faster Python implementation, forwarding the file list, so the overflow can also surface at the re-exec. E.g.: $ make check -j$(nproc) ... make[1]: Leaving directory '/c/msys2/home/alves/gdb/build-testsuite-windows' /c/gdb/src/gdb/testsuite/../../contrib/dg-extract-results.sh: line 39: /ucrt64/bin/python3: Argument list too long /c/gdb/src/gdb/testsuite/../../contrib/dg-extract-results.sh: line 39: /ucrt64/bin/python3: Argument list too long make: *** [Makefile:274: check-parallel] Error 2 ... $ find outputs/ -name gdb.sum | wc -l 2657 Fix it by teaching both dg-extract-results.sh and dg-extract-results.py a new -f option that reads the list of files to process, one per line, from a named file, or from standard input when the argument is "-". The option may be given more than once and mixed with file arguments. A caller can then pipe the file list straight in via "-f -" instead of expanding it onto the command line, so the length limit no longer applies regardless of host or Python flavor. Existing callers can continue working as they were, as passing the file list as one file per argument is still supported. contrib/ChangeLog: * dg-extract-results.py: Handle new "-f list-file" option. * dg-extract-results.sh: Ditto. Diff: --- contrib/dg-extract-results.py | 30 ++++++++++++++++++++++++++---- contrib/dg-extract-results.sh | 17 +++++++++++++++-- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/contrib/dg-extract-results.py b/contrib/dg-extract-results.py old mode 100644 new mode 100755 index 98b0f4989c96..b5d890f64063 --- a/contrib/dg-extract-results.py +++ b/contrib/dg-extract-results.py @@ -164,7 +164,7 @@ class Prog: def usage (self): name = sys.argv[0] sys.stderr.write ('Usage: ' + name - + ''' [-t tool] [-l variant-list] [-L] log-or-sum-file ... + + ''' [-t tool] [-l variant-list] [-L] [-f list-file] log-or-sum-file ... tool The tool (e.g. g++, libffi) for which to create a new test summary file. If not specified then output @@ -174,6 +174,12 @@ class Prog: variants in the files for <tool>. sum-file A test summary file with the format of those created by runtest from DejaGnu. + list-file A file listing the log-or-sum files to process, one + per line. Use "-" to read the list from standard + input. This avoids the command-line length limit + when combining very many files. May be given more + than once, and may be mixed with log-or-sum-file + arguments. If -L is used, merge *.log files instead of *.sum. In this mode the exact order of lines may not be preserved, just different Running *.exp chunks should be in correct order. @@ -189,19 +195,35 @@ class Prog: # Parse the command-line arguments. def parse_cmdline (self): try: - (options, self.files) = getopt.getopt (sys.argv[1:], 'l:t:L') - if len (self.files) == 0: - self.usage() + (options, self.files) = getopt.getopt (sys.argv[1:], 'l:t:Lf:') for (option, value) in options: if option == '-l': self.variations.append (value) elif option == '-t': self.tools.append (value) + elif option == '-f': + self.read_file_list (value) else: self.do_sum = False + if len (self.files) == 0: + self.usage() except getopt.GetoptError as e: self.fatal (None, e.msg) + # Append the files listed in FILENAME, one per line, to self.files. + # FILENAME of "-" means read the list from standard input. Blank + # lines are ignored. + def read_file_list (self, filename): + f = sys.stdin if filename == '-' else open (filename, 'r') + try: + for line in f: + line = line.strip() + if line: + self.files.append (line) + finally: + if f is not sys.stdin: + f.close() + # Try to parse time string TIME, returning an arbitrary time on failure. # Getting this right is just a nice-to-have so failures should be silent. def parse_time (self, time): diff --git a/contrib/dg-extract-results.sh b/contrib/dg-extract-results.sh index 6ea897033b57..f3093f94582e 100755 --- a/contrib/dg-extract-results.sh +++ b/contrib/dg-extract-results.sh @@ -42,7 +42,7 @@ done usage() { cat <<EOF >&2 -Usage: $PROGNAME [-t tool] [-l variant-list] [-L] sum-file ... +Usage: $PROGNAME [-t tool] [-l variant-list] [-L] [-f list-file] sum-file ... tool The tool (e.g. g++, libffi) for which to create a new test summary file. If not specified then all @@ -52,6 +52,11 @@ Usage: $PROGNAME [-t tool] [-l variant-list] [-L] sum-file ... variants in the files for <tool>. sum-file A test summary file with the format of those created by runtest from DejaGnu. + list-file A file listing the sum-files to process, one per line. + Use "-" to read the list from standard input. This + avoids the command-line length limit when combining + very many files. May be given more than once, and + may be mixed with sum-file arguments. If -L is used, merge *.log files instead of *.sum. In this mode the exact order of lines may not be preserved, just different Running *.exp chunks should be in correct order. @@ -69,18 +74,26 @@ msg() { VARIANTS="" TOOL="" MODE="sum" +LIST_FILES="" -while getopts "l:t:L" ARG; do +while getopts "l:t:Lf:" ARG; do case $ARG in l) VARIANTS="${VARIANTS} ${OPTARG}";; t) test -z "$TOOL" || (msg "${PROGNAME}: only one tool can be specified"; exit 1); TOOL="${OPTARG}";; L) MODE="log";; + f) if test "${OPTARG}" = "-" ; then + LIST_FILES="${LIST_FILES} `cat`" + else + LIST_FILES="${LIST_FILES} `cat "${OPTARG}"`" + fi;; \?) usage; exit 0;; esac done shift `expr ${OPTIND} - 1` +set -- ${LIST_FILES} "$@" + if test $# -lt 1 ; then usage exit 1
