There's a linter in contrib/check-internal-format-escaping.py that checks whether the base file for translations (such as French or German) conform to some basic style rules. This mostly affects the diagnostics emitted by GCC.
As the German translation I have added several checks to the linter and added rationales to most of the checks. Since I completely rewrote the linter, it's more convenient to not look at the patch but only the rewritten code. Therefore I've provided both of them. See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90117. See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90119. See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90176. Thoughts?
diff --git a/contrib/check-internal-format-escaping.py b/contrib/check-internal-format-escaping.py index 9c625868012..e06752666b8 100755 --- a/contrib/check-internal-format-escaping.py +++ b/contrib/check-internal-format-escaping.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 # -# Check gcc.pot file for gcc-internal-format and print all strings -# that contain an option that is not wrapped by %<-option_name%>. +# Check gcc.pot file for stylistic issues as described in +# https://gcc.gnu.org/onlinedocs/gccint/Guidelines-for-Diagnostics.html, +# especially in gcc-internal-format messages. # # This file is part of GCC. # @@ -17,52 +18,249 @@ # # You should have received a copy of the GNU General Public License # along with GCC; see the file COPYING3. If not see -# <http://www.gnu.org/licenses/>. */ -# -# -# +# <http://www.gnu.org/licenses/>. import argparse import re +from collections import Counter +from typing import Dict, Match + +import polib + +seen_warnings = Counter() + + +def location(msg: polib.POEntry): + if msg.occurrences: + occ = msg.occurrences[0] + return f'{occ[0]}:{occ[1]}' + return '<unknown location>' + + +def warn(msg: polib.POEntry, + diagnostic_id: str, diagnostic: str, include_msgid=True): + """ + To suppress a warning for a particular message, + add a line "#, gcclint:ignore:{diagnostic_id}" to the message. + """ + + if f'gcclint:ignore:{diagnostic_id}' in msg.flags: + return + + seen_warnings[diagnostic] += 1 + + if include_msgid: + print(f'{location(msg)}: {diagnostic} in {repr(msg.msgid)}') + else: + print(f'{location(msg)}: {diagnostic}') + + +def lint_gcc_internal_format(msg: polib.POEntry): + """ + Checks a single message that has the gcc-internal-format. These + messages use a variety of placeholders like %qs, %<quotes%> and + %q#E. + """ + + msgid: str = msg.msgid + + def outside_quotes(m: Match[str]): + before = msgid[:m.start(0)] + return before.count("%<") == before.count("%>") + + def lint_matching_placeholders(): + """ + Warns when literal values in placeholders are not exactly equal + in the translation. This can happen when doing copy-and-paste + translations of similar messages. + + To avoid these mismatches in the first place, + structurally equal messages are found by + lint_diagnostics_differing_only_in_placeholders. + + This check only applies when checking a finished translation + such as de.po, not gcc.pot. + """ + + if not msg.translated(): + return + + in_msgid = re.findall('%<[^%]+%>', msgid) + in_msgstr = re.findall('%<[^%]+%>', msg.msgstr) + + if set(in_msgid) != set(in_msgstr): + warn(msg, + 'placeholder-mismatch', + f'placeholder mismatch: msgid has {in_msgid}, ' + f'msgstr has {in_msgstr}', + include_msgid=False) + + def lint_option_outside_quotes(): + for match in re.finditer(r'\S+', msgid): + part = match.group() + if not outside_quotes(match): + continue + + if part.startswith('-'): + if len(part) >= 2 and part[1].isalpha(): + if part == '-INF': + continue + + warn(msg, + 'option-outside-quotes', + 'command line option outside %<quotes%>') + + if part.startswith('__builtin_'): + warn(msg, + 'builtin-outside-quotes', + 'builtin function outside %<quotes%>') + + def lint_plain_apostrophe(): + for match in re.finditer("[^%]'", msgid): + if outside_quotes(match): + warn(msg, 'apostrophe', 'apostrophe without leading %') + + def lint_space_before_quote(): + """ + A space before %< is often the result of string literals that + are joined by the C compiler and neither literal has a space + to separate the words. + """ + + for match in re.finditer("(.?[a-zA-Z0-9])%<", msgid): + if match.group(1) != '%s': + warn(msg, + 'no-space-before-quote', + '%< directly following a letter or digit') + + def lint_underscore_outside_quotes(): + """ + An underscore outside of quotes is used in several contexts, + and many of them violate the GCC Guidelines for Diagnostics: + + * names of GCC-internal compiler functions + * names of GCC-internal data structures + * static_cast and the like (which are legitimate) + """ + + for match in re.finditer("_", msgid): + if outside_quotes(match): + warn(msg, + 'underscore-outside-quotes', + 'underscore outside of %<quotes%>') + return + + def lint_may_not(): + """ + The term "may not" may either mean "it could be the case" + or "should not". These two different meanings are sometimes + hard to tell apart. + """ + + if re.search(r'\bmay not\b', msgid): + warn(msg, + 'ambiguous-may-not', + 'the term "may not" is ambiguous') + + def lint_unbalanced_quotes(): + if msgid.count("%<") != msgid.count("%>"): + warn(msg, + 'unbalanced-quotes', + 'unbalanced %< and %> quotes') + + if msg.translated(): + if msg.msgstr.count("%<") != msg.msgstr.count("%>"): + warn(msg, + 'unbalanced-quotes', + 'unbalanced %< and %> quotes') + + def lint_single_space_after_sentence(): + """ + After a sentence there should be two spaces. + """ + + if re.search(r'[.] [A-Z]', msgid): + warn(msg, + 'single-space-after-sentence', + 'single space after sentence') + + def lint_non_canonical_quotes(): + """ + Catches %<%s%>, which can be written in the shorter form %qs. + """ + match = re.search("%<%s%>|'%s'|\"%s\"|`%s'", msgid) + if match: + warn(msg, + 'non-canonical-quotes', + f'placeholder {match.group()} should be written as %qs') + + lint_option_outside_quotes() + lint_plain_apostrophe() + lint_space_before_quote() + lint_underscore_outside_quotes() + lint_may_not() + lint_unbalanced_quotes() + lint_matching_placeholders() + lint_single_space_after_sentence() + lint_non_canonical_quotes() + + +def lint_diagnostics_differing_only_in_placeholders(po: polib.POFile): + """ + Detects messages that are structurally the same, except that they + use different plain strings inside %<quotes%>. These messages can + be merged in order to prevent copy-and-paste mistakes by the + translators. + + See bug 90119. + """ + + seen: Dict[str, polib.POEntry] = {} + + for msg in po: + msg: polib.POEntry + msgid = msg.msgid + + normalized = re.sub('%<[^%]+%>', '%qs', msgid) + if normalized not in seen: + seen[normalized] = msg + seen[msgid] = msg + continue + + prev = seen[normalized] + warn(msg, + 'same-pattern', + f'same pattern for {repr(msgid)} and ' + f'{repr(prev.msgid)} in {location(prev)}', + include_msgid=False) + + +def lint_file(po: polib.POFile): + for msg in po: + msg: polib.POEntry + + if not msg.obsolete and not msg.fuzzy: + if 'gcc-internal-format' in msg.flags: + lint_gcc_internal_format(msg) + + lint_diagnostics_differing_only_in_placeholders(po) + + +def main(): + parser = argparse.ArgumentParser(description='') + parser.add_argument('file', help='pot file') + + args = parser.parse_args() + + po = polib.pofile(args.file) + lint_file(po) + + print() + print('summary:') + for entry in seen_warnings.most_common(): + if entry[1] > 1: + print(f'{entry[1]}\t{entry[0]}') + -parser = argparse.ArgumentParser(description='') -parser.add_argument('file', help = 'pot file') - -args = parser.parse_args() - -origin = None -internal = False - -lines = open(args.file).readlines() -for i, l in enumerate(lines): - l = l.strip() - s = 'msgid ' - if l.startswith('#: '): - origin = l - elif '#, gcc-internal-format' in l: - internal = True - if l.startswith(s) and origin and internal: - j = 0 - while not lines[i + j].startswith('msgstr'): - l = lines[i + j] - if l.startswith(s): - l = l[len(s):] - text = l.strip('"').strip() - if text: - parts = text.split(' ') - for p in parts: - if p.startswith('-'): - if len(p) >= 2 and (p[1].isalpha() and p != '-INF'): - print('%s: %s' % (origin, text)) - elif p.startswith('__builtin_'): - print('%s: %s' % (origin, text)) - if re.search("[^%]'", p): - print('%s: %s' % (origin, text)) - # %< should not be preceded by a non-punctuation - # %character. - if re.search("[a-zA-Z0-9]%<", p): - print('%s: %s' % (origin, text)) - j += 1 - - origin = None - internal = False +if __name__ == '__main__': + main() diff --git a/contrib/check-string-literals.py b/contrib/check-string-literals.py new file mode 100644 index 00000000000..d99997d674d --- /dev/null +++ b/contrib/check-string-literals.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# +# Check source files for string literals that span multiple lines. +# These string literals should have a space either at the end of the +# upper line or at the beginning of the lower line. +# +# This file is part of GCC. +# +# GCC is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 3, or (at your option) any later +# version. +# +# GCC is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License +# along with GCC; see the file COPYING3. If not see +# <http://www.gnu.org/licenses/>. +import os +import re + + +# The string parsing is done ad-hoc. Some day it should be replaced +# by a proper C tokenizer. + +def unescape_string_literal(lit: str): + return lit.replace('\\n', '\n').replace('\\t', '\t') + + +def extract_literal_start(line: str): + m = re.search(r'^"((?:[^"\\]+(?![^"\\]+)|\\.)*)"', line) + return unescape_string_literal(m.group(1)) if m else False + + +def extract_literal_end(line: str): + m = re.search(r'"((?:[^"\\]+(?![^"\\]+)|\\.)*)"$', line) + return unescape_string_literal(m.group(1)) if m else False + + +def lint_line(filename: str, lineno: int, prev: str, curr: str): + ends_with_space = re.search(r'\s$', prev) is not None + starts_with_space = re.search(r'^\s', curr) is not None + + if not ends_with_space and not starts_with_space: + print(f'{filename}:{lineno}: missing space between string literals {repr(prev)} and {repr(curr)}') + + if prev.endswith(' ') and curr.startswith(' '): + print(f'{filename}:{lineno}: double space between string literals {repr(prev)} and {repr(curr)}') + + +def lint_file(filename): + try: + with open(filename, encoding='UTF-8') as f: + prev = "" + lineno = 1 + for line in f: + line = line.strip() + if prev.endswith('"') and line.startswith('"'): + end = extract_literal_end(prev) + start = extract_literal_start(line) + if start and end: + lint_line(filename, lineno, end, start) + + prev = line + lineno += 1 + + except UnicodeDecodeError: + pass + + +def lint_tree(): + for root, dirs, files in os.walk("."): + if 'testsuite' in dirs: + dirs.remove('testsuite') + + for name in files: + _, base = os.path.split(name) + _, ext = os.path.splitext(base) + + if ext.endswith('~'): + continue + if ext == '.po' or ext == '.pot' or ext == '.md': + continue + if base == 'configure' or base.startswith('ChangeLog-'): + continue + + lint_file(os.path.join(root, name)) + + +if __name__ == '__main__': + lint_tree()
#!/usr/bin/env python3 # # Check gcc.pot file for stylistic issues as described in # https://gcc.gnu.org/onlinedocs/gccint/Guidelines-for-Diagnostics.html, # especially in gcc-internal-format messages. # # This file is part of GCC. # # GCC is free software; you can redistribute it and/or modify it under # the terms of the GNU General Public License as published by the Free # Software Foundation; either version 3, or (at your option) any later # version. # # GCC is distributed in the hope that it will be useful, but WITHOUT ANY # WARRANTY; without even the implied warranty of MERCHANTABILITY or # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License # for more details. # # You should have received a copy of the GNU General Public License # along with GCC; see the file COPYING3. If not see # <http://www.gnu.org/licenses/>. import argparse import re from collections import Counter from typing import Dict, Match import polib seen_warnings = Counter() def location(msg: polib.POEntry): if msg.occurrences: occ = msg.occurrences[0] return f'{occ[0]}:{occ[1]}' return '<unknown location>' def warn(msg: polib.POEntry, diagnostic_id: str, diagnostic: str, include_msgid=True): """ To suppress a warning for a particular message, add a line "#, gcclint:ignore:{diagnostic_id}" to the message. """ if f'gcclint:ignore:{diagnostic_id}' in msg.flags: return seen_warnings[diagnostic] += 1 if include_msgid: print(f'{location(msg)}: {diagnostic} in {repr(msg.msgid)}') else: print(f'{location(msg)}: {diagnostic}') def lint_gcc_internal_format(msg: polib.POEntry): """ Checks a single message that has the gcc-internal-format. These messages use a variety of placeholders like %qs, %<quotes%> and %q#E. """ msgid: str = msg.msgid def outside_quotes(m: Match[str]): before = msgid[:m.start(0)] return before.count("%<") == before.count("%>") def lint_matching_placeholders(): """ Warns when literal values in placeholders are not exactly equal in the translation. This can happen when doing copy-and-paste translations of similar messages. To avoid these mismatches in the first place, structurally equal messages are found by lint_diagnostics_differing_only_in_placeholders. This check only applies when checking a finished translation such as de.po, not gcc.pot. """ if not msg.translated(): return in_msgid = re.findall('%<[^%]+%>', msgid) in_msgstr = re.findall('%<[^%]+%>', msg.msgstr) if set(in_msgid) != set(in_msgstr): warn(msg, 'placeholder-mismatch', f'placeholder mismatch: msgid has {in_msgid}, ' f'msgstr has {in_msgstr}', include_msgid=False) def lint_option_outside_quotes(): for match in re.finditer(r'\S+', msgid): part = match.group() if not outside_quotes(match): continue if part.startswith('-'): if len(part) >= 2 and part[1].isalpha(): if part == '-INF': continue warn(msg, 'option-outside-quotes', 'command line option outside %<quotes%>') if part.startswith('__builtin_'): warn(msg, 'builtin-outside-quotes', 'builtin function outside %<quotes%>') def lint_plain_apostrophe(): for match in re.finditer("[^%]'", msgid): if outside_quotes(match): warn(msg, 'apostrophe', 'apostrophe without leading %') def lint_space_before_quote(): """ A space before %< is often the result of string literals that are joined by the C compiler and neither literal has a space to separate the words. """ for match in re.finditer("(.?[a-zA-Z0-9])%<", msgid): if match.group(1) != '%s': warn(msg, 'no-space-before-quote', '%< directly following a letter or digit') def lint_underscore_outside_quotes(): """ An underscore outside of quotes is used in several contexts, and many of them violate the GCC Guidelines for Diagnostics: * names of GCC-internal compiler functions * names of GCC-internal data structures * static_cast and the like (which are legitimate) """ for match in re.finditer("_", msgid): if outside_quotes(match): warn(msg, 'underscore-outside-quotes', 'underscore outside of %<quotes%>') return def lint_may_not(): """ The term "may not" may either mean "it could be the case" or "should not". These two different meanings are sometimes hard to tell apart. """ if re.search(r'\bmay not\b', msgid): warn(msg, 'ambiguous-may-not', 'the term "may not" is ambiguous') def lint_unbalanced_quotes(): if msgid.count("%<") != msgid.count("%>"): warn(msg, 'unbalanced-quotes', 'unbalanced %< and %> quotes') if msg.translated(): if msg.msgstr.count("%<") != msg.msgstr.count("%>"): warn(msg, 'unbalanced-quotes', 'unbalanced %< and %> quotes') def lint_single_space_after_sentence(): """ After a sentence there should be two spaces. """ if re.search(r'[.] [A-Z]', msgid): warn(msg, 'single-space-after-sentence', 'single space after sentence') def lint_non_canonical_quotes(): """ Catches %<%s%>, which can be written in the shorter form %qs. """ match = re.search("%<%s%>|'%s'|\"%s\"|`%s'", msgid) if match: warn(msg, 'non-canonical-quotes', f'placeholder {match.group()} should be written as %qs') lint_option_outside_quotes() lint_plain_apostrophe() lint_space_before_quote() lint_underscore_outside_quotes() lint_may_not() lint_unbalanced_quotes() lint_matching_placeholders() lint_single_space_after_sentence() lint_non_canonical_quotes() def lint_diagnostics_differing_only_in_placeholders(po: polib.POFile): """ Detects messages that are structurally the same, except that they use different plain strings inside %<quotes%>. These messages can be merged in order to prevent copy-and-paste mistakes by the translators. See bug 90119. """ seen: Dict[str, polib.POEntry] = {} for msg in po: msg: polib.POEntry msgid = msg.msgid normalized = re.sub('%<[^%]+%>', '%qs', msgid) if normalized not in seen: seen[normalized] = msg seen[msgid] = msg continue prev = seen[normalized] warn(msg, 'same-pattern', f'same pattern for {repr(msgid)} and ' f'{repr(prev.msgid)} in {location(prev)}', include_msgid=False) def lint_file(po: polib.POFile): for msg in po: msg: polib.POEntry if not msg.obsolete and not msg.fuzzy: if 'gcc-internal-format' in msg.flags: lint_gcc_internal_format(msg) lint_diagnostics_differing_only_in_placeholders(po) def main(): parser = argparse.ArgumentParser(description='') parser.add_argument('file', help='pot file') args = parser.parse_args() po = polib.pofile(args.file) lint_file(po) print() print('summary:') for entry in seen_warnings.most_common(): if entry[1] > 1: print(f'{entry[1]}\t{entry[0]}') if __name__ == '__main__': main()
