commit: cce1dfea834ae526ebbe8506fdad19cc03287730
Author: Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Sat Dec 3 20:56:23 2016 +0000
Commit: Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Sun Dec 4 08:49:47 2016 +0000
URL: https://gitweb.gentoo.org/proj/qa-scripts.git/commit/?id=cce1dfea
Add pkgcheck XML output to HTML formatter scripts
pkgcheck2html/jinja2htmlcompress.py | 150 +++++++++++++++++++++++++++
pkgcheck2html/output.css | 184 ++++++++++++++++++++++++++++++++++
pkgcheck2html/output.html.jinja | 67 +++++++++++++
pkgcheck2html/pkgcheck2html.conf.json | 26 +++++
pkgcheck2html/pkgcheck2html.py | 139 +++++++++++++++++++++++++
5 files changed, 566 insertions(+)
diff --git a/pkgcheck2html/jinja2htmlcompress.py
b/pkgcheck2html/jinja2htmlcompress.py
new file mode 100644
index 0000000..5dfb211
--- /dev/null
+++ b/pkgcheck2html/jinja2htmlcompress.py
@@ -0,0 +1,150 @@
+# -*- coding: utf-8 -*-
+"""
+ jinja2htmlcompress
+ ~~~~~~~~~~~~~~~~~~
+
+ A Jinja2 extension that eliminates useless whitespace at template
+ compilation time without extra overhead.
+
+ :copyright: (c) 2011 by Armin Ronacher.
+ :license: BSD, see LICENSE for more details.
+"""
+import re
+from jinja2.ext import Extension
+from jinja2.lexer import Token, describe_token
+from jinja2 import TemplateSyntaxError
+
+
+_tag_re = re.compile(r'(?:<(/?)([a-zA-Z0-9_-]+)\s*|(>\s*))(?s)')
+_ws_normalize_re = re.compile(r'[ \t\r\n]+')
+
+
+class StreamProcessContext(object):
+
+ def __init__(self, stream):
+ self.stream = stream
+ self.token = None
+ self.stack = []
+
+ def fail(self, message):
+ raise TemplateSyntaxError(message, self.token.lineno,
+ self.stream.name, self.stream.filename)
+
+
+def _make_dict_from_listing(listing):
+ rv = {}
+ for keys, value in listing:
+ for key in keys:
+ rv[key] = value
+ return rv
+
+
+class HTMLCompress(Extension):
+ isolated_elements = set(['script', 'style', 'noscript', 'textarea'])
+ void_elements = set(['br', 'img', 'area', 'hr', 'param', 'input',
+ 'embed', 'col'])
+ block_elements = set(['div', 'p', 'form', 'ul', 'ol', 'li', 'table', 'tr',
+ 'tbody', 'thead', 'tfoot', 'tr', 'td', 'th', 'dl',
+ 'dt', 'dd', 'blockquote', 'h1', 'h2', 'h3', 'h4',
+ 'h5', 'h6', 'pre'])
+ breaking_rules = _make_dict_from_listing([
+ (['p'], set(['#block'])),
+ (['li'], set(['li'])),
+ (['td', 'th'], set(['td', 'th', 'tr', 'tbody', 'thead', 'tfoot'])),
+ (['tr'], set(['tr', 'tbody', 'thead', 'tfoot'])),
+ (['thead', 'tbody', 'tfoot'], set(['thead', 'tbody', 'tfoot'])),
+ (['dd', 'dt'], set(['dl', 'dt', 'dd']))
+ ])
+
+ def is_isolated(self, stack):
+ for tag in reversed(stack):
+ if tag in self.isolated_elements:
+ return True
+ return False
+
+ def is_breaking(self, tag, other_tag):
+ breaking = self.breaking_rules.get(other_tag)
+ return breaking and (tag in breaking or
+ ('#block' in breaking and tag in self.block_elements))
+
+ def enter_tag(self, tag, ctx):
+ while ctx.stack and self.is_breaking(tag, ctx.stack[-1]):
+ self.leave_tag(ctx.stack[-1], ctx)
+ if tag not in self.void_elements:
+ ctx.stack.append(tag)
+
+ def leave_tag(self, tag, ctx):
+ if not ctx.stack:
+ ctx.fail('Tried to leave "%s" but something closed '
+ 'it already' % tag)
+ if tag == ctx.stack[-1]:
+ ctx.stack.pop()
+ return
+ for idx, other_tag in enumerate(reversed(ctx.stack)):
+ if other_tag == tag:
+ for num in xrange(idx + 1):
+ ctx.stack.pop()
+ elif not self.breaking_rules.get(other_tag):
+ break
+
+ def normalize(self, ctx):
+ pos = 0
+ buffer = []
+ def write_data(value):
+ if not self.is_isolated(ctx.stack):
+ value = _ws_normalize_re.sub(' ', value.strip())
+ buffer.append(value)
+
+ for match in _tag_re.finditer(ctx.token.value):
+ closes, tag, sole = match.groups()
+ preamble = ctx.token.value[pos:match.start()]
+ write_data(preamble)
+ if sole:
+ write_data(sole)
+ else:
+ buffer.append(match.group())
+ (closes and self.leave_tag or self.enter_tag)(tag, ctx)
+ pos = match.end()
+
+ write_data(ctx.token.value[pos:])
+ return u''.join(buffer)
+
+ def filter_stream(self, stream):
+ ctx = StreamProcessContext(stream)
+ for token in stream:
+ if token.type != 'data':
+ yield token
+ continue
+ ctx.token = token
+ value = self.normalize(ctx)
+ yield Token(token.lineno, 'data', value)
+
+
+class SelectiveHTMLCompress(HTMLCompress):
+
+ def filter_stream(self, stream):
+ ctx = StreamProcessContext(stream)
+ strip_depth = 0
+ while 1:
+ if stream.current.type == 'block_begin':
+ if stream.look().test('name:strip') or \
+ stream.look().test('name:endstrip'):
+ stream.skip()
+ if stream.current.value == 'strip':
+ strip_depth += 1
+ else:
+ strip_depth -= 1
+ if strip_depth < 0:
+ ctx.fail('Unexpected tag endstrip')
+ stream.skip()
+ if stream.current.type != 'block_end':
+ ctx.fail('expected end of block, got %s' %
+ describe_token(stream.current))
+ stream.skip()
+ if strip_depth > 0 and stream.current.type == 'data':
+ ctx.token = stream.current
+ value = self.normalize(ctx)
+ yield Token(stream.current.lineno, 'data', value)
+ else:
+ yield stream.current
+ stream.next()
diff --git a/pkgcheck2html/output.css b/pkgcheck2html/output.css
new file mode 100644
index 0000000..6888102
--- /dev/null
+++ b/pkgcheck2html/output.css
@@ -0,0 +1,184 @@
+/* (c) 2016 Michał Górny, Patrice Clement */
+/* 2-clause BSD license */
+
+*
+{
+ box-sizing: border-box;
+}
+
+body
+{
+ margin: 0;
+ background-color: #463C65;
+ font-family: sans-serif;
+ font-size: 14px;
+}
+
+address
+{
+ color: white;
+ text-align: center;
+ margin: 1em;
+}
+
+.nav
+{
+ width: 20%;
+ position: absolute;
+ top: 0;
+}
+
+.nav ul
+{
+ list-style: none;
+ margin: 0;
+ padding: 1%;
+}
+
+.nav li
+{
+ padding: 0 1em;
+ border-radius: 4px;
+ margin-bottom: .3em;
+ background-color: #62548F;
+}
+
+.nav li a
+{
+ display: block;
+ width: 100%;
+}
+
+.nav h2
+{
+ color: white;
+ text-align: center;
+ font-size: 300%;
+ font-weight: bold;
+ text-transform: uppercase;
+ font-family: serif;
+}
+
+ul.nav li.header
+{
+ background-color: #463C65;
+}
+
+.content, h1
+{
+ padding: 2%;
+ margin: 0 0 0 20%;
+ background-color: #DDDAEC;
+}
+
+h1 {
+ font-family: serif;
+ color: #23457F;
+ font-size: 400%;
+ line-height: 2em;
+ font-weight: bold;
+ text-transform: uppercase;
+ text-align: center;
+ letter-spacing: .15em;
+}
+
+th
+{
+ text-align: left;
+ padding: 1em 0;
+ color: #23457F;
+}
+
+th.h2
+{
+ font-size: 120%;
+}
+
+th.h3
+{
+ padding-left: 1em;
+ font-size: 110%;
+}
+
+th:target
+{
+ background-color: #dfd;
+}
+
+th small
+{
+ padding-left: .5em;
+ visibility: hidden;
+}
+
+th:hover small
+{
+ visibility: visible;
+}
+
+td
+{
+ background-color: white;
+ line-height: 2em;
+ font-size: 120%;
+ padding-left: .5em;
+ white-space: pre-wrap;
+}
+
+td:hover
+{
+ background-color: #eee;
+}
+
+tr.err td
+{
+ background-color: #7E0202;
+ color: white;
+}
+
+tr.err td:hover
+{
+ background-color: #DA0404;
+}
+
+tr.warn td
+{
+ background-color: orange;
+}
+
+tr.warn td:hover
+{
+ background-color: #FFBB3E;
+}
+
+.nav a
+{
+ font-size: 150%;
+ line-height: 1.5em;
+ text-decoration: none;
+ white-space: pre;
+ overflow: hidden;
+ text-overflow: ellipsis;
+}
+
+.warn a
+{
+ color: orange;
+}
+
+.err a
+{
+ color: #F06F74;
+}
+
+.nav li:hover
+{
+ min-width: 100%;
+ width: -moz-max-content;
+ width: max-content;
+}
+
+.nav a:hover
+{
+ color: white;
+}
diff --git a/pkgcheck2html/output.html.jinja b/pkgcheck2html/output.html.jinja
new file mode 100644
index 0000000..2e44619
--- /dev/null
+++ b/pkgcheck2html/output.html.jinja
@@ -0,0 +1,67 @@
+<!DOCTYPE html>
+<html>
+ <head>
+ <meta charset="utf-8"/>
+ <title>Gentoo CI - QA check results</title>
+ <link rel="stylesheet" type="text/css" href="output.css" />
+ </head>
+
+ <body>
+ <h1>QA check results</h1>
+
+ {% if errors or warnings %}
+ <div class="nav">
+ <h2>issues</h2>
+
+ <ul>
+ {% for g in errors %}
+ <li class="err"><a href="#{{
g|join('/') }}">{{ g|join('/') }}</a></li>
+ {% endfor %}
+ {% for g in warnings %}
+ <li class="warn"><a href="#{{
g|join('/') }}">{{ g|join('/') }}</a></li>
+ {% endfor %}
+ </ul>
+ </div>
+ {% endif %}
+
+ <div class="content">
+ <table>
+ {% for g, r in results %}
+ {% set h2_id = g[0] if g else "global"
%}
+ <tr><th colspan="3" class="h2" id="{{
h2_id }}">
+ {{ g[0] if g else "Global-scope
results" }}
+ <small><a href="#{{ h2_id
}}">¶</a></small>
+ </th></tr>
+
+ {% for g, r in r %}
+ {% if g[0] %}
+ {% set h3_id = g[0] +
"/" + g[1] if g[1] else "_cat" %}
+ <tr><th colspan="3"
class="h3" id="{{ h3_id }}">
+ {{ g[1] if g[1]
else "Category results" }}
+ <small><a
href="#{{ h3_id }}">¶</a></small>
+ </th></tr>
+ {% endif %}
+
+ {% for g, r in r %}
+ {% for rx in r %}
+ {% set
class_str = "" %}
+ {% if
rx.css_class %}
+ {% set
class_str = ' class="' + rx.css_class + '"' %}
+ {% endif %}
+ <tr{{ class_str
}}>
+ <td>{{
g[2] if loop.index == 1 else "" }}</td>
+ <td>{{
rx.class }}</td>
+ <td>{{
rx.msg|escape }}</td>
+ </tr>
+ {% endfor %}
+ {% endfor %}
+ {% endfor %}
+ {% endfor %}
+ </table>
+ </div>
+
+ <address>Generated based on results from: {{ ts.strftime("%F %T
UTC") }}</address>
+ </body>
+</html>
+
+<!-- vim:se ft=jinja : -->
diff --git a/pkgcheck2html/pkgcheck2html.conf.json
b/pkgcheck2html/pkgcheck2html.conf.json
new file mode 100644
index 0000000..f9c597e
--- /dev/null
+++ b/pkgcheck2html/pkgcheck2html.conf.json
@@ -0,0 +1,26 @@
+{
+ "CatMetadataXmlInvalidPkgRef": "err",
+ "VisibleVcsPkg": "err",
+ "MissingUri": "warn",
+ "CatBadlyFormedXml": "err",
+ "Glep31Violation": "err",
+ "PkgBadlyFormedXml": "err",
+ "CatInvalidXml": "err",
+ "CatMetadataXmlInvalidCatRef": "err",
+ "PkgMetadataXmlInvalidCatRef": "err",
+ "ConflictingChksums": "err",
+ "MissingChksum": "warn",
+ "MissingManifest": "err",
+ "CrappyDescription": "warn",
+ "PkgMetadataXmlInvalidPkgRef": "err",
+ "PkgMetadataXmlInvalidProjectError": "err",
+ "PkgInvalidXml": "err",
+ "NonsolvableDeps": "err",
+ "UnusedLocalFlags": "err",
+ "MetadataLoadError": "err",
+ "UnknownManifest": "err",
+ "NoFinalNewline": "err",
+ "UnstatedIUSE": "err",
+ "MetadataError": "err",
+ "WrongIndentFound": "err"
+}
diff --git a/pkgcheck2html/pkgcheck2html.py b/pkgcheck2html/pkgcheck2html.py
new file mode 100755
index 0000000..466d8c1
--- /dev/null
+++ b/pkgcheck2html/pkgcheck2html.py
@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+# vim:se fileencoding=utf8 :
+# (c) 2015-2016 Michał Górny
+# 2-clause BSD license
+
+import argparse
+import datetime
+import io
+import json
+import os
+import os.path
+import sys
+import xml.etree.ElementTree
+
+import jinja2
+
+
+class Result(object):
+ def __init__(self, el, class_mapping):
+ self._el = el
+ self._class_mapping = class_mapping
+
+ def __getattr__(self, key):
+ return self._el.findtext(key) or ''
+
+ @property
+ def css_class(self):
+ return self._class_mapping.get(getattr(self, 'class'), '')
+
+
+def result_sort_key(r):
+ return (r.category, r.package, r.version, getattr(r, 'class'), r.msg)
+
+
+def get_results(input_paths, class_mapping):
+ for input_path in input_paths:
+ checks = xml.etree.ElementTree.parse(input_path).getroot()
+ for r in checks:
+ yield Result(r, class_mapping)
+
+
+def split_result_group(it):
+ for r in it:
+ if not r.category:
+ yield ((), r)
+ elif not r.package:
+ yield ((r.category,), r)
+ elif not r.version:
+ yield ((r.category, r.package), r)
+ else:
+ yield ((r.category, r.package, r.version), r)
+
+
+def group_results(it, level = 3):
+ prev_group = ()
+ prev_l = []
+
+ for g, r in split_result_group(it):
+ if g[:level] != prev_group:
+ if prev_l:
+ yield (prev_group, prev_l)
+ prev_group = g[:level]
+ prev_l = []
+ prev_l.append(r)
+ yield (prev_group, prev_l)
+
+
+def deep_group(it, level = 1):
+ for g, r in group_results(it, level):
+ if level > 3:
+ for x in r:
+ yield x
+ else:
+ yield (g, deep_group(r, level+1))
+
+
+def find_of_class(it, cls, level = 2):
+ for g, r in group_results(it, level):
+ for x in r:
+ if x.css_class == cls:
+ yield g
+ break
+
+
+def get_result_timestamp(paths):
+ for p in paths:
+ st = os.stat(p)
+ return datetime.datetime.utcfromtimestamp(st.st_mtime)
+
+
+def main(*args):
+ p = argparse.ArgumentParser()
+ p.add_argument('-o', '--output', default='-',
+ help='Output HTML file ("-" for stdout)')
+ p.add_argument('-t', '--timestamp', default=None,
+ help='Timestamp for results (git ISO8601-like UTC)')
+ p.add_argument('files', nargs='+',
+ help='Input XML files')
+ args = p.parse_args(args)
+
+ conf_path = os.path.join(os.path.dirname(__file__),
'pkgcheck2html.conf.json')
+ with io.open(conf_path, 'r', encoding='utf8') as f:
+ class_mapping = json.load(f)
+
+ jenv = jinja2.Environment(
+ loader=jinja2.FileSystemLoader(os.path.dirname(__file__)),
+ extensions=['jinja2htmlcompress.HTMLCompress'])
+ t = jenv.get_template('output.html.jinja')
+
+ results = sorted(get_results(args.files, class_mapping),
key=result_sort_key)
+
+ types = {}
+ for r in results:
+ cl = getattr(r, 'class')
+ if cl not in types:
+ types[cl] = 0
+ types[cl] += 1
+
+ if args.timestamp is not None:
+ ts = datetime.datetime.strptime(args.timestamp, '%Y-%m-%d %H:%M:%S')
+ else:
+ ts = get_result_timestamp(args.files)
+
+ out = t.render(
+ results = deep_group(results),
+ warnings = list(find_of_class(results, 'warn')),
+ errors = list(find_of_class(results, 'err')),
+ ts = ts,
+ )
+
+ if args.output == '-':
+ sys.stdout.write(out)
+ else:
+ with io.open(args.output, 'w', encoding='utf8') as f:
+ f.write(out)
+
+
+if __name__ == '__main__':
+ sys.exit(main(*sys.argv[1:]))
