Module: Mesa Branch: main Commit: 45eda069531adf17d28538696ace5dfcbc9ad230 URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=45eda069531adf17d28538696ace5dfcbc9ad230
Author: David Heidelberg <[email protected]> Date: Thu Sep 1 22:50:38 2022 +0200 ci: introduce update_traces_checksum.py Invoking this script takes checksums from all failed jobs and update them in $driver-traces.yml files. ``` .gitlab-ci/bin/update_traces_checksum.py --rev $(git rev-parse HEAD) ``` Signed-off-by: David Heidelberg <[email protected]> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/18329> --- .gitlab-ci/bin/update_traces_checksum.py | 143 +++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/.gitlab-ci/bin/update_traces_checksum.py b/.gitlab-ci/bin/update_traces_checksum.py new file mode 100755 index 00000000000..54c84c1bf64 --- /dev/null +++ b/.gitlab-ci/bin/update_traces_checksum.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# Copyright © 2022 Collabora Ltd. +# Authors: +# David Heidelberg <[email protected]> +# +# SPDX-License-Identifier: MIT + +""" +Helper script to update traces checksums +""" + +import argparse +import bz2 +import glob +import re +import json +import sys +from ruamel.yaml import YAML + +import gitlab +from gitlab_common import get_gitlab_project, read_token, wait_for_pipeline + + +DESCRIPTION_FILE = "export PIGLIT_REPLAY_DESCRIPTION_FILE='.*/install/(.*)'$" +DEVICE_NAME = "export PIGLIT_REPLAY_DEVICE_NAME='(.*)'$" + + +def gather_results( + project, + pipeline, +) -> None: + """Gather results""" + + target_jobs_regex = re.compile(".*-traces([:].*)?$") + + for job in pipeline.jobs.list(all=True, sort="desc"): + if target_jobs_regex.match(job.name) and job.status == "failed": + cur_job = project.jobs.get(job.id) + # get variables + print(f"👁 Looking through logs for the device variable and traces.yml file in {job.name}...") + log = cur_job.trace().decode("unicode_escape").splitlines() + filename: str = '' + dev_name: str = '' + for logline in log: + desc_file = re.search(DESCRIPTION_FILE, logline) + device_name = re.search(DEVICE_NAME, logline) + if desc_file: + filename = desc_file.group(1) + if device_name: + dev_name = device_name.group(1) + + if not filename or not dev_name: + print("! Couldn't find device name or YML file in the logs!") + return + + print(f"👁 Found {dev_name} and file {filename}") + + # find filename in Mesa source + traces_file = glob.glob('./**/' + filename, recursive=True) + # write into it + with open(traces_file[0], 'r', encoding='utf-8') as target_file: + yaml = YAML() + yaml.compact(seq_seq=False, seq_map=False) + yaml.version = 1,2 + yaml.width = 2048 # do not break the text fields + yaml.default_flow_style = None + target = yaml.load(target_file) + + # parse artifact + results_json_bz2 = cur_job.artifact(path="results/results.json.bz2", streamed=False) + results_json = bz2.decompress(results_json_bz2).decode("utf-8") + results = json.loads(results_json) + + for _, value in results["tests"].items(): + if ( + not value['images'] or + not value['images'][0] or + "image_desc" not in value['images'][0] + ): + continue + + trace: str = value['images'][0]['image_desc'] + checksum: str = value['images'][0]['checksum_render'] + + if not checksum: + print(f"Trace {trace} checksum is missing! Abort.") + continue + + if checksum == "error": + print(f"Trace {trace} crashed") + continue + + if ( + checksum in target['traces'][trace][dev_name] and + target['traces'][trace][dev_name]['checksum'] == checksum + ): + continue + + if "label" in target['traces'][trace][dev_name]: + print(f'{trace}: {dev_name}: has label: {target["traces"][trace][dev_name]["label"]}, is it still right?') + + target['traces'][trace][dev_name]['checksum'] = checksum + + with open(traces_file[0], 'w', encoding='utf-8') as target_file: + yaml.dump(target, target_file) + + + +def parse_args() -> None: + """Parse args""" + parser = argparse.ArgumentParser( + description="Tool to generate patch from checksums ", + epilog="Example: update_traces_checksum.py --rev $(git rev-parse HEAD) " + ) + parser.add_argument( + "--rev", metavar="revision", help="repository git revision", required=True + ) + parser.add_argument( + "--token", + metavar="token", + help="force GitLab token, otherwise it's read from ~/.config/gitlab-token", + ) + return parser.parse_args() + + +if __name__ == "__main__": + try: + args = parse_args() + + token = read_token(args.token) + + gl = gitlab.Gitlab(url="https://gitlab.freedesktop.org";, private_token=token) + + cur_project = get_gitlab_project(gl, "mesa") + + print(f"Revision: {args.rev}") + pipe = wait_for_pipeline(cur_project, args.rev) + print(f"Pipeline: {pipe.web_url}") + gather_results(cur_project, pipe) + + sys.exit() + except KeyboardInterrupt: + sys.exit(1)
