Module: Mesa Branch: master Commit: 1cdb6edbe6dcfa3b0d254dc0f1f31e35be3b10ff URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=1cdb6edbe6dcfa3b0d254dc0f1f31e35be3b10ff
Author: Samuel Pitoiset <samuel.pitoi...@gmail.com> Date: Wed Feb 26 09:33:14 2020 +0100 gitlab-ci: add Fossilize support to detect compiler regressions Fossilize is equivalent to vkpipeline-db but it's definitely more robust. This is based on the CI traces system. Signed-off-by: Samuel Pitoiset <samuel.pitoi...@gmail.com> Reviewed-by: Alexandros Frantzis <alexandros.frant...@collabora.com> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/3960> --- .gitlab-ci.yml | 5 +++ .gitlab-ci/fossilize-runner.sh | 16 +++++++ .gitlab-ci/fossils/fossils.sh | 71 ++++++++++++++++++++++++++++++++ .gitlab-ci/fossils/query_fossils_yaml.py | 69 +++++++++++++++++++++++++++++++ .gitlab-ci/prepare-artifacts.sh | 2 + 5 files changed, 163 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ce35c57df3e..62cdb24a412 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -639,6 +639,11 @@ piglit-quick_shader: variables: DEQP_VER: vk +.fossilize-test: + extends: .test-vk + script: + - ./artifacts/fossilize-runner.sh + llvmpipe-gles2: variables: DEQP_VER: gles2 diff --git a/.gitlab-ci/fossilize-runner.sh b/.gitlab-ci/fossilize-runner.sh new file mode 100755 index 00000000000..618c2aa5b72 --- /dev/null +++ b/.gitlab-ci/fossilize-runner.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +set -ex + +if [ -z "$VK_DRIVER" ]; then + echo 'VK_DRIVER must be to something like "radeon" or "intel" for the test run' + exit 1 +fi + +ARTIFACTS=`pwd`/artifacts + +# Set up the driver environment. +export LD_LIBRARY_PATH=`pwd`/install/lib/ +export VK_ICD_FILENAMES=`pwd`/install/share/vulkan/icd.d/"$VK_DRIVER"_icd.x86_64.json + +"$ARTIFACTS/fossils/fossils.sh" "$ARTIFACTS/fossils.yml" diff --git a/.gitlab-ci/fossils/fossils.sh b/.gitlab-ci/fossils/fossils.sh new file mode 100755 index 00000000000..fa49efca2ce --- /dev/null +++ b/.gitlab-ci/fossils/fossils.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +FOSSILS_SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +FOSSILS_YAML="$(readlink -f "$1")" + +clone_fossils_db() +{ + local repo="$1" + local commit="$2" + rm -rf fossils-db + git clone --no-checkout "$repo" fossils-db + (cd fossils-db; git reset "$commit" || git reset "origin/$commit") +} + +query_fossils_yaml() +{ + python3 "$FOSSILS_SCRIPT_DIR/query_fossils_yaml.py" \ + --file "$FOSSILS_YAML" "$@" +} + +create_clean_git() +{ + rm -rf .clean_git + cp -R .git .clean_git +} + +restore_clean_git() +{ + rm -rf .git + cp -R .clean_git .git +} + +fetch_fossil() +{ + local fossil="${1//,/?}" + echo -n "[fetch_fossil] Fetching $1... " + local output=$(git lfs pull -I "$fossil" 2>&1) + local ret=0 + if [[ $? -ne 0 || ! -f "$1" ]]; then + echo "ERROR" + echo "$output" + ret=1 + else + echo "OK" + fi + restore_clean_git + return $ret +} + +if [[ -n "$(query_fossils_yaml fossils_db_repo)" ]]; then + clone_fossils_db "$(query_fossils_yaml fossils_db_repo)" \ + "$(query_fossils_yaml fossils_db_commit)" + cd fossils-db +else + echo "Warning: No fossils-db entry in $FOSSILS_YAML, assuming fossils-db is current directory" +fi + +# During git operations various git objects get created which +# may take up significant space. Store a clean .git instance, +# which we restore after various git operations to keep our +# storage consumption low. +create_clean_git + +for fossil in $(query_fossils_yaml fossils) +do + fetch_fossil "$fossil" || exit $? + fossilize-replay $fossil || exit $? + rm $fossil +done + +exit $ret diff --git a/.gitlab-ci/fossils/query_fossils_yaml.py b/.gitlab-ci/fossils/query_fossils_yaml.py new file mode 100644 index 00000000000..42e4b8e1c08 --- /dev/null +++ b/.gitlab-ci/fossils/query_fossils_yaml.py @@ -0,0 +1,69 @@ +#!/usr/bin/python3 + +# Copyright (c) 2019 Collabora Ltd +# Copyright (c) 2020 Valve Corporation +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# SPDX-License-Identifier: MIT + +import argparse +import yaml + +def cmd_fossils_db_repo(args): + with open(args.file, 'r') as f: + y = yaml.safe_load(f) + print(y['fossils-db']['repo']) + +def cmd_fossils_db_commit(args): + with open(args.file, 'r') as f: + y = yaml.safe_load(f) + print(y['fossils-db']['commit']) + +def cmd_fossils(args): + with open(args.file, 'r') as f: + y = yaml.safe_load(f) + + fossils = list(y['fossils']) + if len(fossils) == 0: + return + + print('\n'.join((t['path'] for t in fossils))) + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--file', required=True, + help='the name of the yaml file') + + subparsers = parser.add_subparsers(help='sub-command help') + + parser_fossils_db_repo = subparsers.add_parser('fossils_db_repo') + parser_fossils_db_repo.set_defaults(func=cmd_fossils_db_repo) + + parser_fossils_db_commit = subparsers.add_parser('fossils_db_commit') + parser_fossils_db_commit.set_defaults(func=cmd_fossils_db_commit) + + parser_fossils = subparsers.add_parser('fossils') + parser_fossils.set_defaults(func=cmd_fossils) + + args = parser.parse_args() + args.func(args) + +if __name__ == "__main__": + main() diff --git a/.gitlab-ci/prepare-artifacts.sh b/.gitlab-ci/prepare-artifacts.sh index 93910528d99..8a4b2510714 100755 --- a/.gitlab-ci/prepare-artifacts.sh +++ b/.gitlab-ci/prepare-artifacts.sh @@ -30,6 +30,8 @@ cp -Rp .gitlab-ci/traces.yml artifacts/ cp -Rp .gitlab-ci/tracie artifacts/ cp -Rp .gitlab-ci/tracie-runner-gl.sh artifacts/ cp -Rp .gitlab-ci/tracie-runner-vk.sh artifacts/ +cp -Rp .gitlab-ci/fossils artifacts/ +cp -Rp .gitlab-ci/fossilize-runner.sh artifacts/ # Tar up the install dir so that symlinks and hardlinks aren't each # packed separately in the zip file. _______________________________________________ mesa-commit mailing list mesa-commit@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/mesa-commit
