Module: Mesa Branch: main Commit: 65b6edee3e9e7db921abb51e009779410dcd64b3 URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=65b6edee3e9e7db921abb51e009779410dcd64b3
Author: Guilherme Gallo <[email protected]> Date: Fri Jul 15 18:13:00 2022 -0300 ci/bin: Add utility to find jobs dependencies Use GraphQL API from Gitlab to find jobs dependencies in a pipeline. E.g: Find all dependencies for jobs starting with "iris-" ```sh .gitlab-ci/bin/gitlab_gql.py --sha $(git -C ../mesa-fast-fix rev-parse HEAD) --print-dag --regex "iris-.*" ``` Signed-off-by: Guilherme Gallo <[email protected]> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/17791> --- .gitlab-ci/bin/download_gl_schema.sh | 11 ++++ .gitlab-ci/bin/gitlab_gql.py | 117 +++++++++++++++++++++++++++++++++++ .gitlab-ci/bin/pipeline_details.gql | 86 +++++++++++++++++++++++++ .gitlab-ci/bin/requirements.txt | 1 + .graphqlrc.yml | 2 + 5 files changed, 217 insertions(+) diff --git a/.gitlab-ci/bin/download_gl_schema.sh b/.gitlab-ci/bin/download_gl_schema.sh new file mode 100755 index 00000000000..41858df4c12 --- /dev/null +++ b/.gitlab-ci/bin/download_gl_schema.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +# Helper script to download the schema GraphQL from Gitlab to enable IDEs to +# assist the developer to edit gql files + +SOURCE_DIR=$(dirname "$(realpath "$0")") + +( + cd $SOURCE_DIR || exit 1 + gql-cli https://gitlab.freedesktop.org/api/graphql --print-schema > schema.graphql +) diff --git a/.gitlab-ci/bin/gitlab_gql.py b/.gitlab-ci/bin/gitlab_gql.py new file mode 100755 index 00000000000..04082ea7a35 --- /dev/null +++ b/.gitlab-ci/bin/gitlab_gql.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 + +import re +from argparse import ArgumentParser, Namespace +from dataclasses import dataclass, field +from itertools import chain +from pathlib import Path +from typing import Any, Pattern + +from gql import Client, gql +from gql.transport.aiohttp import AIOHTTPTransport +from graphql import DocumentNode + +Dag = dict[str, list[str]] + + +@dataclass +class GitlabGQL: + _transport: Any = field(init=False) + client: Client = field(init=False) + url: str = "https://gitlab.freedesktop.org/api/graphql"; + + def __post_init__(self): + self._setup_gitlab_gql_client() + + def _setup_gitlab_gql_client(self) -> Client: + # Select your transport with a defined url endpoint + self._transport = AIOHTTPTransport(url=self.url) + + # Create a GraphQL client using the defined transport + self.client = Client( + transport=self._transport, fetch_schema_from_transport=True + ) + + def query(self, gql_file: Path | str, params: dict[str, Any]) -> dict[str, Any]: + # Provide a GraphQL query + source_path = Path(__file__).parent + pipeline_query_file = source_path / gql_file + + query: DocumentNode + with open(pipeline_query_file, "r") as f: + pipeline_query = f.read() + query = gql(pipeline_query) + + # Execute the query on the transport + return self.client.execute(query, variable_values=params) + + +def create_job_needs_dag( + gl_gql: GitlabGQL, params +) -> tuple[Dag, dict[str, dict[str, Any]]]: + + result = gl_gql.query("pipeline_details.gql", params) + dag = {} + jobs = {} + pipeline = result["project"]["pipeline"] + if not pipeline: + raise RuntimeError(f"Could not find any pipelines for {params}") + + for stage in pipeline["stages"]["nodes"]: + for stage_job in stage["groups"]["nodes"]: + for job in stage_job["jobs"]["nodes"]: + needs = job.pop("needs")["nodes"] + jobs[job["name"]] = job + dag[job["name"]] = {node["name"] for node in needs} + + for job, needs in dag.items(): + needs: set + partial = True + + while partial: + next_depth = {n for dn in needs for n in dag[dn]} + partial = not needs.issuperset(next_depth) + needs = needs.union(next_depth) + + dag[job] = needs + + return dag, jobs + + +def filter_dag(dag: Dag, regex: Pattern) -> Dag: + return {job: needs for job, needs in dag.items() if re.match(regex, job)} + + +def print_dag(dag: Dag) -> None: + for job, needs in dag.items(): + print(f"{job}:") + print(f"\t{' '.join(needs)}") + print() + + +def parse_args() -> Namespace: + parser = ArgumentParser() + parser.add_argument("-pp", "--project-path", type=str, default="mesa/mesa") + parser.add_argument("--sha", type=str, required=True) + parser.add_argument("--regex", type=str, required=False) + parser.add_argument("--print-dag", action="store_true") + + return parser.parse_args() + + +def main(): + args = parse_args() + gl_gql = GitlabGQL() + + if args.print_dag: + dag, jobs = create_job_needs_dag( + gl_gql, {"projectPath": args.project_path, "sha": args.sha} + ) + + if args.regex: + dag = filter_dag(dag, re.compile(args.regex)) + print_dag(dag) + + +if __name__ == "__main__": + main() diff --git a/.gitlab-ci/bin/pipeline_details.gql b/.gitlab-ci/bin/pipeline_details.gql new file mode 100644 index 00000000000..e735ed8d396 --- /dev/null +++ b/.gitlab-ci/bin/pipeline_details.gql @@ -0,0 +1,86 @@ +fragment LinkedPipelineData on Pipeline { + id + iid + path + cancelable + retryable + userPermissions { + updatePipeline + } + status: detailedStatus { + id + group + label + icon + } + sourceJob { + id + name + } + project { + id + name + fullPath + } +} + +query getPipelineDetails($projectPath: ID!, $sha: String!) { + project(fullPath: $projectPath) { + id + pipeline(sha: $sha) { + id + iid + complete + downstream { + nodes { + ...LinkedPipelineData + } + } + upstream { + ...LinkedPipelineData + } + stages { + nodes { + id + name + status: detailedStatus { + id + action { + id + icon + path + title + } + } + groups { + nodes { + id + status: detailedStatus { + id + label + group + icon + } + name + size + jobs { + nodes { + id + name + kind + scheduledAt + needs { + nodes { + id + name + } + } + } + } + } + } + } + } + } + } +} diff --git a/.gitlab-ci/bin/requirements.txt b/.gitlab-ci/bin/requirements.txt index 115e8c29492..d07b936555b 100644 --- a/.gitlab-ci/bin/requirements.txt +++ b/.gitlab-ci/bin/requirements.txt @@ -1,2 +1,3 @@ colorama==0.4.5 +gql==3.4.0 python-gitlab==3.5.0 diff --git a/.graphqlrc.yml b/.graphqlrc.yml new file mode 100644 index 00000000000..d807b387b5e --- /dev/null +++ b/.graphqlrc.yml @@ -0,0 +1,2 @@ +schema: 'schema.graphql' +documents: 'src/**/*.{graphql,js,ts,jsx,tsx}'
