Copilot commented on code in PR #12388: URL: https://github.com/apache/gluten/pull/12388#discussion_r3555568061
########## .github/workflows/util/delta-spark-ut/compare-test-results.py: ########## @@ -0,0 +1,747 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Gate / seed / aggregate the Delta-on-Gluten unit test results. + +Running delta-io/delta's ScalaTest suite against the Gluten Velox bundle +produces many *expected* failures (Gluten does not yet support every Delta +code path). To keep the red/green signal meaningful while we fix those +failures incrementally, we maintain a committed baseline of known failing +tests (``known-failures.txt``) and compare each CI run against it. + +This script has three modes: + +``enforce`` (default, per shard) + Parse the JUnit XML produced by ``sbt spark/test`` (ScalaTest ``-u`` + reporter) and compare against the baseline: + + * regression -- a test that FAILED but is NOT in the baseline. These + fail the build: a previously-passing test just started failing. + * expected -- a test that failed and IS in the baseline. Ignored. + * fixed -- a baseline test that now PASSES. By default these also + fail the build (``--fail-on-fixed true``) so the baseline stays honest + and contributors remove entries as they fix them. + + If the baseline file exists but is empty (not yet bootstrapped) the mode + automatically degrades to ``seed`` so the first run is never spuriously red. + A *missing* ``--known-failures`` file is treated as a configuration error + (the gate fails) so a mis-referenced path can't silently pass. + +``seed`` (bootstrap / ``update_baseline``) + Never fails. Just writes the current shard's failing tests so the baseline + can be (re)generated from a real run. + +``aggregate`` (final job) + Merge every shard's ``--failures-out`` / ``--ran-out`` file into a single, + sorted, ready-to-commit ``known-failures.txt`` and report stale baseline + entries (tests no longer present in any shard). Pass ``--expected-shards N`` + to fail when fewer than ``N`` shards contributed gate lists (a shard that + died before writing them), so an incomplete baseline is never produced. + +Flaky quarantine (``--flaky-tests``) + Some Delta-on-Gluten failures are non-deterministic (e.g. a native bug that + only triggers on certain runtime plans), so they are neither a stable pass + nor a stable failure and cannot live in the baseline: baselining them turns + the gate red on every run where they pass, and leaving them out turns it red + on every run where they fail. ``flaky-tests.txt`` quarantines them -- a + quarantined test never counts as a regression (when it fails) nor as + now-passing (when it passes), and is excluded from the regenerated baseline. + Its SUITE is an fnmatch glob (so one line covers a root-cause family across + generated suite variants); its TEST name is matched exactly. + +Flaky quarantine by error signature (``--flaky-error-patterns``) + When a failure is caused by a known nondeterministic bug that surfaces on a + *different test each run* (e.g. the native Delta DV bitmap row-index error), + matching by test name is whack-a-mole. ``flaky-error-patterns.txt`` instead + quarantines by root cause: each line is a regex matched against a failed + test's <failure>/<error> text, and any failure that matches is treated as + flaky regardless of which test it landed on. + +Baseline file format (``known-failures.txt``):: + + # comment lines start with '#' + <fully.qualified.SuiteName>#<test display name> + +The suite is always a JVM class name (dot-separated, never starts with '#'), +so a line whose first non-space character is '#' is unambiguously a comment, +and the FIRST '#' after the suite separates suite from the (possibly +'#'-containing) test name. + +Only the Python standard library is used so the script runs in the bare +centos image used by the Delta UT pipeline with no ``pip install``. +""" + +import argparse +import fnmatch +import glob +import os +import re +import sys +import xml.etree.ElementTree as ET + +# Synthetic "test name" recorded when a whole suite aborts (e.g. beforeAll +# throws) so that the JUnit XML reports a suite-level error with no per-test +# <testcase>. Without this, a suite that used to pass but now aborts entirely +# would record zero failing testcases and the regression would be missed. +SUITE_ABORTED = "<suite aborted>" + + +class NoReportsError(RuntimeError): + """Raised when no JUnit <testsuite> elements are found under reports_dir.""" + + +class CorruptReportError(NoReportsError): + """Raised when an expected JUnit report file (TEST-*.xml) fails to parse. + + Subclasses NoReportsError so the enforce/seed handler treats a truncated + report as a hard data error (exit 2) instead of silently dropping the + suite's results and letting the gate pass on partial data. + """ + + +SEP = "#" + + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +# --------------------------------------------------------------------------- # +# Baseline (known-failures.txt) parsing / formatting +# --------------------------------------------------------------------------- # +def format_entry(suite, test): + return "{}{}{}".format(suite, SEP, test) + + +def parse_entry(line): + """Parse a 'suite#test' line into (suite, test) or return None for blanks/comments.""" + stripped = line.strip() + if not stripped or stripped.startswith("#"): + return None + idx = stripped.find(SEP) + if idx < 0: + # No separator: treat the whole line as a suite-level entry. + return (stripped, SUITE_ABORTED) + return (stripped[:idx], stripped[idx + len(SEP) :]) + + +def load_entries(path): + """Load a set of (suite, test) tuples from a baseline/shard-list file.""" + entries = set() + if not path or not os.path.exists(path): + return entries + with open(path, "r", encoding="utf-8") as fh: + for line in fh: + parsed = parse_entry(line) + if parsed is not None: + entries.add(parsed) + return entries + + +def make_is_flaky(flaky_entries): + """Build a predicate that matches a (suite, test) tuple against flaky entries. + + A flaky entry quarantines a test whose failure is known to be non-deterministic + (see flaky-tests.txt). The entry's SUITE is treated as an fnmatch glob so a + single line can cover a root-cause family across generated suite variants + (e.g. ``*DVs*Suite`` matches every deletion-vector merge suite, ``*`` matches + any suite); the TEST name is matched exactly (test names are freeform and may + contain glob metacharacters, so they are never globbed). + """ + exact = set() + globbed = [] + for suite, test in flaky_entries: + if any(ch in suite for ch in "*?["): + globbed.append((suite, test)) + else: + exact.add((suite, test)) + + def is_flaky(entry): + if entry in exact: + return True + suite, test = entry + for glob_suite, glob_test in globbed: + if test == glob_test and fnmatch.fnmatchcase(suite, glob_suite): + return True + return False + + return is_flaky + + +def load_patterns(path): + """Load flaky-error regex patterns from a file (one per line). + + Blank lines and ``#`` comments are ignored. Each remaining line is compiled + as a case-sensitive regex. These match the FAILURE TEXT of a failed test + (its JUnit <failure>/<error> message + stack), so a test that fails with a + known-nondeterministic native error (e.g. the Delta DV bitmap row-index bug) + can be quarantined by root cause instead of by exact test name. + """ + patterns = [] + if not path or not os.path.exists(path): + return patterns + with open(path, encoding="utf-8") as fh: + for line in fh: + line = line.rstrip("\n") + if not line.strip() or line.lstrip().startswith("#"): + continue + patterns.append(re.compile(line)) + return patterns Review Comment: `load_patterns()` compiles each non-comment line as a regex without any error context. If a contributor adds an invalid pattern to flaky-error-patterns.txt, the workflow will fail with an unhelpful Python stack trace that doesn’t identify which line/pattern was invalid. Catch `re.error` and rethrow with the file path + line number + pattern text so failures are actionable. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
