This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-2-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-2-test by this push:
new cb5a95e247c [v3-2-test] Gate LINE_THRESHOLD on production-code churn
only (#67722) (#67728)
cb5a95e247c is described below
commit cb5a95e247cd7beff5cd841708ca19d05accf9f8
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Jun 2 19:47:14 2026 +0200
[v3-2-test] Gate LINE_THRESHOLD on production-code churn only (#67722)
(#67728)
`SelectiveChecks._is_large_enough_pr`'s line-count check previously
summed every changed line (minus a small set of lock/newsfragment
exclusions), so a 1000-line docs or test-only PR was treated as the
same risk shape as a 1000-line scheduler change and would force the
full CI matrix. The file-count check stays unchanged; only the line
count is narrowed.
Compose the existing `PYTHON_PRODUCTION_FILES`,
`JAVASCRIPT_PRODUCTION_FILES`, and `HELM_FILES` groups for the
production-code filter rather than rolling a parallel pattern set,
and tighten the first two to actually match production-only paths:
- `PYTHON_PRODUCTION_FILES` now excludes test paths (the old
`^providers/.*\.py` matched provider tests too), adds `task-sdk/src/`,
`airflow-ctl/src/`, `shared/*/src/`, and excludes `openapi-gen/`,
`i18n/locales/`, generated datamodels within those trees.
- `JAVASCRIPT_PRODUCTION_FILES` excludes `openapi-gen/` and translation
bundles for the same reason.
`run_python_scans` / `run_javascript_scans` (the other consumers of
`*_PRODUCTION_FILES`) also become more accurate as a side-effect — SAST
and SCA targets are now production code, not tests.
Five test cases added to cover test-only, docs-only, generated-only,
and mixed (above and below threshold) PRs; the existing three cases
still hold.
(cherry picked from commit d3877ba028de3add30600aba190fa3bc49a71934)
Co-authored-by: Jarek Potiuk <[email protected]>
---
.../src/airflow_breeze/utils/selective_checks.py | 52 +++++++++++++++----
dev/breeze/tests/test_selective_checks.py | 60 ++++++++++++++++++++++
2 files changed, 103 insertions(+), 9 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index 3b1b1c730a4..8546ab7bf47 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -199,15 +199,27 @@ CI_FILE_GROUP_MATCHES: HashableDict[FileGroupForCi] =
HashableDict(
r"^providers/elasticsearch/.*",
],
FileGroupForCi.PYTHON_PRODUCTION_FILES: [
- r"^airflow-core/src/airflow/.*\.py",
- r"^providers/.*\.py",
- r"^pyproject.toml",
- r"^hatch_build.py",
+ # Production Python source the runtime ships — excludes tests,
docs,
+ # dev tooling, and generated files within those trees. Used by
+ # `run_python_scans` (SAST/SCA target) and the line-threshold check
+ # in `_is_large_enough_pr` to decide whether a PR's diff is large
+ # enough to force the full test matrix.
+
r"^airflow-core/src/airflow/(?!.*/(?:openapi-gen|i18n/locales)/).*\.py$",
+ r"^task-sdk/src/airflow/(?!.*_generated\.py$).*\.py$",
+ r"^airflow-ctl/src/airflowctl/(?!.*generated\.py$).*\.py$",
+ r"^providers/(?:[^/]+/)+src/.*\.py$",
+ r"^shared/[^/]+/src/.*\.py$",
+ r"^pyproject\.toml$",
+ r"^hatch_build\.py$",
],
FileGroupForCi.JAVASCRIPT_PRODUCTION_FILES: [
- r"^airflow-core/src/airflow/.*\.[jt]sx?",
- r"^airflow-core/src/airflow/.*\.lock",
- r"^airflow-core/src/airflow/ui/.*\.yaml$",
+ # Exclude the openapi-gen tree and translation bundles — those are
+ # generated / data files that ride under the same prefixes but
+ # carry no behavioral risk and would otherwise distort the
+ # production-code line-count gate.
+
r"^airflow-core/src/airflow/(?!.*/(?:openapi-gen|i18n/locales)/).*\.[jt]sx?$",
+ r"^airflow-core/src/airflow/.*\.lock$",
+
r"^airflow-core/src/airflow/ui/(?!.*/(?:openapi-gen|i18n/locales)/).*\.yaml$",
r"^airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/.*\.yaml$",
],
FileGroupForCi.API_FILES: [
@@ -694,6 +706,12 @@ class SelectiveChecks:
The heuristics are based on number of files changed and total lines
changed,
while excluding generated files which can be ignored.
+
+ The line-count check (``LINE_THRESHOLD``) only counts lines in
production-code
+ files — tests, docs, newsfragments, generated files, translations, dev
tooling,
+ and similar low-risk paths do not contribute to the line count. A
1000-line test
+ or docs PR is not the same shape of risk as a 1000-line change to
scheduler
+ code, and only the latter should trigger the full test matrix.
"""
FILE_THRESHOLD = 25
LINE_THRESHOLD = 500
@@ -724,9 +742,24 @@ class SelectiveChecks:
console_print("[warning]Cannot determine if PR is big enough,
skipping the check[/]")
return False
+ # The line-count gate only counts churn in production code. We compose
+ # the existing `*_PRODUCTION_FILES` and helm groups rather than rolling
+ # a bespoke pattern set, so the definition of "production code" stays
+ # in lockstep with the rest of CI (e.g. SAST scans targeted by
+ # `run_python_scans` / `run_javascript_scans`).
+ production_files = list(
+ dict.fromkeys(
+ self._matching_files(FileGroupForCi.PYTHON_PRODUCTION_FILES,
CI_FILE_GROUP_MATCHES)
+ +
self._matching_files(FileGroupForCi.JAVASCRIPT_PRODUCTION_FILES,
CI_FILE_GROUP_MATCHES)
+ + self._matching_files(FileGroupForCi.HELM_FILES,
CI_FILE_GROUP_MATCHES)
+ )
+ )
+ if not production_files:
+ return False
+
try:
result = run_command(
- ["git", "diff", "--numstat",
f"{self._commit_ref}^...{self._commit_ref}"] + relevant_files,
+ ["git", "diff", "--numstat",
f"{self._commit_ref}^...{self._commit_ref}"] + production_files,
capture_output=True,
text=True,
cwd=AIRFLOW_ROOT_PATH,
@@ -748,7 +781,8 @@ class SelectiveChecks:
if total_lines >= LINE_THRESHOLD:
console_print(
f"[warning]Running full set of tests because PR
changes {total_lines} lines "
- f"in {files_changed} files[/]"
+ f"of production code in {len(production_files)}
file(s) "
+ f"(of {files_changed} relevant file(s))[/]"
)
return True
except Exception:
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index 0dfe565ad96..6ab7ded9ce1 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -3457,6 +3457,66 @@ def test_large_pr_by_file_count(files, expected_outputs:
dict[str, str]):
},
id="Single large file with 1000 lines",
),
+ pytest.param(
+ tuple(f"airflow-core/tests/unit/models/test_file{i}.py" for i in
range(10)),
+
"\n".join([f"100\t100\tairflow-core/tests/unit/models/test_file{i}.py" for i in
range(10)]),
+ {
+ "full-tests-needed": "false",
+ },
+ id="Large test-only PR (2000 lines) does not trigger full tests",
+ ),
+ pytest.param(
+ ("docs/index.rst",
"airflow-core/docs/security/security_model.rst"),
+
"600\t600\tdocs/index.rst\n400\t400\tairflow-core/docs/security/security_model.rst",
+ {
+ "full-tests-needed": "false",
+ },
+ id="Large docs-only PR does not trigger full tests",
+ ),
+ pytest.param(
+ (
+ "airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts",
+ "airflow-ctl/src/airflowctl/api/datamodels/generated.py",
+ "task-sdk/src/airflow/sdk/api/datamodels/_generated.py",
+ ),
+ "\n".join(
+ [
+
"400\t400\tairflow-core/src/airflow/ui/openapi-gen/queries/queries.ts",
+
"400\t400\tairflow-ctl/src/airflowctl/api/datamodels/generated.py",
+
"400\t400\ttask-sdk/src/airflow/sdk/api/datamodels/_generated.py",
+ ]
+ ),
+ {
+ "full-tests-needed": "false",
+ },
+ id="Generated-only large PR does not trigger full tests",
+ ),
+ # In mixed PRs the production-file filter narrows the `git diff
--numstat`
+ # call to the production paths, so the mocked stdout below only
contains
+ # the production-file rows (mirroring what real git would return for
+ # that filtered argument list).
+ pytest.param(
+ tuple(
+ [f"airflow-core/src/airflow/models/file{i}.py" for i in
range(5)]
+ + [f"airflow-core/tests/unit/models/test_file{i}.py" for i in
range(5)]
+ ),
+ "\n".join([f"60\t60\tairflow-core/src/airflow/models/file{i}.py"
for i in range(5)]),
+ {
+ "full-tests-needed": "true",
+ },
+ id="Mixed PR with 600 production lines triggers (test lines
excluded but prod >= 500)",
+ ),
+ pytest.param(
+ tuple(
+ [f"airflow-core/src/airflow/models/file{i}.py" for i in
range(5)]
+ + [f"airflow-core/tests/unit/models/test_file{i}.py" for i in
range(5)]
+ ),
+ "\n".join([f"20\t20\tairflow-core/src/airflow/models/file{i}.py"
for i in range(5)]),
+ {
+ "full-tests-needed": "false",
+ },
+ id="Mixed PR with only 200 production lines does not trigger (test
lines excluded)",
+ ),
],
)
def test_large_pr_by_line_count(files, git_diff_output, expected_outputs:
dict[str, str]):
