This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 92909e53cbf Make branch tests more resilient to provider dependency
changes (#53776)
92909e53cbf is described below
commit 92909e53cbfaaf9dd5698ae7ee4da6d21a06d309
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sat Jul 26 09:47:56 2025 +0200
Make branch tests more resilient to provider dependency changes (#53776)
While we want to follow automated upgrades of dependencies and constraints
on vX-Y-test branches, in order to pull latest providers from PyPI,
this might have some undesired effects - for example some provider's
dependencies might break tests from importing or produce false
tests results if such tests depend on - for example - importing all
provider example dags. This happens for example for serialization
tests - which import all example dags to test that serialization
works for them.
Such tests should be simply skipped when target branch is different
than main.
This PR adds "skip_if_not_on_main" marker and uses it for two tests
that are currently failing on v3-0-test branch due to weaviate
dependency upgrades causing import error and some undesired warnings.
---
.github/workflows/ci-amd.yml | 11 +++++++++++
.github/workflows/ci-arm.yml | 6 ++++++
.github/workflows/run-unit-tests.yml | 5 +++++
.github/workflows/special-tests.yml | 10 +++++++++-
airflow-core/tests/unit/always/test_providers_manager.py | 3 ++-
.../tests/unit/serialization/test_dag_serialization.py | 6 ++++--
devel-common/src/tests_common/test_utils/markers.py | 6 ++++++
7 files changed, 43 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml
index 982729d9bf7..311d44c47f9 100644
--- a/.github/workflows/ci-amd.yml
+++ b/.github/workflows/ci-amd.yml
@@ -384,6 +384,8 @@ jobs:
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests
}}
use-uv: ${{ needs.build-info.outputs.use-uv }}
+ default-branch: ${{ needs.build-info.outputs.default-branch }}
+
if: needs.build-info.outputs.run-tests == 'true'
tests-postgres-providers:
@@ -412,6 +414,7 @@ jobs:
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests
}}
use-uv: ${{ needs.build-info.outputs.use-uv }}
+ default-branch: ${{ needs.build-info.outputs.default-branch }}
if: needs.build-info.outputs.run-tests == 'true'
tests-mysql-core:
@@ -440,6 +443,7 @@ jobs:
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests
}}
use-uv: ${{ needs.build-info.outputs.use-uv }}
+ default-branch: ${{ needs.build-info.outputs.default-branch }}
if: needs.build-info.outputs.run-tests == 'true'
tests-mysql-providers:
@@ -468,6 +472,7 @@ jobs:
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests
}}
use-uv: ${{ needs.build-info.outputs.use-uv }}
+ default-branch: ${{ needs.build-info.outputs.default-branch }}
if: needs.build-info.outputs.run-tests == 'true'
@@ -499,6 +504,7 @@ jobs:
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests
}}
use-uv: ${{ needs.build-info.outputs.use-uv }}
+ default-branch: ${{ needs.build-info.outputs.default-branch }}
if: needs.build-info.outputs.run-tests == 'true'
tests-sqlite-providers:
@@ -529,6 +535,7 @@ jobs:
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests
}}
use-uv: ${{ needs.build-info.outputs.use-uv }}
+ default-branch: ${{ needs.build-info.outputs.default-branch }}
if: needs.build-info.outputs.run-tests == 'true'
@@ -559,6 +566,7 @@ jobs:
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests
}}
use-uv: ${{ needs.build-info.outputs.use-uv }}
+ default-branch: ${{ needs.build-info.outputs.default-branch }}
if: needs.build-info.outputs.run-tests == 'true'
tests-non-db-providers:
@@ -588,6 +596,7 @@ jobs:
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests
}}
use-uv: ${{ needs.build-info.outputs.use-uv }}
+ default-branch: ${{ needs.build-info.outputs.default-branch }}
if: needs.build-info.outputs.run-tests == 'true'
tests-special:
@@ -673,6 +682,7 @@ jobs:
monitor-delay-time-in-seconds: 120
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests
}}
use-uv: ${{ needs.build-info.outputs.use-uv }}
+ default-branch: ${{ needs.build-info.outputs.default-branch }}
tests-with-lowest-direct-resolution-providers:
name: "Low dep tests: providers"
@@ -702,6 +712,7 @@ jobs:
monitor-delay-time-in-seconds: 120
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests
}}
use-uv: ${{ needs.build-info.outputs.use-uv }}
+ default-branch: ${{ needs.build-info.outputs.default-branch }}
build-prod-images:
name: Build PROD images
diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml
index f351c09cb24..7664291c813 100644
--- a/.github/workflows/ci-arm.yml
+++ b/.github/workflows/ci-arm.yml
@@ -312,6 +312,7 @@ jobs:
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests
}}
use-uv: ${{ needs.build-info.outputs.use-uv }}
+ default-branch: ${{ needs.build-info.outputs.default-branch }}
if: needs.build-info.outputs.run-tests == 'true'
tests-postgres-providers:
@@ -340,6 +341,7 @@ jobs:
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests
}}
use-uv: ${{ needs.build-info.outputs.use-uv }}
+ default-branch: ${{ needs.build-info.outputs.default-branch }}
if: needs.build-info.outputs.run-tests == 'true'
tests-sqlite-core:
@@ -370,6 +372,7 @@ jobs:
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests
}}
use-uv: ${{ needs.build-info.outputs.use-uv }}
+ default-branch: ${{ needs.build-info.outputs.default-branch }}
if: needs.build-info.outputs.run-tests == 'true'
tests-sqlite-providers:
@@ -400,6 +403,7 @@ jobs:
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests
}}
use-uv: ${{ needs.build-info.outputs.use-uv }}
+ default-branch: ${{ needs.build-info.outputs.default-branch }}
if: needs.build-info.outputs.run-tests == 'true'
@@ -430,6 +434,7 @@ jobs:
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests
}}
use-uv: ${{ needs.build-info.outputs.use-uv }}
+ default-branch: ${{ needs.build-info.outputs.default-branch }}
if: needs.build-info.outputs.run-tests == 'true'
tests-non-db-providers:
@@ -459,6 +464,7 @@ jobs:
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests
}}
use-uv: ${{ needs.build-info.outputs.use-uv }}
+ default-branch: ${{ needs.build-info.outputs.default-branch }}
if: needs.build-info.outputs.run-tests == 'true'
build-prod-images:
diff --git a/.github/workflows/run-unit-tests.yml
b/.github/workflows/run-unit-tests.yml
index aa7d124648c..d330c3b96d5 100644
--- a/.github/workflows/run-unit-tests.yml
+++ b/.github/workflows/run-unit-tests.yml
@@ -120,6 +120,10 @@ on: # yamllint disable-line rule:truthy
description: "Whether to use uv"
required: true
type: string
+ default-branch:
+ description: "The default branch of the repository"
+ required: true
+ type: string
permissions:
contents: read
jobs:
@@ -164,6 +168,7 @@ jobs:
UPGRADE_BOTO: "${{ inputs.upgrade-boto }}"
AIRFLOW_MONITOR_DELAY_TIME_IN_SECONDS:
"${{inputs.monitor-delay-time-in-seconds}}"
VERBOSE: "true"
+ DEFAULT_BRANCH: "${{ inputs.default-branch }}"
if: inputs.test-group == 'core' || inputs.skip-providers-tests != 'true'
steps:
- name: "Cleanup repo"
diff --git a/.github/workflows/special-tests.yml
b/.github/workflows/special-tests.yml
index a41ab1693c8..552e47190cc 100644
--- a/.github/workflows/special-tests.yml
+++ b/.github/workflows/special-tests.yml
@@ -110,6 +110,7 @@ jobs:
debug-resources: ${{ inputs.debug-resources }}
skip-providers-tests: ${{ inputs.skip-providers-tests }}
use-uv: ${{ inputs.use-uv }}
+ default-branch: ${{ inputs.default-branch }}
tests-min-sqlalchemy-providers:
name: "Min SQLAlchemy test: providers"
@@ -134,6 +135,7 @@ jobs:
debug-resources: ${{ inputs.debug-resources }}
skip-providers-tests: ${{ inputs.skip-providers-tests }}
use-uv: ${{ inputs.use-uv }}
+ default-branch: ${{ inputs.default-branch }}
tests-boto-core:
name: "Latest Boto test: core"
@@ -159,6 +161,7 @@ jobs:
debug-resources: ${{ inputs.debug-resources }}
skip-providers-tests: ${{ inputs.skip-providers-tests }}
use-uv: ${{ inputs.use-uv }}
+ default-branch: ${{ inputs.default-branch }}
tests-boto-providers:
name: "Latest Boto test: providers"
@@ -184,7 +187,7 @@ jobs:
debug-resources: ${{ inputs.debug-resources }}
skip-providers-tests: ${{ inputs.skip-providers-tests }}
use-uv: ${{ inputs.use-uv }}
-
+ default-branch: ${{ inputs.default-branch }}
tests-pendulum-2-core:
name: "Pendulum2 test: core"
@@ -210,6 +213,7 @@ jobs:
debug-resources: ${{ inputs.debug-resources }}
skip-providers-tests: ${{ inputs.skip-providers-tests }}
use-uv: ${{ inputs.use-uv }}
+ default-branch: ${{ inputs.default-branch }}
tests-pendulum-2-providers:
name: "Pendulum2 test: providers"
@@ -235,6 +239,7 @@ jobs:
debug-resources: ${{ inputs.debug-resources }}
skip-providers-tests: ${{ inputs.skip-providers-tests }}
use-uv: ${{ inputs.use-uv }}
+ default-branch: ${{ inputs.default-branch }}
tests-quarantined-core:
name: "Quarantined test: core"
@@ -259,6 +264,7 @@ jobs:
debug-resources: ${{ inputs.debug-resources }}
skip-providers-tests: ${{ inputs.skip-providers-tests }}
use-uv: ${{ inputs.use-uv }}
+ default-branch: ${{ inputs.default-branch }}
tests-quarantined-providers:
name: "Quarantined test: providers"
@@ -283,6 +289,7 @@ jobs:
debug-resources: ${{ inputs.debug-resources }}
skip-providers-tests: ${{ inputs.skip-providers-tests }}
use-uv: ${{ inputs.use-uv }}
+ default-branch: ${{ inputs.default-branch }}
tests-system-core:
name: "System test: ${{ matrix.test-group }}"
@@ -307,3 +314,4 @@ jobs:
debug-resources: ${{ inputs.debug-resources }}
skip-providers-tests: ${{ inputs.skip-providers-tests }}
use-uv: ${{ inputs.use-uv }}
+ default-branch: ${{ inputs.default-branch }}
diff --git a/airflow-core/tests/unit/always/test_providers_manager.py
b/airflow-core/tests/unit/always/test_providers_manager.py
index 0f14fc399d6..11859e6fff5 100644
--- a/airflow-core/tests/unit/always/test_providers_manager.py
+++ b/airflow-core/tests/unit/always/test_providers_manager.py
@@ -38,7 +38,7 @@ from airflow.providers_manager import (
ProvidersManager,
)
-from tests_common.test_utils.markers import
skip_if_force_lowest_dependencies_marker
+from tests_common.test_utils.markers import
skip_if_force_lowest_dependencies_marker, skip_if_not_on_main
from tests_common.test_utils.paths import AIRFLOW_ROOT_PATH
@@ -230,6 +230,7 @@ class TestProviderManager:
raise AssertionError("There are warnings generated during hook
imports. Please fix them")
assert [w.message for w in warning_records if "hook-class-names" in
str(w.message)] == []
+ @skip_if_not_on_main
@pytest.mark.execution_timeout(150)
def test_hook_values(self):
provider_dependencies = json.loads(
diff --git a/airflow-core/tests/unit/serialization/test_dag_serialization.py
b/airflow-core/tests/unit/serialization/test_dag_serialization.py
index 234bb51fbb2..497e1111707 100644
--- a/airflow-core/tests/unit/serialization/test_dag_serialization.py
+++ b/airflow-core/tests/unit/serialization/test_dag_serialization.py
@@ -87,7 +87,7 @@ from airflow.utils.module_loading import qualname
from airflow.utils.operator_resources import Resources
from tests_common.test_utils.config import conf_vars
-from tests_common.test_utils.markers import
skip_if_force_lowest_dependencies_marker
+from tests_common.test_utils.markers import
skip_if_force_lowest_dependencies_marker, skip_if_not_on_main
from tests_common.test_utils.mock_operators import (
AirflowLink,
AirflowLink2,
@@ -477,12 +477,14 @@ class TestStringifiedDAGs:
# Skip that test if latest botocore is used - it reads all example dags
and in case latest botocore
# is upgraded to latest, usually aiobotocore can't be installed and some
of the system tests will fail with
- # import errors.
+ # import errors. Also skip if not running on main branch - some of the
example dags might fail due to
+ # outdated imports in past branches
@pytest.mark.skipif(
os.environ.get("UPGRADE_BOTO", "") == "true",
reason="This test is skipped when latest botocore is installed",
)
@skip_if_force_lowest_dependencies_marker
+ @skip_if_not_on_main
@pytest.mark.db_test
def test_serialization(self):
"""Serialization and deserialization should work for every DAG and
Operator."""
diff --git a/devel-common/src/tests_common/test_utils/markers.py
b/devel-common/src/tests_common/test_utils/markers.py
index 2d7b4407077..2eb91e3faf2 100644
--- a/devel-common/src/tests_common/test_utils/markers.py
+++ b/devel-common/src/tests_common/test_utils/markers.py
@@ -25,3 +25,9 @@ skip_if_force_lowest_dependencies_marker = pytest.mark.skipif(
os.environ.get("FORCE_LOWEST_DEPENDENCIES", "") == "true",
reason="When lowest dependencies are set only some providers are loaded",
)
+
+
+skip_if_not_on_main = pytest.mark.skipif(
+ os.environ.get("DEFAULT_BRANCH", "main") != "main",
+ reason="This test is only run on main branch in CI",
+)
