This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new c29b6e8e0ba Move `list_py_file_paths` test to the right file (#45521)
c29b6e8e0ba is described below
commit c29b6e8e0ba739aa66ddf371cd3fa0c79e3ce651
Author: Jed Cunningham <[email protected]>
AuthorDate: Thu Jan 9 14:14:58 2025 -0700
Move `list_py_file_paths` test to the right file (#45521)
---
tests/jobs/test_scheduler_job.py | 31 -------------------------------
tests/utils/test_file.py | 34 +++++++++++++++++++++++++++++++++-
2 files changed, 33 insertions(+), 32 deletions(-)
diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py
index 497ae4fb1d4..328ae1742fb 100644
--- a/tests/jobs/test_scheduler_job.py
+++ b/tests/jobs/test_scheduler_job.py
@@ -69,7 +69,6 @@ from airflow.sdk.definitions.asset import Asset
from airflow.serialization.serialized_objects import SerializedDAG
from airflow.timetables.base import DataInterval
from airflow.utils import timezone
-from airflow.utils.file import list_py_file_paths
from airflow.utils.session import create_session, provide_session
from airflow.utils.state import DagRunState, State, TaskInstanceState
from airflow.utils.types import DagRunType
@@ -3531,36 +3530,6 @@ class TestSchedulerJob:
assert logical_date == running_date, "Running Date must match
Execution Date"
- def test_list_py_file_paths(self):
- """
- [JIRA-1357] Test the 'list_py_file_paths' function used by the
- scheduler to list and load DAGs.
- """
- detected_files = set()
- expected_files = set()
- # No_dags is empty, _invalid_ is ignored by .airflowignore
- ignored_files = {
- "no_dags.py",
- "test_invalid_cron.py",
- "test_invalid_dup_task.py",
- "test_ignore_this.py",
- "test_invalid_param.py",
- "test_invalid_param2.py",
- "test_invalid_param3.py",
- "test_invalid_param4.py",
- "test_nested_dag.py",
- "test_imports.py",
- "__init__.py",
- }
- for root, _, files in os.walk(TEST_DAG_FOLDER):
- for file_name in files:
- if file_name.endswith((".py", ".zip")):
- if file_name not in ignored_files:
- expected_files.add(f"{root}/{file_name}")
- for file_path in list_py_file_paths(TEST_DAG_FOLDER):
- detected_files.add(file_path)
- assert detected_files == expected_files
-
def test_adopt_or_reset_orphaned_tasks_nothing(self):
"""Try with nothing."""
scheduler_job = Job()
diff --git a/tests/utils/test_file.py b/tests/utils/test_file.py
index da1c6fa2fc8..6b5b5b55004 100644
--- a/tests/utils/test_file.py
+++ b/tests/utils/test_file.py
@@ -25,11 +25,18 @@ from unittest import mock
import pytest
from airflow.utils import file as file_utils
-from airflow.utils.file import correct_maybe_zipped, find_path_from_directory,
open_maybe_zipped
+from airflow.utils.file import (
+ correct_maybe_zipped,
+ find_path_from_directory,
+ list_py_file_paths,
+ open_maybe_zipped,
+)
from tests.models import TEST_DAGS_FOLDER
from tests_common.test_utils.config import conf_vars
+TEST_DAG_FOLDER = os.environ["AIRFLOW__CORE__DAGS_FOLDER"]
+
def might_contain_dag(file_path: str, zip_file: zipfile.ZipFile | None = None):
return False
@@ -212,6 +219,31 @@ class TestListPyFilesPath:
assert len(modules) == 0
+ def test_list_py_file_paths(self):
+ detected_files = set()
+ expected_files = set()
+ # No_dags is empty, _invalid_ is ignored by .airflowignore
+ ignored_files = {
+ "no_dags.py",
+ "test_invalid_cron.py",
+ "test_invalid_dup_task.py",
+ "test_ignore_this.py",
+ "test_invalid_param.py",
+ "test_invalid_param2.py",
+ "test_invalid_param3.py",
+ "test_invalid_param4.py",
+ "test_nested_dag.py",
+ "test_imports.py",
+ "__init__.py",
+ }
+ for root, _, files in os.walk(TEST_DAG_FOLDER):
+ for file_name in files:
+ if file_name.endswith((".py", ".zip")):
+ if file_name not in ignored_files:
+ expected_files.add(f"{root}/{file_name}")
+ detected_files = set(list_py_file_paths(TEST_DAG_FOLDER))
+ assert detected_files == expected_files
+
@pytest.mark.parametrize(
"edge_filename, expected_modification",
