This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 11acefa462b Cleanup: Remove redundant code (#48817)
11acefa462b is described below
commit 11acefa462b6b8fe2b43042e23ce3ccba7e878e3
Author: Kaxil Naik <[email protected]>
AuthorDate: Sat Apr 5 12:25:54 2025 +0530
Cleanup: Remove redundant code (#48817)
---
airflow-core/src/airflow/utils/api_migration.py | 37 --------
airflow-core/src/airflow/utils/cli.py | 4 -
airflow-core/src/airflow/utils/configuration.py | 50 -----------
airflow-core/src/airflow/utils/db.py | 4 -
airflow-core/src/airflow/utils/docs.py | 37 --------
airflow-core/src/airflow/utils/helpers.py | 56 +-----------
airflow-core/src/airflow/utils/mixins.py | 46 ----------
airflow-core/src/airflow/utils/sqlalchemy.py | 23 +----
airflow-core/src/airflow/utils/task_group.py | 80 -----------------
airflow-core/tests/unit/utils/test_helpers.py | 104 -----------------------
airflow-core/tests/unit/utils/test_task_group.py | 45 ++--------
task-sdk/src/airflow/sdk/definitions/dag.py | 1 -
12 files changed, 11 insertions(+), 476 deletions(-)
diff --git a/airflow-core/src/airflow/utils/api_migration.py
b/airflow-core/src/airflow/utils/api_migration.py
deleted file mode 100644
index 28baa1cdc5a..00000000000
--- a/airflow-core/src/airflow/utils/api_migration.py
+++ /dev/null
@@ -1,37 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-Utilities for migration legacy endpoints to FastAPI.
-
-This module can be deleted once the AIP-84 is completed and the legacy API is
deleted.
-"""
-
-from __future__ import annotations
-
-from typing import Callable, TypeVar
-
-RT = TypeVar("RT")
-
-
-def mark_fastapi_migration_done(function: Callable[..., RT]) -> Callable[...,
RT]:
- """
- Mark an endpoint as migrated over to the new FastAPI API.
-
- This will help track what endpoints need to be migrated and the one that
can be safely deleted.
- """
- return function
diff --git a/airflow-core/src/airflow/utils/cli.py
b/airflow-core/src/airflow/utils/cli.py
index e2137237f69..be27cd47677 100644
--- a/airflow-core/src/airflow/utils/cli.py
+++ b/airflow-core/src/airflow/utils/cli.py
@@ -381,10 +381,6 @@ def should_use_colors(args) -> bool:
return is_terminal_support_colors()
-def should_ignore_depends_on_past(args) -> bool:
- return args.depends_on_past == "ignore"
-
-
def suppress_logs_and_warning(f: T) -> T:
"""Suppress logging and warning messages in cli functions."""
diff --git a/airflow-core/src/airflow/utils/configuration.py
b/airflow-core/src/airflow/utils/configuration.py
deleted file mode 100644
index 5d1bf51a580..00000000000
--- a/airflow-core/src/airflow/utils/configuration.py
+++ /dev/null
@@ -1,50 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-import json
-import os
-from tempfile import mkstemp
-
-from airflow.configuration import conf
-from airflow.utils.platform import IS_WINDOWS
-
-
-def tmp_configuration_copy(chmod=0o600, include_env=True, include_cmds=True):
- """
- Return a path for a temporary file including a full copy of the
configuration settings.
-
- :param include_env: Should the value of configuration from ``AIRFLOW__``
- environment variables be included or not
- :param include_cmds: Should the result of calling any ``*_cmd`` config be
- set (True, default), or should the _cmd options be left as the
- command to run (False)
- :return: a path to a temporary file
- """
- cfg_dict = conf.as_dict(
- display_sensitive=True, raw=True, include_cmds=include_cmds,
include_env=include_env
- )
- temp_fd, cfg_path = mkstemp()
-
- with os.fdopen(temp_fd, "w") as temp_file:
- # Set the permissions before we write anything to it.
- if chmod is not None and not IS_WINDOWS:
- os.fchmod(temp_fd, chmod)
- json.dump(cfg_dict, temp_file)
-
- return cfg_path
diff --git a/airflow-core/src/airflow/utils/db.py
b/airflow-core/src/airflow/utils/db.py
index 483c363425e..477858b733d 100644
--- a/airflow-core/src/airflow/utils/db.py
+++ b/airflow-core/src/airflow/utils/db.py
@@ -96,10 +96,6 @@ _REVISION_HEADS_MAP: dict[str, str] = {
}
-def _format_airflow_moved_table_name(source_table, version, category):
- return "__".join([settings.AIRFLOW_MOVED_TABLE_PREFIX,
version.replace(".", "_"), category, source_table])
-
-
@provide_session
def merge_conn(conn: Connection, session: Session = NEW_SESSION):
"""Add new Connection."""
diff --git a/airflow-core/src/airflow/utils/docs.py
b/airflow-core/src/airflow/utils/docs.py
index a1217091c52..b256b18c383 100644
--- a/airflow-core/src/airflow/utils/docs.py
+++ b/airflow-core/src/airflow/utils/docs.py
@@ -16,13 +16,6 @@
# under the License.
from __future__ import annotations
-import sys
-
-if sys.version_info >= (3, 10):
- from importlib import metadata
-else:
- import importlib_metadata as metadata # type: ignore[no-redef]
-
def get_docs_url(page: str | None = None) -> str:
"""Prepare link to Airflow documentation."""
@@ -37,33 +30,3 @@ def get_docs_url(page: str | None = None) -> str:
if page:
result = result + page
return result
-
-
-def get_project_url_from_metadata(provider_name: str):
- """Return the Project-URL from metadata."""
- return metadata.metadata(provider_name).get_all("Project-URL")
-
-
-def get_doc_url_for_provider(provider_name: str, provider_version: str) -> str:
- """Prepare link to Airflow Provider documentation."""
- try:
- from urllib.parse import urlparse
-
- metadata_items = get_project_url_from_metadata(provider_name)
- if isinstance(metadata_items, str):
- metadata_items = [metadata_items]
- if metadata_items:
- for item in metadata_items:
- if item.lower().startswith("documentation"):
- _, _, url = item.partition(",")
- if url:
- url = url.strip()
- parsed_url = urlparse(url)
- if url and (parsed_url.scheme in ("http", "https") and
bool(parsed_url.netloc)):
- return url
- except metadata.PackageNotFoundError:
- pass
- # Fallback if provider is apache one
- if provider_name.startswith("apache-airflow"):
- return
f"https://airflow.apache.org/docs/{provider_name}/{provider_version}/"
- return
"https://airflow.apache.org/docs/apache-airflow-providers/index.html#creating-your-own-providers"
diff --git a/airflow-core/src/airflow/utils/helpers.py
b/airflow-core/src/airflow/utils/helpers.py
index 881379b79f9..601373dd32a 100644
--- a/airflow-core/src/airflow/utils/helpers.py
+++ b/airflow-core/src/airflow/utils/helpers.py
@@ -22,8 +22,7 @@ import itertools
import re
import signal
from collections.abc import Generator, Iterable, Mapping, MutableMapping
-from datetime import datetime
-from functools import cache, reduce
+from functools import cache
from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast
from urllib.parse import urljoin
@@ -60,42 +59,6 @@ def validate_key(k: str, max_length: int = 250):
)
-def validate_instance_args(instance: object, expected_arg_types: dict[str,
Any]) -> None:
- """Validate that the instance has the expected types for the arguments."""
- for arg_name, expected_arg_type in expected_arg_types.items():
- instance_arg_value = getattr(instance, arg_name, None)
- if instance_arg_value is not None and not
isinstance(instance_arg_value, expected_arg_type):
- raise TypeError(
- f"'{arg_name}' has an invalid type {type(instance_arg_value)}
with value "
- f"{instance_arg_value}, expected type is {expected_arg_type}"
- )
-
-
-def validate_group_key(k: str, max_length: int = 200):
- """Validate value used as a group key."""
- if not isinstance(k, str):
- raise TypeError(f"The key has to be a string and is {type(k)}:{k}")
- if len(k) > max_length:
- raise AirflowException(f"The key has to be less than {max_length}
characters")
- if not GROUP_KEY_REGEX.match(k):
- raise AirflowException(
- f"The key {k!r} has to be made of alphanumeric characters, dashes
and underscores exclusively"
- )
-
-
-def alchemy_to_dict(obj: Any) -> dict | None:
- """Transform a SQLAlchemy model instance into a dictionary."""
- if not obj:
- return None
- output = {}
- for col in obj.__table__.columns:
- value = getattr(obj, col.name)
- if isinstance(value, datetime):
- value = value.isoformat()
- output[col.name] = value
- return output
-
-
def ask_yesno(question: str, default: bool | None = None) -> bool:
"""Get a yes or no answer from the user."""
yes = {"yes", "y"}
@@ -137,14 +100,6 @@ def is_container(obj: Any) -> bool:
return hasattr(obj, "__iter__") and not isinstance(obj, str)
-def as_tuple(obj: Any) -> tuple:
- """Return obj as a tuple if obj is a container, otherwise return a tuple
containing obj."""
- if is_container(obj):
- return tuple(obj)
- else:
- return tuple([obj])
-
-
def chunks(items: list[T], chunk_size: int) -> Generator[list[T], None, None]:
"""Yield successive chunks of a given size from a list of items."""
if chunk_size <= 0:
@@ -153,15 +108,6 @@ def chunks(items: list[T], chunk_size: int) ->
Generator[list[T], None, None]:
yield items[i : i + chunk_size]
-def reduce_in_chunks(fn: Callable[[S, list[T]], S], iterable: list[T],
initializer: S, chunk_size: int = 0):
- """Split the list of items into chunks of a given size and pass each chunk
through the reducer."""
- if not iterable:
- return initializer
- if chunk_size == 0:
- chunk_size = len(iterable)
- return reduce(fn, chunks(iterable, chunk_size), initializer)
-
-
def as_flattened_list(iterable: Iterable[Iterable[T]]) -> list[T]:
"""
Return an iterable with one level flattened.
diff --git a/airflow-core/src/airflow/utils/mixins.py
b/airflow-core/src/airflow/utils/mixins.py
deleted file mode 100644
index 78484f8375e..00000000000
--- a/airflow-core/src/airflow/utils/mixins.py
+++ /dev/null
@@ -1,46 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from __future__ import annotations
-
-import multiprocessing
-import multiprocessing.context
-
-from airflow.configuration import conf
-
-
-class MultiprocessingStartMethodMixin:
- """Convenience class to add support for different types of
multiprocessing."""
-
- def _get_multiprocessing_start_method(self) -> str:
- """
- Determine method of creating new processes.
-
- Checks if the mp_start_method is set in configs, else, it uses the OS
default.
- """
- if conf.has_option("core", "mp_start_method"):
- return conf.get_mandatory_value("core", "mp_start_method")
-
- method = multiprocessing.get_start_method()
- if not method:
- raise ValueError("Failed to determine start method")
- return method
-
- def _get_multiprocessing_context(self) ->
multiprocessing.context.DefaultContext:
- mp_start_method = self._get_multiprocessing_start_method()
- return multiprocessing.get_context(mp_start_method) # type: ignore
diff --git a/airflow-core/src/airflow/utils/sqlalchemy.py
b/airflow-core/src/airflow/utils/sqlalchemy.py
index 818d0e97576..3e9d3d85555 100644
--- a/airflow-core/src/airflow/utils/sqlalchemy.py
+++ b/airflow-core/src/airflow/utils/sqlalchemy.py
@@ -21,12 +21,12 @@ import contextlib
import copy
import datetime
import logging
-from collections.abc import Generator, Iterable
+from collections.abc import Generator
from importlib import metadata
from typing import TYPE_CHECKING, Any
from packaging import version
-from sqlalchemy import TIMESTAMP, PickleType, event, nullsfirst, tuple_
+from sqlalchemy import TIMESTAMP, PickleType, event, nullsfirst
from sqlalchemy.dialects import mysql
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.types import JSON, Text, TypeDecorator
@@ -39,8 +39,6 @@ if TYPE_CHECKING:
from kubernetes.client.models.v1_pod import V1Pod
from sqlalchemy.exc import OperationalError
from sqlalchemy.orm import Query, Session
- from sqlalchemy.sql import ColumnElement, Select
- from sqlalchemy.sql.expression import ColumnOperators
from sqlalchemy.types import TypeEngine
@@ -442,23 +440,6 @@ def is_lock_not_available_error(error: OperationalError):
return False
-def tuple_in_condition(
- columns: tuple[ColumnElement, ...],
- collection: Iterable[Any] | Select,
- *,
- session: Session | None = None,
-) -> ColumnOperators:
- """
- Generate a tuple-in-collection operator to use in ``.where()``.
-
- Kept for backward compatibility. Remove when providers drop support for
- apache-airflow<3.0.
-
- :meta private:
- """
- return tuple_(*columns).in_(collection)
-
-
def get_orm_mapper():
"""Get the correct ORM mapper for the installed SQLAlchemy version."""
import sqlalchemy.orm.mapper
diff --git a/airflow-core/src/airflow/utils/task_group.py
b/airflow-core/src/airflow/utils/task_group.py
index 27d3af1ef1c..51270fa473a 100644
--- a/airflow-core/src/airflow/utils/task_group.py
+++ b/airflow-core/src/airflow/utils/task_group.py
@@ -82,83 +82,3 @@ def task_group_to_dict(task_item_or_group,
parent_group_is_mapped=False):
"children": children,
"type": "task",
}
-
-
-def task_group_to_dict_legacy(task_item_or_group):
- """
- Legacy function to create a nested dict representation of this TaskGroup
and its children used to construct the Graph.
-
- TODO: To remove for airflow 3 once the legacy UI is deleted.
- """
- from airflow.models.abstractoperator import AbstractOperator
- from airflow.models.mappedoperator import MappedOperator
-
- if isinstance(task := task_item_or_group, AbstractOperator):
- setup_teardown_type = {}
- is_mapped = {}
- if task.is_setup is True:
- setup_teardown_type["setupTeardownType"] = "setup"
- elif task.is_teardown is True:
- setup_teardown_type["setupTeardownType"] = "teardown"
- if isinstance(task, MappedOperator):
- is_mapped["isMapped"] = True
- return {
- "id": task.task_id,
- "value": {
- "label": task.label,
- "labelStyle": f"fill:{task.ui_fgcolor};",
- "style": f"fill:{task.ui_color};",
- "rx": 5,
- "ry": 5,
- **is_mapped,
- **setup_teardown_type,
- },
- }
- task_group = task_item_or_group
- is_mapped = isinstance(task_group, MappedTaskGroup)
- children = [
- task_group_to_dict_legacy(child)
- for child in sorted(task_group.children.values(), key=lambda t:
t.label)
- ]
-
- if task_group.upstream_group_ids or task_group.upstream_task_ids:
- children.append(
- {
- "id": task_group.upstream_join_id,
- "value": {
- "label": "",
- "labelStyle": f"fill:{task_group.ui_fgcolor};",
- "style": f"fill:{task_group.ui_color};",
- "shape": "circle",
- },
- }
- )
-
- if task_group.downstream_group_ids or task_group.downstream_task_ids:
- # This is the join node used to reduce the number of edges between two
TaskGroup.
- children.append(
- {
- "id": task_group.downstream_join_id,
- "value": {
- "label": "",
- "labelStyle": f"fill:{task_group.ui_fgcolor};",
- "style": f"fill:{task_group.ui_color};",
- "shape": "circle",
- },
- }
- )
-
- return {
- "id": task_group.group_id,
- "value": {
- "label": task_group.label,
- "labelStyle": f"fill:{task_group.ui_fgcolor};",
- "style": f"fill:{task_group.ui_color}",
- "rx": 5,
- "ry": 5,
- "clusterLabelPos": "top",
- "tooltip": task_group.tooltip,
- "isMapped": is_mapped,
- },
- "children": children,
- }
diff --git a/airflow-core/tests/unit/utils/test_helpers.py
b/airflow-core/tests/unit/utils/test_helpers.py
index e046dcf6001..401f0c48fcf 100644
--- a/airflow-core/tests/unit/utils/test_helpers.py
+++ b/airflow-core/tests/unit/utils/test_helpers.py
@@ -32,8 +32,6 @@ from airflow.utils.helpers import (
exactly_one,
merge_dicts,
prune_dict,
- validate_group_key,
- validate_instance_args,
validate_key,
)
from airflow.utils.types import NOTSET
@@ -83,13 +81,6 @@ class TestHelpers:
assert list(helpers.chunks([1], 1)) == [[1]]
assert list(helpers.chunks([1, 2, 3], 2)) == [[1, 2], [3]]
- def test_reduce_in_chunks(self):
- assert helpers.reduce_in_chunks(lambda x, y: [*x, y], [1, 2, 3, 4, 5],
[]) == [[1, 2, 3, 4, 5]]
-
- assert helpers.reduce_in_chunks(lambda x, y: [*x, y], [1, 2, 3, 4, 5],
[], 2) == [[1, 2], [3, 4], [5]]
-
- assert helpers.reduce_in_chunks(lambda x, y: x + y[0] * y[1], [1, 2,
3, 4], 0, 2) == 14
-
def test_is_container(self):
assert not helpers.is_container("a string is not a container")
assert helpers.is_container(["a", "list", "is", "a", "container"])
@@ -99,27 +90,6 @@ class TestHelpers:
# Pass an object that is not iter nor a string.
assert not helpers.is_container(10)
- def test_as_tuple(self):
- assert helpers.as_tuple("a string is not a container") == ("a string
is not a container",)
-
- assert helpers.as_tuple(["a", "list", "is", "a", "container"]) == (
- "a",
- "list",
- "is",
- "a",
- "container",
- )
-
- def test_as_tuple_iter(self):
- test_list = ["test_str"]
- as_tup = helpers.as_tuple(test_list)
- assert tuple(test_list) == as_tup
-
- def test_as_tuple_no_iter(self):
- test_str = "test_str"
- as_tup = helpers.as_tuple(test_str)
- assert (test_str,) == as_tup
-
def test_convert_camel_to_snake(self):
assert helpers.convert_camel_to_snake("LocalTaskJob") ==
"local_task_job"
assert helpers.convert_camel_to_snake("somethingVeryRandom") ==
"something_very_random"
@@ -195,47 +165,6 @@ class TestHelpers:
else:
validate_key(key_id)
- @pytest.mark.parametrize(
- "key_id, message, exception",
- [
- (3, "The key has to be a string and is <class 'int'>:3",
TypeError),
- (None, "The key has to be a string and is <class
'NoneType'>:None", TypeError),
- ("simple_key", None, None),
- ("simple-key", None, None),
- (
- "group.simple_key",
- "The key 'group.simple_key' has to be made of alphanumeric "
- "characters, dashes and underscores exclusively",
- AirflowException,
- ),
- (
- "root.group-name.simple_key",
- "The key 'root.group-name.simple_key' has to be made of
alphanumeric "
- "characters, dashes and underscores exclusively",
- AirflowException,
- ),
- (
- "key with space",
- "The key 'key with space' has to be made of alphanumeric "
- "characters, dashes and underscores exclusively",
- AirflowException,
- ),
- (
- "key_with_!",
- "The key 'key_with_!' has to be made of alphanumeric "
- "characters, dashes and underscores exclusively",
- AirflowException,
- ),
- (" " * 201, "The key has to be less than 200 characters",
AirflowException),
- ],
- )
- def test_validate_group_key(self, key_id, message, exception):
- if message:
- with pytest.raises(exception, match=re.escape(message)):
- validate_group_key(key_id)
- else:
- validate_group_key(key_id)
-
def test_exactly_one(self):
"""
Checks that when we set ``true_count`` elements to "truthy", and
others to "falsy",
@@ -341,36 +270,3 @@ class SchedulerJobRunner(MockJobRunner):
class TriggererJobRunner(MockJobRunner):
job_type = "TriggererJob"
-
-
-class ClassToValidateArgs:
- def __init__(self, name, age, active):
- self.name = name
- self.age = age
- self.active = active
-
-
-# Edge cases
[email protected](
- "instance, expected_arg_types",
- [
- (ClassToValidateArgs("Alice", 30, None), {"name": str, "age": int,
"active": bool}),
- (ClassToValidateArgs(None, 25, True), {"name": str, "age": int,
"active": bool}),
- ],
-)
-def test_validate_instance_args_raises_no_error(instance, expected_arg_types):
- validate_instance_args(instance, expected_arg_types)
-
-
-# Error cases
[email protected](
- "instance, expected_arg_types",
- [
- (ClassToValidateArgs("Alice", "thirty", True), {"name": str, "age":
int, "active": bool}),
- (ClassToValidateArgs("Bob", 25, "yes"), {"name": str, "age": int,
"active": bool}),
- (ClassToValidateArgs(123, 25, True), {"name": str, "age": int,
"active": bool}),
- ],
-)
-def test_validate_instance_args_raises_error(instance, expected_arg_types):
- with pytest.raises(TypeError):
- validate_instance_args(instance, expected_arg_types)
diff --git a/airflow-core/tests/unit/utils/test_task_group.py
b/airflow-core/tests/unit/utils/test_task_group.py
index ac1c1ee4cc5..a49c3871750 100644
--- a/airflow-core/tests/unit/utils/test_task_group.py
+++ b/airflow-core/tests/unit/utils/test_task_group.py
@@ -35,7 +35,7 @@ from airflow.models.dag import DAG
from airflow.models.xcom_arg import XComArg
from airflow.providers.standard.operators.empty import EmptyOperator
from airflow.utils.dag_edges import dag_edges
-from airflow.utils.task_group import TaskGroup, task_group_to_dict,
task_group_to_dict_legacy
+from airflow.utils.task_group import TaskGroup, task_group_to_dict
from tests_common.test_utils.compat import BashOperator, PythonOperator
from unit.models import DEFAULT_DATE
@@ -244,7 +244,6 @@ def test_build_task_group_context_manager():
assert set(dag.task_group.children.keys()) == {"task1", "group234",
"task5"}
assert group34.group_id == "group234.group34"
- assert task_group_to_dict_legacy(dag.task_group) == EXPECTED_JSON_LEGACY
assert task_group_to_dict(dag.task_group) == EXPECTED_JSON
@@ -266,21 +265,17 @@ def test_build_task_group():
task1 >> group234
group34 >> task5
- assert task_group_to_dict_legacy(dag.task_group) == EXPECTED_JSON_LEGACY
assert task_group_to_dict(dag.task_group) == EXPECTED_JSON
-def extract_node_id(node, include_label=False, from_legacy=False):
+def extract_node_id(node, include_label=False):
ret = {"id": node["id"]}
if include_label:
- if from_legacy:
- ret["label"] = node["value"]["label"]
- else:
- ret["label"] = node["label"]
+ ret["label"] = node["label"]
if "children" in node:
children = []
for child in node["children"]:
- children.append(extract_node_id(child,
include_label=include_label, from_legacy=from_legacy))
+ children.append(extract_node_id(child,
include_label=include_label))
ret["children"] = children
@@ -347,10 +342,6 @@ def test_build_task_group_with_prefix():
],
}
- assert (
- extract_node_id(task_group_to_dict_legacy(dag.task_group),
include_label=True, from_legacy=True)
- == expected_node_id
- )
assert extract_node_id(task_group_to_dict(dag.task_group),
include_label=True) == expected_node_id
@@ -415,7 +406,6 @@ def test_build_task_group_with_task_decorator():
],
}
- assert extract_node_id(task_group_to_dict_legacy(dag.task_group),
from_legacy=True) == expected_node_id
assert extract_node_id(task_group_to_dict(dag.task_group)) ==
expected_node_id
edges = dag_edges(dag)
@@ -479,7 +469,6 @@ def test_sub_dag_task_group():
],
}
- assert extract_node_id(task_group_to_dict_legacy(subdag.task_group),
from_legacy=True) == expected_node_id
assert extract_node_id(task_group_to_dict(subdag.task_group)) ==
expected_node_id
edges = dag_edges(subdag)
@@ -547,8 +536,7 @@ def test_dag_edges():
group_d << group_c
- nodes_legacy = task_group_to_dict_legacy(dag.task_group)
- nodes = task_group_to_dict_legacy(dag.task_group)
+ nodes = task_group_to_dict(dag.task_group)
edges = dag_edges(dag)
expected_node_id = {
@@ -595,8 +583,7 @@ def test_dag_edges():
],
}
- assert extract_node_id(nodes_legacy) == expected_node_id
- assert extract_node_id(nodes, from_legacy=False) == expected_node_id
+ assert extract_node_id(nodes) == expected_node_id
assert sorted((e["source_id"], e["target_id"]) for e in edges) == [
("group_a.downstream_join_id", "group_c.upstream_join_id"),
@@ -852,7 +839,6 @@ def test_build_task_group_deco_context_manager():
],
}
- assert extract_node_id(task_group_to_dict_legacy(dag.task_group),
from_legacy=True) == node_ids
assert extract_node_id(task_group_to_dict(dag.task_group)) == node_ids
@@ -1031,7 +1017,6 @@ def test_task_group_context_mix():
],
}
- assert extract_node_id(task_group_to_dict_legacy(dag.task_group),
from_legacy=True) == node_ids
assert extract_node_id(task_group_to_dict(dag.task_group)) == node_ids
@@ -1124,7 +1109,6 @@ def test_duplicate_task_group_id():
],
}
- assert extract_node_id(task_group_to_dict_legacy(dag.task_group),
from_legacy=True) == node_ids
assert extract_node_id(task_group_to_dict(dag.task_group)) == node_ids
@@ -1187,7 +1171,6 @@ def test_call_taskgroup_twice():
],
}
- assert extract_node_id(task_group_to_dict_legacy(dag.task_group),
from_legacy=True) == node_ids
assert extract_node_id(task_group_to_dict(dag.task_group)) == node_ids
@@ -1657,25 +1640,13 @@ def test_task_group_arrow_with_setup_group():
assert set(t1.operator.downstream_task_ids) == set()
assert set(t2.operator.downstream_task_ids) == set()
- def get_nodes(group, from_legacy=False):
- if from_legacy:
- d = task_group_to_dict_legacy(group)
- else:
- d = task_group_to_dict(group)
+ def get_nodes(group):
+ d = task_group_to_dict(group)
new_d = {}
new_d["id"] = d["id"]
new_d["children"] = [{"id": x["id"]} for x in d["children"]]
return new_d
- assert get_nodes(g1, from_legacy=True) == {
- "id": "group_1",
- "children": [
- {"id": "group_1.setup_1"},
- {"id": "group_1.setup_2"},
- {"id": "group_1.downstream_join_id"},
- ],
- }
-
assert get_nodes(g1) == {
"id": "group_1",
"children": [
diff --git a/task-sdk/src/airflow/sdk/definitions/dag.py
b/task-sdk/src/airflow/sdk/definitions/dag.py
index 6b157a6d14e..691a4915011 100644
--- a/task-sdk/src/airflow/sdk/definitions/dag.py
+++ b/task-sdk/src/airflow/sdk/definitions/dag.py
@@ -70,7 +70,6 @@ from airflow.utils.decorators import
fixup_decorator_warning_stack
from airflow.utils.trigger_rule import TriggerRule
if TYPE_CHECKING:
- # TODO: Task-SDK: Remove pendulum core dep
from pendulum.tz.timezone import FixedTimezone, Timezone
from airflow.decorators import TaskDecoratorCollection