This is an automated email from the ASF dual-hosted git repository.
jscheffl pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 1a8ef7c1448 Remove helm-related code from airflow-core (#64427)
1a8ef7c1448 is described below
commit 1a8ef7c14480dda308644aef4e554a540d845e89
Author: Przemysław Mirowski <[email protected]>
AuthorDate: Mon Mar 30 22:53:57 2026 +0200
Remove helm-related code from airflow-core (#64427)
* Remove helm-related code from airflow-core
* Update breeze tests
---
airflow-core/tests/unit/charts/__init__.py | 16 -
airflow-core/tests/unit/charts/conftest.py | 25 --
.../tests/unit/charts/helm_template_generator.py | 170 ----------
airflow-core/tests/unit/charts/log_groomer.py | 345 ---------------------
.../tests/test_pytest_args_for_test_types.py | 1 -
5 files changed, 557 deletions(-)
diff --git a/airflow-core/tests/unit/charts/__init__.py
b/airflow-core/tests/unit/charts/__init__.py
deleted file mode 100644
index 13a83393a91..00000000000
--- a/airflow-core/tests/unit/charts/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/airflow-core/tests/unit/charts/conftest.py
b/airflow-core/tests/unit/charts/conftest.py
deleted file mode 100644
index 6db2ca0b46e..00000000000
--- a/airflow-core/tests/unit/charts/conftest.py
+++ /dev/null
@@ -1,25 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-import pytest
-
-
[email protected](autouse=True, scope="session")
-def initialize_airflow_tests(request):
- # Skip airflow tests initialization for all Helm tests
- return
diff --git a/airflow-core/tests/unit/charts/helm_template_generator.py
b/airflow-core/tests/unit/charts/helm_template_generator.py
deleted file mode 100644
index 04e6828e877..00000000000
--- a/airflow-core/tests/unit/charts/helm_template_generator.py
+++ /dev/null
@@ -1,170 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-import json
-import subprocess
-from functools import cache
-from io import StringIO
-from pathlib import Path
-from tempfile import NamedTemporaryFile
-from typing import Any
-
-import jmespath
-import jsonschema
-import requests
-import yaml
-from kubernetes.client.api_client import ApiClient
-
-api_client = ApiClient()
-
-CHART_DIR = Path(__file__).resolve().parents[4] / "chart"
-
-DEFAULT_KUBERNETES_VERSION = "1.30.13"
-BASE_URL_SPEC = (
- f"https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/";
- f"v{DEFAULT_KUBERNETES_VERSION}-standalone-strict"
-)
-
-crd_lookup = {
- "keda.sh/v1alpha1::ScaledObject":
"https://raw.githubusercontent.com/kedacore/keda/v2.0.0/config/crd/bases/keda.sh_scaledobjects.yaml";,
- # This object type was removed in k8s v1.22.0
- "networking.k8s.io/v1beta1::Ingress":
"https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.21.0/ingress-networking-v1beta1.json";,
-}
-
-
-@cache
-def get_schema_k8s(api_version, kind, kubernetes_version):
- api_version = api_version.lower()
- kind = kind.lower()
-
- if "/" in api_version:
- ext, _, api_version = api_version.partition("/")
- ext = ext.split(".")[0]
- url = f"{BASE_URL_SPEC}/{kind}-{ext}-{api_version}.json"
- else:
- url = f"{BASE_URL_SPEC}/{kind}-{api_version}.json"
- request = requests.get(url)
- request.raise_for_status()
- schema = json.loads(
- request.text.replace(
- "kubernetesjsonschema.dev",
"raw.githubusercontent.com/yannh/kubernetes-json-schema/master"
- )
- )
- return schema
-
-
-def get_schema_crd(api_version, kind):
- url = crd_lookup.get(f"{api_version}::{kind}")
- if not url:
- return None
- response = requests.get(url)
- yaml_schema = response.content.decode("utf-8")
- schema = yaml.safe_load(StringIO(yaml_schema))
- return schema
-
-
-@cache
-def create_validator(api_version, kind, kubernetes_version):
- schema = get_schema_crd(api_version, kind)
- if not schema:
- schema = get_schema_k8s(api_version, kind, kubernetes_version)
- jsonschema.Draft7Validator.check_schema(schema)
- validator = jsonschema.Draft7Validator(schema)
- return validator
-
-
-def validate_k8s_object(instance, kubernetes_version):
- # Skip PostgreSQL chart
- labels = jmespath.search("metadata.labels", instance)
- if "helm.sh/chart" in labels:
- chart = labels["helm.sh/chart"]
- else:
- chart = labels.get("chart")
-
- if chart and "postgresql" in chart:
- return
-
- validate = create_validator(instance.get("apiVersion"),
instance.get("kind"), kubernetes_version)
- validate.validate(instance)
-
-
-class HelmFailedError(subprocess.CalledProcessError):
- def __str__(self):
- return f"Helm command failed. Args: {self.args}\nStderr:
\n{self.stderr.decode('utf-8')}"
-
-
-def render_chart(
- name="release-name",
- values=None,
- show_only=None,
- chart_dir=None,
- kubernetes_version=DEFAULT_KUBERNETES_VERSION,
- namespace=None,
-):
- """
- Function that renders a helm chart into dictionaries. For helm chart
testing only
- """
- values = values or {}
- chart_dir = chart_dir or str(CHART_DIR)
- namespace = namespace or "default"
- with NamedTemporaryFile() as tmp_file:
- content = yaml.dump(values)
- tmp_file.write(content.encode())
- tmp_file.flush()
- command = [
- "helm",
- "template",
- name,
- chart_dir,
- "--values",
- tmp_file.name,
- "--kube-version",
- kubernetes_version,
- "--namespace",
- namespace,
- ]
- if show_only:
- for i in show_only:
- command.extend(["--show-only", i])
- result = subprocess.run(command, check=False, capture_output=True,
cwd=chart_dir)
- if result.returncode:
- raise HelmFailedError(result.returncode, result.args,
result.stdout, result.stderr)
- templates = result.stdout
- k8s_objects = yaml.full_load_all(templates)
- k8s_objects = [k8s_object for k8s_object in k8s_objects if k8s_object]
- for k8s_object in k8s_objects:
- validate_k8s_object(k8s_object, kubernetes_version)
- return k8s_objects
-
-
-def prepare_k8s_lookup_dict(k8s_objects) -> dict[tuple[str, str], dict[str,
Any]]:
- """
- Helper to create a lookup dict from k8s_objects.
- The keys of the dict are the k8s object's kind and name
- """
- k8s_obj_by_key = {
- (k8s_object["kind"], k8s_object["metadata"]["name"]): k8s_object for
k8s_object in k8s_objects
- }
- return k8s_obj_by_key
-
-
-def render_k8s_object(obj, type_to_render):
- """
- Function that renders dictionaries into k8s objects. For helm chart
testing only.
- """
- return api_client._ApiClient__deserialize_model(obj, type_to_render)
diff --git a/airflow-core/tests/unit/charts/log_groomer.py
b/airflow-core/tests/unit/charts/log_groomer.py
deleted file mode 100644
index b516acd6210..00000000000
--- a/airflow-core/tests/unit/charts/log_groomer.py
+++ /dev/null
@@ -1,345 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-import jmespath
-import pytest
-from chart_utils.helm_template_generator import render_chart
-
-
-class LogGroomerTestBase:
- obj_name: str = ""
- folder: str = ""
-
- def test_log_groomer_collector_default_enabled(self):
- if self.obj_name == "dag-processor":
- values = {"dagProcessor": {"enabled": True}}
- else:
- values = None
-
- docs = render_chart(
- values=values,
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"]
- )
-
- assert len(jmespath.search("spec.template.spec.containers", docs[0]))
== 2
- assert f"{self.obj_name}-log-groomer" in [
- c["name"] for c in
jmespath.search("spec.template.spec.containers", docs[0])
- ]
-
- def test_log_groomer_collector_can_be_disabled(self):
- if self.obj_name == "dag-processor":
- values = {
- "dagProcessor": {
- "enabled": True,
- "logGroomerSidecar": {"enabled": False},
- }
- }
- else:
- values = {f"{self.folder}": {"logGroomerSidecar": {"enabled":
False}}}
-
- docs = render_chart(
- values=values,
-
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
- )
-
- actual = jmespath.search("spec.template.spec.containers", docs[0])
-
- assert len(actual) == 1
-
- def test_log_groomer_collector_default_command_and_args(self):
- if self.obj_name == "dag-processor":
- values = {"dagProcessor": {"enabled": True}}
- else:
- values = None
-
- docs = render_chart(
- values=values,
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"]
- )
-
- assert jmespath.search("spec.template.spec.containers[1].command",
docs[0]) is None
- assert jmespath.search("spec.template.spec.containers[1].args",
docs[0]) == ["bash", "/clean-logs"]
-
- def test_log_groomer_collector_default_retention_days(self):
- if self.obj_name == "dag-processor":
- values = {"dagProcessor": {"enabled": True}}
- else:
- values = None
-
- docs = render_chart(
- values=values,
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"]
- )
-
- assert (
- jmespath.search("spec.template.spec.containers[1].env[0].name",
docs[0])
- == "AIRFLOW__LOG_RETENTION_DAYS"
- )
- assert
jmespath.search("spec.template.spec.containers[1].env[0].value", docs[0]) ==
"15"
-
- def test_log_groomer_collector_custom_env(self):
- env = [
- {"name": "APP_RELEASE_NAME", "value": "{{ .Release.Name
}}-airflow"},
- {"name": "APP__LOG_RETENTION_DAYS", "value": "5"},
- ]
-
- if self.obj_name == "dag-processor":
- values = {"dagProcessor": {"enabled": True, "logGroomerSidecar":
{"env": env}}}
- else:
- values = {
- "workers": {"logGroomerSidecar": {"env": env}},
- "scheduler": {"logGroomerSidecar": {"env": env}},
- "triggerer": {"logGroomerSidecar": {"env": env}},
- }
-
- docs = render_chart(
- values=values,
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"]
- )
-
- assert {"name": "APP_RELEASE_NAME", "value": "release-name-airflow"}
in jmespath.search(
- "spec.template.spec.containers[1].env", docs[0]
- )
- assert {"name": "APP__LOG_RETENTION_DAYS", "value": "5"} in
jmespath.search(
- "spec.template.spec.containers[1].env", docs[0]
- )
-
- @pytest.mark.parametrize("command", [None, ["custom", "command"]])
- @pytest.mark.parametrize("args", [None, ["custom", "args"]])
- def test_log_groomer_command_and_args_overrides(self, command, args):
- if self.obj_name == "dag-processor":
- values = {
- "dagProcessor": {
- "enabled": True,
- "logGroomerSidecar": {"command": command, "args": args},
- }
- }
- else:
- values = {f"{self.folder}": {"logGroomerSidecar": {"command":
command, "args": args}}}
-
- docs = render_chart(
- values=values,
-
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
- )
-
- assert command ==
jmespath.search("spec.template.spec.containers[1].command", docs[0])
- assert args ==
jmespath.search("spec.template.spec.containers[1].args", docs[0])
-
- def test_log_groomer_command_and_args_overrides_are_templated(self):
- if self.obj_name == "dag-processor":
- values = {
- "dagProcessor": {
- "enabled": True,
- "logGroomerSidecar": {
- "command": ["{{ .Release.Name }}"],
- "args": ["{{ .Release.Service }}"],
- },
- }
- }
- else:
- values = {
- f"{self.folder}": {
- "logGroomerSidecar": {
- "command": ["{{ .Release.Name }}"],
- "args": ["{{ .Release.Service }}"],
- }
- }
- }
-
- docs = render_chart(
- values=values,
-
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
- )
-
- assert jmespath.search("spec.template.spec.containers[1].command",
docs[0]) == ["release-name"]
- assert jmespath.search("spec.template.spec.containers[1].args",
docs[0]) == ["Helm"]
-
- @pytest.mark.parametrize(("retention_days", "retention_result"), [(None,
None), (30, "30")])
- def test_log_groomer_retention_days_overrides(self, retention_days,
retention_result):
- if self.obj_name == "dag-processor":
- values = {
- "dagProcessor": {"enabled": True, "logGroomerSidecar":
{"retentionDays": retention_days}}
- }
- else:
- values = {f"{self.folder}": {"logGroomerSidecar":
{"retentionDays": retention_days}}}
-
- docs = render_chart(
- values=values,
-
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
- )
-
- if retention_result:
- assert (
- jmespath.search(
-
"spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_RETENTION_DAYS'].value
| [0]",
- docs[0],
- )
- == retention_result
- )
- else:
- assert len(jmespath.search("spec.template.spec.containers[1].env",
docs[0])) == 2
-
- @pytest.mark.parametrize(("frequency_minutes", "frequency_result"),
[(None, None), (20, "20")])
- def test_log_groomer_frequency_minutes_overrides(self, frequency_minutes,
frequency_result):
- if self.obj_name == "dag-processor":
- values = {
- "dagProcessor": {
- "enabled": True,
- "logGroomerSidecar": {"frequencyMinutes":
frequency_minutes},
- }
- }
- else:
- values = {f"{self.folder}": {"logGroomerSidecar":
{"frequencyMinutes": frequency_minutes}}}
-
- docs = render_chart(
- values=values,
-
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
- )
-
- if frequency_result:
- assert (
- jmespath.search(
-
"spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES'].value
| [0]",
- docs[0],
- )
- == frequency_result
- )
- else:
- assert len(jmespath.search("spec.template.spec.containers[1].env",
docs[0])) == 2
-
- @pytest.mark.parametrize(
- ("max_size_bytes", "max_size_result"), [(None, None), (1234567890,
"1234567890")]
- )
- def test_log_groomer_max_size_bytes_overrides(self, max_size_bytes,
max_size_result):
- if self.obj_name == "dag-processor":
- values = {
- "dagProcessor": {
- "enabled": True,
- "logGroomerSidecar": {"maxSizeBytes": max_size_bytes},
- }
- }
- else:
- values = {f"{self.folder}": {"logGroomerSidecar": {"maxSizeBytes":
max_size_bytes}}}
-
- docs = render_chart(
- values=values,
-
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
- )
-
- if max_size_result:
- assert (
- jmespath.search(
-
"spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_MAX_SIZE_BYTES'].value
| [0]",
- docs[0],
- )
- == max_size_result
- )
- else:
- assert (
- jmespath.search(
-
"spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_MAX_SIZE_BYTES'].value
| [0]",
- docs[0],
- )
- is None
- )
-
- @pytest.mark.parametrize(("max_size_percent", "max_size_result"), [(None,
None), (80, "80")])
- def test_log_groomer_max_size_percent_overrides(self, max_size_percent,
max_size_result):
- if self.obj_name == "dag-processor":
- values = {
- "dagProcessor": {
- "enabled": True,
- "logGroomerSidecar": {"maxSizePercent": max_size_percent},
- }
- }
- else:
- values = {f"{self.folder}": {"logGroomerSidecar":
{"maxSizePercent": max_size_percent}}}
-
- docs = render_chart(
- values=values,
-
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
- )
-
- if max_size_result:
- assert (
- jmespath.search(
-
"spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_MAX_SIZE_PERCENT'].value
| [0]",
- docs[0],
- )
- == max_size_result
- )
- else:
- assert (
- jmespath.search(
-
"spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_MAX_SIZE_PERCENT'].value
| [0]",
- docs[0],
- )
- is None
- )
-
- def test_log_groomer_resources(self):
- if self.obj_name == "dag-processor":
- values = {
- "dagProcessor": {
- "enabled": True,
- "logGroomerSidecar": {
- "resources": {
- "requests": {"memory": "2Gi", "cpu": "1"},
- "limits": {"memory": "3Gi", "cpu": "2"},
- }
- },
- }
- }
- else:
- values = {
- f"{self.folder}": {
- "logGroomerSidecar": {
- "resources": {
- "requests": {"memory": "2Gi", "cpu": "1"},
- "limits": {"memory": "3Gi", "cpu": "2"},
- }
- }
- }
- }
-
- docs = render_chart(
- values=values,
-
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
- )
-
- assert jmespath.search("spec.template.spec.containers[1].resources",
docs[0]) == {
- "limits": {
- "cpu": "2",
- "memory": "3Gi",
- },
- "requests": {
- "cpu": "1",
- "memory": "2Gi",
- },
- }
-
- def test_log_groomer_has_airflow_home(self):
- if self.obj_name == "dag-processor":
- values = {"dagProcessor": {"enabled": True}}
- else:
- values = None
-
- docs = render_chart(
- values=values,
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"]
- )
-
- assert (
-
jmespath.search("spec.template.spec.containers[1].env[?name=='AIRFLOW_HOME'].name
| [0]", docs[0])
- == "AIRFLOW_HOME"
- )
diff --git a/dev/breeze/tests/test_pytest_args_for_test_types.py
b/dev/breeze/tests/test_pytest_args_for_test_types.py
index bb7d30f3ef1..91a2a54b328 100644
--- a/dev/breeze/tests/test_pytest_args_for_test_types.py
+++ b/dev/breeze/tests/test_pytest_args_for_test_types.py
@@ -156,7 +156,6 @@ def _find_all_integration_folders() -> list[str]:
[
"airflow-core/tests/unit/assets",
"airflow-core/tests/unit/callbacks",
- "airflow-core/tests/unit/charts",
"airflow-core/tests/unit/cluster_policies",
"airflow-core/tests/unit/config_templates",
"airflow-core/tests/unit/dag_processing",
