(airflow) branch main updated: Fix check served logs logic (#41272)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new fdda4478e5 Fix check served logs logic (#41272)
fdda4478e5 is described below
commit fdda4478e5829c08aa5059f8ad86f92922313d19
Author: Daniel Standish <15932138+dstand...@users.noreply.github.com>
AuthorDate: Mon Aug 5 18:29:50 2024 -0700
Fix check served logs logic (#41272)
---
airflow/utils/log/file_task_handler.py | 16 ++--
tests/utils/test_log_handlers.py | 46 ++
2 files changed, 14 insertions(+), 48 deletions(-)
diff --git a/airflow/utils/log/file_task_handler.py
b/airflow/utils/log/file_task_handler.py
index 2ae15b454a..e99ffae0c9 100644
--- a/airflow/utils/log/file_task_handler.py
+++ b/airflow/utils/log/file_task_handler.py
@@ -381,28 +381,23 @@ class FileTaskHandler(logging.Handler):
executor_messages: list[str] = []
executor_logs: list[str] = []
served_logs: list[str] = []
-is_in_running_or_deferred = ti.state in (
-TaskInstanceState.RUNNING,
-TaskInstanceState.DEFERRED,
-)
-is_up_for_retry = ti.state == TaskInstanceState.UP_FOR_RETRY
with suppress(NotImplementedError):
remote_messages, remote_logs = self._read_remote_logs(ti,
try_number, metadata)
messages_list.extend(remote_messages)
+has_k8s_exec_pod = False
if ti.state == TaskInstanceState.RUNNING:
response = self._executor_get_task_log(ti, try_number)
if response:
executor_messages, executor_logs = response
if executor_messages:
messages_list.extend(executor_messages)
+has_k8s_exec_pod = True
if not (remote_logs and ti.state not in State.unfinished):
# when finished, if we have remote logs, no need to check local
worker_log_full_path = Path(self.local_base, worker_log_rel_path)
local_messages, local_logs =
self._read_from_local(worker_log_full_path)
messages_list.extend(local_messages)
-if (is_in_running_or_deferred or is_up_for_retry) and not
executor_messages and not remote_logs:
-# While task instance is still running and we don't have either
executor nor remote logs, look for served logs
-# This is for cases when users have not setup remote logging nor
shared drive for logs
+if ti.state in (TaskInstanceState.RUNNING, TaskInstanceState.DEFERRED)
and not has_k8s_exec_pod:
served_messages, served_logs = self._read_from_logs_server(ti,
worker_log_rel_path)
messages_list.extend(served_messages)
elif ti.state not in State.unfinished and not (local_logs or
remote_logs):
@@ -422,7 +417,10 @@ class FileTaskHandler(logging.Handler):
)
log_pos = len(logs)
messages = "".join([f"*** {x}\n" for x in messages_list])
-end_of_log = ti.try_number != try_number or not
is_in_running_or_deferred
+end_of_log = ti.try_number != try_number or ti.state not in (
+TaskInstanceState.RUNNING,
+TaskInstanceState.DEFERRED,
+)
if metadata and "log_pos" in metadata:
previous_chars = metadata["log_pos"]
logs = logs[previous_chars:] # Cut off previously passed log test
as new tail
diff --git a/tests/utils/test_log_handlers.py b/tests/utils/test_log_handlers.py
index a9d9bfbabe..8d0b96435b 100644
--- a/tests/utils/test_log_handlers.py
+++ b/tests/utils/test_log_handlers.py
@@ -316,58 +316,26 @@ class TestFileTaskLogHandler:
else:
mock_k8s_get_task_log.assert_not_called()
-# We are not testing TaskInstanceState.DEFERRED in this test because with
the current testing setup,
-# as it creates an inconsistent tests that succeeds in local but fails in
CI. See https://github.com/apache/airflow/pull/39496#issuecomment-2149692239
-# TODO: Fix the test setup so it is possible to test
TaskInstanceState.DEFERRED as well.
-@pytest.mark.parametrize("state", [TaskInstanceState.RUNNING,
TaskInstanceState.UP_FOR_RETRY])
-def test__read_for_celery_executor_fallbacks_to_worker(self, state,
create_task_instance):
+def test__read_for_celery_executor_fallbacks_to_worker(self,
create_task_instance):
"""Test for executors which do not have `get_task_log` method, it
fallbacks to reading
-log from worker if and only if remote logs aren't found"""
+log from worker"""
executor_name = "CeleryExecutor"
-# Reading logs from worker should occur when the task is either
running, deferred, or up for retry.
+
ti = cre
(airflow) branch main updated (4d27069d9c -> f5836bcb7d)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 4d27069d9c Adjust default extractor's on_failure detection for airflow 2.10 fix (#41094) add f5836bcb7d Revert "Adjust gantt width based on task history dates (#41192)" (#41216) No new revisions were added by this update. Summary of changes: airflow/www/static/js/dag/details/gantt/Row.tsx | 17 +- airflow/www/static/js/dag/details/gantt/index.tsx | 67 --- 2 files changed, 25 insertions(+), 59 deletions(-)
(airflow) branch main updated: Fix ApiException handling when adopting completed pods (#41109)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new b014077fe3 Fix ApiException handling when adopting completed pods
(#41109)
b014077fe3 is described below
commit b014077fe31853b857a081f18a56552abdae3427
Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com>
AuthorDate: Wed Jul 31 06:16:21 2024 -0600
Fix ApiException handling when adopting completed pods (#41109)
When trying to complete adopted pods, if we encounter an ApiException,
we should assume we were unable to adopt the pod meaning we shouldn't
add that to our `running` set. If we do, `running` can fill up over time
with tasks the executor isn't actually watching.
---
.../kubernetes/executors/kubernetes_executor.py| 2 ++
.../executors/test_kubernetes_executor.py | 35 ++
2 files changed, 37 insertions(+)
diff --git a/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py
b/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py
index 11065ede7f..612f64c8f3 100644
--- a/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py
+++ b/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py
@@ -694,6 +694,8 @@ class KubernetesExecutor(BaseExecutor):
)
except ApiException as e:
self.log.info("Failed to adopt pod %s. Reason: %s",
pod.metadata.name, e)
+continue
+
ti_id = annotations_to_key(pod.metadata.annotations)
self.running.add(ti_id)
diff --git
a/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py
b/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py
index 1ed9399b59..1ae36356b2 100644
--- a/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py
+++ b/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py
@@ -1127,6 +1127,41 @@ class TestKubernetesExecutor:
)
assert executor.running == expected_running_ti_keys
+
@mock.patch("airflow.providers.cncf.kubernetes.executors.kubernetes_executor.DynamicClient")
+
@mock.patch("airflow.providers.cncf.kubernetes.kube_client.get_kube_client")
+def test_adopt_completed_pods_api_exception(self, mock_kube_client,
mock_kube_dynamic_client):
+"""We should gracefully handle exceptions when adopting completed pods
from other schedulers"""
+executor = self.kubernetes_executor
+executor.scheduler_job_id = "modified"
+executor.kube_client = mock_kube_client
+executor.kube_config.kube_namespace = "somens"
+pod_names = ["one", "two"]
+
+def get_annotations(pod_name):
+return {
+"dag_id": "dag",
+"run_id": "run_id",
+"task_id": pod_name,
+"try_number": "1",
+}
+
+mock_kube_dynamic_client.return_value.get.return_value.items = [
+k8s.V1Pod(
+metadata=k8s.V1ObjectMeta(
+name=pod_name,
+labels={"airflow-worker": pod_name},
+annotations=get_annotations(pod_name),
+namespace="somens",
+)
+)
+for pod_name in pod_names
+]
+
+mock_kube_client.patch_namespaced_pod.side_effect =
ApiException(status=400)
+executor._adopt_completed_pods(mock_kube_client)
+assert len(pod_names) ==
mock_kube_client.patch_namespaced_pod.call_count
+assert executor.running == set()
+
@mock.patch("airflow.providers.cncf.kubernetes.kube_client.get_kube_client")
def test_not_adopt_unassigned_task(self, mock_kube_client):
"""
(airflow) branch main updated: Fix typo in contributing doc about skipping unit_tests (#41108)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new 8a1b84dc54 Fix typo in contributing doc about skipping unit_tests (#41108) 8a1b84dc54 is described below commit 8a1b84dc54b403985e64a9cdb88e9731e83c8b39 Author: raphaelauv AuthorDate: Tue Jul 30 05:38:41 2024 +0200 Fix typo in contributing doc about skipping unit_tests (#41108) --- contributing-docs/testing/unit_tests.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contributing-docs/testing/unit_tests.rst b/contributing-docs/testing/unit_tests.rst index fd8ace55e0..c83f391e52 100644 --- a/contributing-docs/testing/unit_tests.rst +++ b/contributing-docs/testing/unit_tests.rst @@ -1187,7 +1187,7 @@ are not part of the public API. We deal with it in one of the following ways: from tests.test_utils.compat import AIRFLOW_V_2_7_PLUS - @pytest.mark.skip(not AIRFLOW_V_2_7_PLUS, reason="The tests should be skipped for Airflow < 2.7") + @pytest.mark.skipif(not AIRFLOW_V_2_7_PLUS, reason="The tests should be skipped for Airflow < 2.7") def some_test_that_only_works_for_airflow_2_7_plus(): pass
(airflow) branch main updated (95b5a0a2ac -> f56f70088e)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 95b5a0a2ac Refactor dataproc system tests (#40720) add f56f70088e Helm chart 1.15.0 has been released (#40998) No new revisions were added by this update. Summary of changes: .github/ISSUE_TEMPLATE/airflow_helmchart_bug_report.yml | 3 ++- chart/Chart.yaml| 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-)
(airflow-site) branch helm-chart-1.15.0 deleted (was e1e57adc12)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch helm-chart-1.15.0 in repository https://gitbox.apache.org/repos/asf/airflow-site.git was e1e57adc12 Announce Helm Chart 1.15.0 The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
(airflow-site) branch main updated: Announce Helm Chart 1.15.0 (#1048)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow-site.git The following commit(s) were added to refs/heads/main by this push: new 19ddba77f0 Announce Helm Chart 1.15.0 (#1048) 19ddba77f0 is described below commit 19ddba77f096db849b1dcb42672f5e2100966fbd Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com> AuthorDate: Wed Jul 24 08:41:19 2024 -0600 Announce Helm Chart 1.15.0 (#1048) --- landing-pages/site/content/en/announcements/_index.md | 9 + 1 file changed, 9 insertions(+) diff --git a/landing-pages/site/content/en/announcements/_index.md b/landing-pages/site/content/en/announcements/_index.md index caaf6106ea..3ec874f449 100644 --- a/landing-pages/site/content/en/announcements/_index.md +++ b/landing-pages/site/content/en/announcements/_index.md @@ -13,6 +13,15 @@ menu: **Note:** Follow [@ApacheAirflow](https://twitter.com/ApacheAirflow) on Twitter for the latest news and announcements! +# July 24, 2024 + +We've just released Apache **Airflow Helm chart 1.15.0**. + + ArtifactHub: https://artifacthub.io/packages/helm/apache-airflow/airflow \ + Docs: https://airflow.apache.org/docs/helm-chart/1.15.0/ \ +️ Release Notes: https://airflow.apache.org/docs/helm-chart/1.15.0/release_notes.html \ +襁 Sources: https://airflow.apache.org/docs/helm-chart/1.15.0/installing-helm-chart-from-sources.html + # July 16, 2024 We’ve just released Apache **Airflow 2.9.3**.
(airflow-site) branch helm-chart-1.15.0 created (now e1e57adc12)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch helm-chart-1.15.0 in repository https://gitbox.apache.org/repos/asf/airflow-site.git at e1e57adc12 Announce Helm Chart 1.15.0 This branch includes the following new commits: new e1e57adc12 Announce Helm Chart 1.15.0 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(airflow-site) 01/01: Announce Helm Chart 1.15.0
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch helm-chart-1.15.0 in repository https://gitbox.apache.org/repos/asf/airflow-site.git commit e1e57adc12e1665dd11e53406be23784039798cb Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com> AuthorDate: Wed Jul 24 08:15:28 2024 -0600 Announce Helm Chart 1.15.0 --- landing-pages/site/content/en/announcements/_index.md | 9 + 1 file changed, 9 insertions(+) diff --git a/landing-pages/site/content/en/announcements/_index.md b/landing-pages/site/content/en/announcements/_index.md index caaf6106ea..3ec874f449 100644 --- a/landing-pages/site/content/en/announcements/_index.md +++ b/landing-pages/site/content/en/announcements/_index.md @@ -13,6 +13,15 @@ menu: **Note:** Follow [@ApacheAirflow](https://twitter.com/ApacheAirflow) on Twitter for the latest news and announcements! +# July 24, 2024 + +We've just released Apache **Airflow Helm chart 1.15.0**. + + ArtifactHub: https://artifacthub.io/packages/helm/apache-airflow/airflow \ + Docs: https://airflow.apache.org/docs/helm-chart/1.15.0/ \ +️ Release Notes: https://airflow.apache.org/docs/helm-chart/1.15.0/release_notes.html \ +襁 Sources: https://airflow.apache.org/docs/helm-chart/1.15.0/installing-helm-chart-from-sources.html + # July 16, 2024 We’ve just released Apache **Airflow 2.9.3**.
svn commit: r70517 - /release/airflow/helm-chart/1.13.1/
Author: jedcunningham Date: Wed Jul 24 14:14:06 2024 New Revision: 70517 Log: Remove old Helm Chart release: 1.13.1 Removed: release/airflow/helm-chart/1.13.1/
(airflow) annotated tag helm-chart/1.15.0 updated (ff7463b162 -> f0d710e311)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to annotated tag helm-chart/1.15.0 in repository https://gitbox.apache.org/repos/asf/airflow.git *** WARNING: tag helm-chart/1.15.0 was modified! *** from ff7463b162 (commit) to f0d710e311 (tag) tagging ff7463b1624d91fb6478ec367bbf84b88fb9b83d (commit) replaces providers-amazon/8.26.0 by Jed Cunningham on Wed Jul 24 08:32:15 2024 -0400 - Log - Apache Airflow Helm Chart 1.15.0 -BEGIN PGP SIGNATURE- iQIzBAABCAAdFiEE4aHphPVbjygL2cuiC7cWOJKi5I4FAmag9E8ACgkQC7cWOJKi 5I6G/xAAhzO1powXii7v3fuoFluM7kVPQ6MCMiwfSLQVEPXjwz2PgapNGPtuuU7v nbPszJ4MxtZYmUrZ7iUkPZ5kpytrKS1lk0ZLC8j2oxUIBOjvJqeiVTbsj/wLsf2D 8nfpRYHAqvkG4CbqQYokk7+UcDOqYprLTO2wybh8ZGdZoJrdR/lWOUg+sdtKb5xh vcOA73bnFTEexByJuIxMst8GR2e7O82Pjq1T2600EBXNWA9laEoSvNazrpc4zTKb CIpRajeMUy0uE+uQRH/BvIafyoNry3t5jThkcnOZCnKY5yBQdxzZIw26RpatuUc+ 7HK0v/Q271ozQUVgMd4AcPIWcabKzO4Jy/aQUr58gmVCaDAocsVULyYknEYugCUl TyZX3Ih1LXW6EiDvl0xPMVp8Fj0TxzBHm4NCpOZNdqho0wwBrDQGKbAYPsHnFIRG YCPtMWZLZTbL+tzmeBaN4cQ7AI7eba32ct7XElvjyS7nTdg0Dr5wf247uddQZnvJ j2FMcN3LqCtbXDoQvOiUSkPf/SpAWQhTI69E/c6yx2k1YgmvYbivrxAhAOsQMBUk vmS4VLcpQERqGFuH3RDb6KUXG3hKtZEhr/Umux5oC4i12Xwwn2ozl1D3g7HeSb4K 2Nr2A911xPeow9WFiRkIMcC3NSYaBAwRHbOuRq+FrcWJZy/S98Y= =fDvZ -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
svn commit: r70515 - /release/airflow/helm-chart/1.15.0/
Author: jedcunningham Date: Wed Jul 24 12:31:00 2024 New Revision: 70515 Log: Release Airflow Helm Chart Check 1.15.0 from 1.15.0rc1 Added: release/airflow/helm-chart/1.15.0/ release/airflow/helm-chart/1.15.0/airflow-1.15.0.tgz - copied unchanged from r70514, dev/airflow/helm-chart/1.15.0rc1/airflow-1.15.0.tgz release/airflow/helm-chart/1.15.0/airflow-1.15.0.tgz.asc - copied unchanged from r70514, dev/airflow/helm-chart/1.15.0rc1/airflow-1.15.0.tgz.asc release/airflow/helm-chart/1.15.0/airflow-1.15.0.tgz.prov - copied unchanged from r70514, dev/airflow/helm-chart/1.15.0rc1/airflow-1.15.0.tgz.prov release/airflow/helm-chart/1.15.0/airflow-1.15.0.tgz.sha512 - copied unchanged from r70514, dev/airflow/helm-chart/1.15.0rc1/airflow-1.15.0.tgz.sha512 release/airflow/helm-chart/1.15.0/airflow-chart-1.15.0-source.tar.gz - copied unchanged from r70514, dev/airflow/helm-chart/1.15.0rc1/airflow-chart-1.15.0-source.tar.gz release/airflow/helm-chart/1.15.0/airflow-chart-1.15.0-source.tar.gz.asc - copied unchanged from r70514, dev/airflow/helm-chart/1.15.0rc1/airflow-chart-1.15.0-source.tar.gz.asc release/airflow/helm-chart/1.15.0/airflow-chart-1.15.0-source.tar.gz.sha512 - copied unchanged from r70514, dev/airflow/helm-chart/1.15.0rc1/airflow-chart-1.15.0-source.tar.gz.sha512
svn commit: r70445 - /dev/airflow/helm-chart/1.14.0rc1/
Author: jedcunningham Date: Sun Jul 21 03:07:26 2024 New Revision: 70445 Log: Remove old Helm Chart release: 1.14.0rc1 Removed: dev/airflow/helm-chart/1.14.0rc1/
svn commit: r70444 [2/2] - /dev/airflow/helm-chart/1.15.0rc1/
Added: dev/airflow/helm-chart/1.15.0rc1/index.yaml == --- dev/airflow/helm-chart/1.15.0rc1/index.yaml (added) +++ dev/airflow/helm-chart/1.15.0rc1/index.yaml Sun Jul 21 03:06:47 2024 @@ -0,0 +1,2246 @@ +apiVersion: v1 +entries: + airflow: + - annotations: + artifacthub.io/changes: | +- description: Add git-sync container lifecycle hooks + kind: added + links: + - name: '#40369' +url: https://github.com/apache/airflow/pull/40369 +- description: Add init containers for jobs + kind: added + links: + - name: '#40454' +url: https://github.com/apache/airflow/pull/40454 +- description: Add persistent volume claim retention policy + kind: added + links: + - name: '#40271' +url: https://github.com/apache/airflow/pull/40271 +- description: Add annotations for Redis StatefulSet + kind: added + links: + - name: '#40281' +url: https://github.com/apache/airflow/pull/40281 +- description: Add ``dags.gitSync.sshKey``, which allows the git-sync private key +to be configured in the values file directly + kind: added + links: + - name: '#39936' +url: https://github.com/apache/airflow/pull/39936 +- description: Link in ``UIAlert`` to production guide when a dynamic webserver secret +is used now opens in a new tab + kind: changed + links: + - name: '#40635' +url: https://github.com/apache/airflow/pull/40635 +- description: Support disabling helm hooks on ``extraConfigMaps`` and ``extraSecrets`` + kind: changed + links: + - name: '#40294' +url: https://github.com/apache/airflow/pull/40294 +- description: Add git-sync ssh secret to DAG processor + kind: fixed + links: + - name: '#40691' +url: https://github.com/apache/airflow/pull/40691 +- description: Fix duplicated ``safeToEvict`` annotations + kind: fixed + links: + - name: '#40554' +url: https://github.com/apache/airflow/pull/40554 +- description: Add missing ``triggerer.keda.usePgbouncer`` to values.yaml + kind: fixed + links: + - name: '#40614' +url: https://github.com/apache/airflow/pull/40614 +- description: Trim leading ``//`` character using mysql backend + kind: fixed + links: + - name: '#40401' +url: https://github.com/apache/airflow/pull/40401 +- description: 'Docs: Updating chart download link to use the Apache download CDN' + kind: changed + links: + - name: '#40618' +url: https://github.com/apache/airflow/pull/40618 +- description: 'Misc: Update PgBouncer exporter image to ``airflow-pgbouncer-exporter-2024.06.18-0.17.0``' + kind: changed + links: + - name: '#40318' +url: https://github.com/apache/airflow/pull/40318 +- description: 'Misc: Default airflow version to 2.9.3' + kind: changed + links: + - name: '#40816' +url: https://github.com/apache/airflow/pull/40816 +- description: 'Misc: Fix ``startupProbe`` timing comment' + kind: changed + links: + - name: '#40412' +url: https://github.com/apache/airflow/pull/40412 + artifacthub.io/links: | +- name: Documentation + url: https://airflow.apache.org/docs/helm-chart/1.15.0/ + artifacthub.io/screenshots: | +- title: DAGs View + url: https://airflow.apache.org/docs/apache-airflow/2.9.3/_images/dags.png +- title: Datasets View + url: https://airflow.apache.org/docs/apache-airflow/2.9.3/_images/datasets.png +- title: Grid View + url: https://airflow.apache.org/docs/apache-airflow/2.9.3/_images/grid.png +- title: Graph View + url: https://airflow.apache.org/docs/apache-airflow/2.9.3/_images/graph.png +- title: Calendar View + url: https://airflow.apache.org/docs/apache-airflow/2.9.3/_images/calendar.png +- title: Variable View + url: https://airflow.apache.org/docs/apache-airflow/2.9.3/_images/variable_hidden.png +- title: Gantt Chart + url: https://airflow.apache.org/docs/apache-airflow/2.9.3/_images/gantt.png +- title: Task Duration + url: https://airflow.apache.org/docs/apache-airflow/2.9.3/_images/duration.png +- title: Code View + url: https://airflow.apache.org/docs/apache-airflow/2.9.3/_images/code.png +apiVersion: v2 +appVersion: 2.9.3 +created: "2024-07-20T23:05:35.701277-04:00" +dependencies: +-
(airflow) annotated tag helm-chart/1.15.0rc1 updated (ff7463b162 -> 06b6d399fa)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to annotated tag helm-chart/1.15.0rc1 in repository https://gitbox.apache.org/repos/asf/airflow.git *** WARNING: tag helm-chart/1.15.0rc1 was modified! *** from ff7463b162 (commit) to 06b6d399fa (tag) tagging ff7463b1624d91fb6478ec367bbf84b88fb9b83d (commit) replaces providers-amazon/8.26.0 by Jed Cunningham on Sat Jul 20 21:59:42 2024 -0500 - Log - Apache Airflow Helm Chart 1.15.0rc1 -BEGIN PGP SIGNATURE- iQIzBAABCAAdFiEE4aHphPVbjygL2cuiC7cWOJKi5I4FAmaceaAACgkQC7cWOJKi 5I53hg//aqDHup76eyTEb2Xc2Aeofk2X/EGYQFGfYH8cF6wVIG8Ov9t5BYNVgDG+ 0X4jHSTRIHLp7K8NYUopg04U+IPJp2IYSaUZPxXM11wFz/MiRSHxSDEShfiiRZSQ 6x4UMQcflu1BKLl+sGLEBxlEDBzm6wu1Wcq1StM6RkJcLkfIw10GY59bIyi5SbQb jEF4XkgkdA+u/jKkeGJK4QfvaKE5agc9pK2Aqwi4LUYIy0GTYguZhXIH0m4B7964 Jb+/FpCBCFpn3Y40Cix3EcywGHZ/P7Mkpy2NLhxqepo3Sltnxr5AQ/9bBGlBbtlh 1+GV83ILzGA9GIZUYHvgUHBc4uEQ4xQm9jpMvCmZOAOs+SNfNrJ2q4FyuD4VN4MP YLWyRnwd7WI9K900NymTpITr+zrl4hwA4p2+Hfim80gU/5BZNrT6l/fyPja7QoDZ y83wLSJJ2pJL6GMVTSl6FMQrZNn7wZG412TrtzMO0EWoFQW06UCsqia1vQL1SeH7 mpdu2XK29bJ7fa2Jn8VDBtk9BiT4a6hqVEzwlkmoiNNeJNNOoFBijVZl9UNi1ZSY KM6fUowSdMvuTLD0NgmXcQZDM4usoE5p9VaMYGu2R+Tq+jEBMvNKulVhq+tKsDKp XjM0a5RBW+Wiha8oytD3f66VTUeIKFYljE72v9pK8pHsuO0Bs/8= =dfQm -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
svn commit: r70444 [1/2] - /dev/airflow/helm-chart/1.15.0rc1/
Author: jedcunningham Date: Sun Jul 21 03:06:47 2024 New Revision: 70444 Log: Add artifacts for Helm Chart 1.15.0rc1 Added: dev/airflow/helm-chart/1.15.0rc1/ dev/airflow/helm-chart/1.15.0rc1/airflow-1.15.0.tgz (with props) dev/airflow/helm-chart/1.15.0rc1/airflow-1.15.0.tgz.asc dev/airflow/helm-chart/1.15.0rc1/airflow-1.15.0.tgz.prov dev/airflow/helm-chart/1.15.0rc1/airflow-1.15.0.tgz.sha512 dev/airflow/helm-chart/1.15.0rc1/airflow-chart-1.15.0-source.tar.gz (with props) dev/airflow/helm-chart/1.15.0rc1/airflow-chart-1.15.0-source.tar.gz.asc dev/airflow/helm-chart/1.15.0rc1/airflow-chart-1.15.0-source.tar.gz.sha512 dev/airflow/helm-chart/1.15.0rc1/index.yaml Added: dev/airflow/helm-chart/1.15.0rc1/airflow-1.15.0.tgz == Binary file - no diff available. Propchange: dev/airflow/helm-chart/1.15.0rc1/airflow-1.15.0.tgz -- svn:mime-type = application/octet-stream Added: dev/airflow/helm-chart/1.15.0rc1/airflow-1.15.0.tgz.asc == --- dev/airflow/helm-chart/1.15.0rc1/airflow-1.15.0.tgz.asc (added) +++ dev/airflow/helm-chart/1.15.0rc1/airflow-1.15.0.tgz.asc Sun Jul 21 03:06:47 2024 @@ -0,0 +1,17 @@ +-BEGIN PGP SIGNATURE- + +iQJNBAABCAA3FiEE4aHphPVbjygL2cuiC7cWOJKi5I4FAmacem0ZHGplZGN1bm5p +bmdoYW1AYXBhY2hlLm9yZwAKCRALtxY4kqLkjpJ9D/wPwuxfK7uIdBaILXol6jeo +4SbTpxeQv0xPWUGI8eGfBC89tlcuNmlBA0H156TlE+fqk09j5KR08lr6p7495jrI +XQa+F0PzHeoQ4lga3a0GzJdwBH2vzeEQZCI4wXXI78UIPCqFRDhs2+LlWwK+StM8 ++STq13+tHBLh5U1BQ/dZIeZuUhrobLjbjux/K/NnzG5+p14X4Okxv/gm2gNrAhBf +frpbGTjfXAgwOY2+OceQh+qd7TFtOc4HbJU9qG+x5S4MmugJFEEEXFcKX8ymgCbE +R8F/XJTJaZrAsJLxJHjrco3NkUg+mImt1rOxgrOjFlpM6FCL1O3Pudmzjz9rD6pB +UYWuBOI7buemYaokEB9cgAqN/zo6u6jQ5TyjKzYQcKvbZTvzxVJezsJ7LdswRZKe ++8wuVnHAxs7+O6dcD6+S524bErkQAtYv2CQeAjbfh6PGLHsNR2AGdfKdLaeWsUFO +U5Zr6CM9m6hMd4i2RsSdYREpHeAZEtFJRoXzlcix5N7b0rT3EAcpJbJbEcY+nCqw +9A0QJMNYpBfTL9cXMro9O94ceKzmtqmliaaiPELj6NVHdrHhRYGRATZ4mIBn6tSH +m0imXAf6fDpRmzSeRmRb9e6relcWqSL2gnK5ruqIyOf0eK8tyl2gP5O/uTmLTj87 +VhrkrNmLQP3DcJysV1hrKw== +=PdHT +-END PGP SIGNATURE- Added: dev/airflow/helm-chart/1.15.0rc1/airflow-1.15.0.tgz.prov == --- dev/airflow/helm-chart/1.15.0rc1/airflow-1.15.0.tgz.prov (added) +++ dev/airflow/helm-chart/1.15.0rc1/airflow-1.15.0.tgz.prov Sun Jul 21 03:06:47 2024 @@ -0,0 +1,149 @@ +-BEGIN PGP SIGNED MESSAGE- +Hash: SHA256 + +annotations: + artifacthub.io/changes: | +- description: Add git-sync container lifecycle hooks + kind: added + links: + - name: '#40369' +url: https://github.com/apache/airflow/pull/40369 +- description: Add init containers for jobs + kind: added + links: + - name: '#40454' +url: https://github.com/apache/airflow/pull/40454 +- description: Add persistent volume claim retention policy + kind: added + links: + - name: '#40271' +url: https://github.com/apache/airflow/pull/40271 +- description: Add annotations for Redis StatefulSet + kind: added + links: + - name: '#40281' +url: https://github.com/apache/airflow/pull/40281 +- description: Add ``dags.gitSync.sshKey``, which allows the git-sync private key +to be configured in the values file directly + kind: added + links: + - name: '#39936' +url: https://github.com/apache/airflow/pull/39936 +- description: Link in ``UIAlert`` to production guide when a dynamic webserver secret +is used now opens in a new tab + kind: changed + links: + - name: '#40635' +url: https://github.com/apache/airflow/pull/40635 +- description: Support disabling helm hooks on ``extraConfigMaps`` and ``extraSecrets`` + kind: changed + links: + - name: '#40294' +url: https://github.com/apache/airflow/pull/40294 +- description: Add git-sync ssh secret to DAG processor + kind: fixed + links: + - name: '#40691' +url: https://github.com/apache/airflow/pull/40691 +- description: Fix duplicated ``safeToEvict`` annotations + kind: fixed + links: + - name: '#40554' +url: https://github.com/apache/airflow/pull/40554 +- description: Add missing ``triggerer.keda.usePgbouncer`` to values.yaml + kind: fixed + links: + - name: '#40614' +url: https://github.com/apache/airflow/pull/40614 +- description: Trim leading ``//`` character using mysql backend + kind: fixed + links: + - name: '#40401' +url: https://github.com/apache/airflow/pull/40401 +- description: 'Docs: Updating chart download link to use the Apache download CDN' + kind: changed + links: + - name: '#40618' +url
(airflow) branch main updated: Chart: Update release notes for 1.15.0 (#40903)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new ff7463b162 Chart: Update release notes for 1.15.0 (#40903) ff7463b162 is described below commit ff7463b1624d91fb6478ec367bbf84b88fb9b83d Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com> AuthorDate: Sat Jul 20 19:22:21 2024 -0600 Chart: Update release notes for 1.15.0 (#40903) --- chart/RELEASE_NOTES.rst | 1 + chart/reproducible_build.yaml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/chart/RELEASE_NOTES.rst b/chart/RELEASE_NOTES.rst index 908c2c04a1..73695c69f1 100644 --- a/chart/RELEASE_NOTES.rst +++ b/chart/RELEASE_NOTES.rst @@ -47,6 +47,7 @@ New Features - Add persistent volume claim retention policy (#40271) - Add annotations for Redis StatefulSet (#40281) - Add ``dags.gitSync.sshKey``, which allows the git-sync private key to be configured in the values file directly (#39936) +- Add ``extraEnvFrom`` to git-sync containers (#39031) Improvements diff --git a/chart/reproducible_build.yaml b/chart/reproducible_build.yaml index 9fd4dc1caf..8225fafc1f 100644 --- a/chart/reproducible_build.yaml +++ b/chart/reproducible_build.yaml @@ -1,2 +1,2 @@ -release-notes-hash: 6b7332be7822cb343877b9d8a5f4fabd -source-date-epoch: 1721448456 +release-notes-hash: dfd00a8e0d62f80ead188af7cfb6d8b4 +source-date-epoch: 1721481612
(airflow) branch main updated (1819c858ce -> c2a54efefb)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 1819c858ce Chart: Helm chart 1.15.0 release notes (#40898) add c2a54efefb Chart: Add extraEnvFrom to git-sync containers (#39031) No new revisions were added by this update. Summary of changes: chart/templates/_helpers.yaml| 9 + chart/values.schema.json | 12 chart/values.yaml| 10 ++ helm_tests/airflow_aux/test_pod_template_file.py | 2 ++ helm_tests/other/test_git_sync_scheduler.py | 4 5 files changed, 37 insertions(+)
(airflow) branch main updated (0f4884c3ca -> 1819c858ce)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 0f4884c3ca [AIP-49] OpenTelemetry Traces for Apache Airflow Part 2 (#40802) add 1819c858ce Chart: Helm chart 1.15.0 release notes (#40898) No new revisions were added by this update. Summary of changes: chart/Chart.yaml | 132 -- chart/RELEASE_NOTES.rst | 50 +++ chart/newsfragments/40318.misc.rst| 1 - chart/newsfragments/40816.significant.rst | 3 - chart/reproducible_build.yaml | 4 +- 5 files changed, 106 insertions(+), 84 deletions(-) delete mode 100644 chart/newsfragments/40318.misc.rst delete mode 100644 chart/newsfragments/40816.significant.rst
(airflow) branch main updated (06b19eb8b0 -> 83fe28be6d)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 06b19eb8b0 Fix `RedshiftCreateClusterOperator` to always specify `PubliclyAccessible` (#40872) add 83fe28be6d Add Shahar as a committer (#40876) No new revisions were added by this update. Summary of changes: .asf.yaml | 1 - dev/breeze/src/airflow_breeze/global_constants.py | 1 + docs/apache-airflow/project.rst | 1 + 3 files changed, 2 insertions(+), 1 deletion(-)
(airflow) branch main updated: fix: duplicated keys in annotations caused by airfowPodAnnotations value being overwritten by safeToEvict value of worker (#40554)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 5323471166 fix: duplicated keys in annotations caused by
airfowPodAnnotations value being overwritten by safeToEvict value of worker
(#40554)
5323471166 is described below
commit 5323471166ec9b2dfaf7ec0940a5a55b50d49e73
Author: Reza J. Bavaghoush
AuthorDate: Tue Jul 9 18:19:24 2024 +0200
fix: duplicated keys in annotations caused by airfowPodAnnotations value
being overwritten by safeToEvict value of worker (#40554)
---
chart/files/pod-template-file.kubernetes-helm-yaml | 2 +-
helm_tests/airflow_aux/test_pod_template_file.py | 21 +
2 files changed, 22 insertions(+), 1 deletion(-)
diff --git a/chart/files/pod-template-file.kubernetes-helm-yaml
b/chart/files/pod-template-file.kubernetes-helm-yaml
index 4219225616..259f6a71b8 100644
--- a/chart/files/pod-template-file.kubernetes-helm-yaml
+++ b/chart/files/pod-template-file.kubernetes-helm-yaml
@@ -27,7 +27,7 @@
{{- $containerSecurityContext := include "containerSecurityContext" (list .
.Values.workers) }}
{{- $containerLifecycleHooks := or .Values.workers.containerLifecycleHooks
.Values.containerLifecycleHooks }}
{{- $safeToEvict := dict "cluster-autoscaler.kubernetes.io/safe-to-evict"
(.Values.workers.safeToEvict | toString) }}
-{{- $podAnnotations := mergeOverwrite .Values.airflowPodAnnotations
$safeToEvict .Values.workers.podAnnotations }}
+{{- $podAnnotations := mergeOverwrite (deepCopy .Values.airflowPodAnnotations)
$safeToEvict .Values.workers.podAnnotations }}
apiVersion: v1
kind: Pod
metadata:
diff --git a/helm_tests/airflow_aux/test_pod_template_file.py
b/helm_tests/airflow_aux/test_pod_template_file.py
index 1efccbf410..43a8d19423 100644
--- a/helm_tests/airflow_aux/test_pod_template_file.py
+++ b/helm_tests/airflow_aux/test_pod_template_file.py
@@ -660,6 +660,27 @@ class TestPodTemplateFile:
"cluster-autoscaler.kubernetes.io/safe-to-evict": "true" if
safe_to_evict else "false"
}
+def test_safe_to_evict_annotation_other_services(self):
+"""Workers' safeToEvict value should not overwrite safeToEvict value
of other services."""
+docs = render_chart(
+values={
+"workers": {"safeToEvict": False},
+"scheduler": {"safeToEvict": True},
+"triggerer": {"safeToEvict": True},
+"executor": "KubernetesExecutor",
+"dagProcessor": {"enabled": True, "safeToEvict": True},
+},
+show_only=[
+"templates/dag-processor/dag-processor-deployment.yaml",
+"templates/triggerer/triggerer-deployment.yaml",
+"templates/scheduler/scheduler-deployment.yaml",
+],
+chart_dir=self.temp_chart_dir,
+)
+for doc in docs:
+annotations =
jmespath.search("spec.template.metadata.annotations", doc)
+assert
annotations.get("cluster-autoscaler.kubernetes.io/safe-to-evict") == "true"
+
def test_workers_pod_annotations(self):
docs = render_chart(
values={"workers": {"podAnnotations": {"my_annotation":
"annotated!"}}},
(airflow) branch RNHTTR-patch-1 deleted (was a9ef92ea27)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch RNHTTR-patch-1 in repository https://gitbox.apache.org/repos/asf/airflow.git was a9ef92ea27 Remove RNHTTR from collaberators in .asf.yaml The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
(airflow) branch main updated (a62bd83188 -> ad988b2a5d)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from a62bd83188 Enable enforcing pydocstyle rule D213 in ruff. (#40448) add ad988b2a5d Fix some docstrings for the D213 rule (#40464) No new revisions were added by this update. Summary of changes: airflow/serialization/serialized_objects.py | 6 -- airflow/timetables/base.py | 6 -- 2 files changed, 8 insertions(+), 4 deletions(-)
(airflow) branch main updated (3eea44e9a7 -> a62bd83188)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 3eea44e9a7 Chart: Add init containers for jobs (#40454) add a62bd83188 Enable enforcing pydocstyle rule D213 in ruff. (#40448) No new revisions were added by this update. Summary of changes: airflow/api/client/api_client.py | 15 +- airflow/api/client/json_client.py | 27 ++- airflow/api/common/experimental/get_code.py| 3 +- .../api/common/experimental/get_dag_run_state.py | 3 +- airflow/api/common/mark_tasks.py | 9 +- airflow/api/common/trigger_dag.py | 6 +- .../endpoints/dag_warning_endpoint.py | 3 +- airflow/api_connexion/schemas/dag_run_schema.py| 3 +- airflow/auth/managers/base_auth_manager.py | 3 +- airflow/cli/cli_parser.py | 3 +- airflow/cli/commands/connection_command.py | 3 +- airflow/cli/commands/daemon_utils.py | 3 +- airflow/cli/commands/provider_command.py | 3 +- airflow/cli/commands/task_command.py | 6 +- airflow/cli/utils.py | 3 +- airflow/configuration.py | 6 +- airflow/dag_processing/manager.py | 3 +- airflow/datasets/__init__.py | 24 ++- airflow/decorators/base.py | 9 +- airflow/decorators/bash.py | 6 +- airflow/decorators/task_group.py | 9 +- airflow/executors/base_executor.py | 9 +- airflow/executors/executor_loader.py | 12 +- airflow/io/path.py | 12 +- .../pre_7_4_0_compatibility/pod_generator.py | 6 +- airflow/migrations/env.py | 6 +- .../versions/0001_1_5_0_current_schema.py | 3 +- .../versions/0002_1_5_0_create_is_encrypted.py | 3 +- .../versions/0003_1_5_0_for_compatibility.py | 3 +- .../0004_1_5_0_more_logging_into_task_isntance.py | 3 +- .../versions/0005_1_5_2_job_id_indices.py | 3 +- .../versions/0006_1_6_0_adding_extra_to_log.py | 3 +- .../migrations/versions/0007_1_6_0_add_dagrun.py | 3 +- .../versions/0008_1_6_0_task_duration.py | 3 +- .../versions/0009_1_6_0_dagrun_config.py | 3 +- .../0010_1_6_2_add_password_column_to_user.py | 3 +- .../versions/0011_1_6_2_dagrun_start_end.py| 3 +- ...7_0_add_notification_sent_column_to_sla_miss.py | 3 +- ..._1_7_0_add_a_column_to_track_the_encryption_.py | 3 +- ...4_1_7_0_add_is_encrypted_column_to_variable_.py | 3 +- .../versions/0015_1_7_1_rename_user_table.py | 3 +- .../versions/0016_1_7_1_add_ti_state_index.py | 3 +- .../0017_1_7_1_add_task_fails_journal_table.py | 3 +- .../versions/0018_1_7_1_add_dag_stats_table.py | 3 +- ...1_7_1_add_fractional_seconds_to_mysql_tables.py | 3 +- .../versions/0020_1_7_1_xcom_dag_task_indices.py | 3 +- .../0021_1_7_1_add_pid_field_to_taskinstance.py| 3 +- ..._7_1_add_dag_id_state_index_on_dag_run_table.py | 3 +- ..._1_8_2_add_max_tries_column_to_task_instance.py | 3 +- ..._1_8_2_make_xcom_value_column_a_large_binary.py | 3 +- .../versions/0025_1_8_2_add_ti_job_id_index.py | 3 +- .../0026_1_8_2_increase_text_size_for_mysql.py | 3 +- .../0027_1_10_0_add_time_zone_awareness.py | 3 +- ...1_10_0_add_kubernetes_resource_checkpointing.py | 3 +- ..._1_10_0_add_executor_config_to_task_instance.py | 3 +- ...0_1_10_0_add_kubernetes_scheduler_uniqueness.py | 3 +- .../migrations/versions/0031_1_10_0_merge_heads.py | 3 +- .../0032_1_10_0_fix_mysql_not_null_constraint.py | 3 +- .../versions/0033_1_10_0_fix_sqlite_foreign_key.py | 3 +- .../versions/0034_1_10_0_index_taskfail.py | 3 +- .../versions/0035_1_10_2_add_idx_log_dag.py| 3 +- .../0036_1_10_2_add_index_to_taskinstance.py | 3 +- .../0037_1_10_2_add_task_reschedule_table.py | 3 +- .../versions/0038_1_10_2_add_sm_dag_index.py | 3 +- .../versions/0039_1_10_2_add_superuser_field.py| 3 +- .../versions/0040_1_10_3_add_fields_to_dag.py | 3 +- .../0041_1_10_3_add_schedule_interval_to_dag.py| 3 +- ..._1_10_3_task_reschedule_fk_on_cascade_delete.py | 3 +- ...3_1_10_4_make_taskinstance_pool_not_nullable.py | 3 +- .../0044_1_10_7_add_serialized_dag_table.py| 3 +- .../versions/0045_1_10_7_add_root_dag_id_to_dag.py | 3 +- ...0_5_change_datetime_to_datetime2_6_on_mssql_.py | 18 +- .../0047_1_10_4_increase_queue_name_size_limit.py | 3 +- .../versions/0048_1_10_3_remove_dag_stat_table.py | 3 +- .../migrations/versions/0049_1_10_7_merge_heads.py | 3
(airflow) branch main updated (91e6e6055b -> 3eea44e9a7)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 91e6e6055b support auth key from content and from file (#40390) add 3eea44e9a7 Chart: Add init containers for jobs (#40454) No new revisions were added by this update. Summary of changes: chart/templates/jobs/create-user-job.yaml | 4 chart/templates/jobs/migrate-database-job.yaml | 4 chart/values.schema.json| 16 chart/values.yaml | 6 ++ helm_tests/airflow_aux/test_create_user_job.py | 17 + helm_tests/airflow_aux/test_migrate_database_job.py | 17 + 6 files changed, 64 insertions(+)
(airflow) branch main updated: Ensure try_number incremented for empty operator (#40426)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new 6694f47e73 Ensure try_number incremented for empty operator (#40426) 6694f47e73 is described below commit 6694f47e73f94d7db2f46accfa57e7ba2659b841 Author: Daniel Standish <15932138+dstand...@users.noreply.github.com> AuthorDate: Wed Jun 26 10:11:25 2024 -0700 Ensure try_number incremented for empty operator (#40426) --- airflow/models/dagrun.py| 1 + tests/models/test_dagrun.py | 24 2 files changed, 25 insertions(+) diff --git a/airflow/models/dagrun.py b/airflow/models/dagrun.py index 4773a89d1d..423c624f0c 100644 --- a/airflow/models/dagrun.py +++ b/airflow/models/dagrun.py @@ -1589,6 +1589,7 @@ class DagRun(Base, LoggingMixin): start_date=timezone.utcnow(), end_date=timezone.utcnow(), duration=0, +try_number=TI.try_number + 1, ) .execution_options( synchronize_session=False, diff --git a/tests/models/test_dagrun.py b/tests/models/test_dagrun.py index 93e0611243..021d09e889 100644 --- a/tests/models/test_dagrun.py +++ b/tests/models/test_dagrun.py @@ -36,6 +36,7 @@ from airflow.models.dagrun import DagRun, DagRunNote from airflow.models.taskinstance import TaskInstance, TaskInstanceNote, clear_task_instances from airflow.models.taskmap import TaskMap from airflow.models.taskreschedule import TaskReschedule +from airflow.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator from airflow.operators.python import ShortCircuitOperator from airflow.serialization.serialized_objects import SerializedDAG @@ -2019,6 +2020,29 @@ def test_schedule_tis_start_trigger(dag_maker, session): assert ti.state == TaskInstanceState.DEFERRED +def test_schedule_tis_empty_operator_try_number(dag_maker, session: Session): +""" +When empty operator is not actually run, then we need to increment the try_number, +since ordinarily it's incremented when scheduled, but empty operator is generally not scheduled. +""" + +with dag_maker(session=session): +BashOperator(task_id="real_task", bash_command="echo 1") +EmptyOperator(task_id="empty_task") + +dr: DagRun = dag_maker.create_dagrun(session=session) +session.commit() +tis = dr.task_instances +dr.schedule_tis(tis, session=session) +session.commit() +session.expunge_all() +tis = dr.get_task_instances(session=session) +real_ti = next(x for x in tis if x.task_id == "real_task") +empty_ti = next(x for x in tis if x.task_id == "empty_task") +assert real_ti.try_number == 1 +assert empty_ti.try_number == 1 + + def test_mapped_expand_kwargs(dag_maker): with dag_maker():
(airflow) branch main updated: Fix startupProbe timing comment (#40412)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new 70972e617a Fix startupProbe timing comment (#40412) 70972e617a is described below commit 70972e617a92a885345139abf03bcd4990bc5aae Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com> AuthorDate: Mon Jun 24 21:50:06 2024 -0600 Fix startupProbe timing comment (#40412) The default config only allows for 1 minute for each component to start up. --- chart/values.yaml | 9 +++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/chart/values.yaml b/chart/values.yaml index c623f3e9dd..5e4e50dfd5 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -811,13 +811,14 @@ scheduler: periodSeconds: 60 command: ~ - # Wait for at most 10 minutes (6*10s) for the scheduler container to startup. - # livenessProbe kicks in after the startup + # Wait for at most 1 minute (6*10s) for the scheduler container to startup. + # livenessProbe kicks in after the first successful startupProbe startupProbe: failureThreshold: 6 periodSeconds: 10 timeoutSeconds: 20 command: ~ + # Airflow 2.0 allows users to run multiple schedulers, # However this feature is only recommended for MySQL 8+ and Postgres replicas: 1 @@ -1252,6 +1253,8 @@ _rpcServer: periodSeconds: 10 scheme: HTTP + # Wait for at most 1 minute (6*10s) for the RPC server container to startup. + # livenessProbe kicks in after the first successful startupProbe startupProbe: timeoutSeconds: 20 failureThreshold: 6 @@ -1286,6 +1289,8 @@ webserver: periodSeconds: 10 scheme: HTTP + # Wait for at most 1 minute (6*10s) for the webserver container to startup. + # livenessProbe kicks in after the first successful startupProbe startupProbe: timeoutSeconds: 20 failureThreshold: 6
(airflow) branch main updated: Add annotations for Redis StatefulSet (#40281)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new ed57711866 Add annotations for Redis StatefulSet (#40281)
ed57711866 is described below
commit ed57711866527d4e56ce195944b26607317b1f26
Author: Masoud Arbabi <160050366+cybersavvy...@users.noreply.github.com>
AuthorDate: Fri Jun 21 16:57:19 2024 +0200
Add annotations for Redis StatefulSet (#40281)
---
chart/templates/redis/redis-statefulset.yaml | 3 +++
chart/values.schema.json | 8
chart/values.yaml| 3 +++
helm_tests/airflow_aux/test_annotations.py | 26 ++
4 files changed, 40 insertions(+)
diff --git a/chart/templates/redis/redis-statefulset.yaml
b/chart/templates/redis/redis-statefulset.yaml
index 5aa9e13598..cbc504e0a5 100644
--- a/chart/templates/redis/redis-statefulset.yaml
+++ b/chart/templates/redis/redis-statefulset.yaml
@@ -41,6 +41,9 @@ metadata:
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
+ {{- if .Values.redis.annotations }}
+ annotations: {{- toYaml .Values.redis.annotations | nindent 4 }}
+ {{- end }}
spec:
serviceName: {{ include "airflow.fullname" . }}-redis
selector:
diff --git a/chart/values.schema.json b/chart/values.schema.json
index 4b4b8af381..fed2168ea7 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -7611,6 +7611,14 @@
"additionalProperties": {
"type": "string"
}
+},
+"annotations": {
+"description": "Annotations for the redis.",
+"type": "object",
+"default": {},
+"additionalProperties": {
+"type": "string"
+}
}
}
},
diff --git a/chart/values.yaml b/chart/values.yaml
index 478ed2e49b..c623f3e9dd 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -2230,6 +2230,9 @@ redis:
enabled: true
terminationGracePeriodSeconds: 600
+ # Annotations for Redis Statefulset
+ annotations: {}
+
# Create ServiceAccount
serviceAccount:
# default value is true
diff --git a/helm_tests/airflow_aux/test_annotations.py
b/helm_tests/airflow_aux/test_annotations.py
index 8f297bf73b..902bacd755 100644
--- a/helm_tests/airflow_aux/test_annotations.py
+++ b/helm_tests/airflow_aux/test_annotations.py
@@ -426,3 +426,29 @@ class TestPerComponentPodAnnotations:
for k, v in expected_annotations.items():
assert k in annotations
assert v == annotations[k]
+
+
+class TestRedisAnnotations:
+"""Tests Redis Annotations."""
+
+def test_redis_annotations_are_added(self):
+# Test Case
+values = {"redis": {"annotations": {"example": "redis"}}}
+show_only = "templates/redis/redis-statefulset.yaml"
+expected_annotations = {"example": "redis"}
+
+k8s_objects = render_chart(
+values=values,
+show_only=[show_only],
+)
+
+# This test relies on the convention that the helm chart puts
annotations
+# in its own .yaml file, so by specifying `show_only`,
+# we should only get a single k8s_object here - the target object that
+# we hope to test on.
+assert len(k8s_objects) == 1
+obj = k8s_objects[0]
+
+for k, v in expected_annotations.items():
+assert k in obj["metadata"]["annotations"]
+assert v == obj["metadata"]["annotations"][k]
(airflow) branch main updated (13437cba97 -> f81abd6145)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
from 13437cba97 Helm chart 1.14.0 has been released (#40313)
add f81abd6145 add git-sync-ssh secret template (#39936)
No new revisions were added by this update.
Summary of changes:
chart/templates/_helpers.yaml | 11 +--
.../templates/scheduler/scheduler-deployment.yaml | 2 +-
...-keytab-secret.yaml => git-ssh-key-secret.yaml} | 12 ++--
.../templates/triggerer/triggerer-deployment.yaml | 4 +--
.../templates/webserver/webserver-deployment.yaml | 2 +-
chart/templates/workers/worker-deployment.yaml | 2 +-
chart/values.schema.json | 8 +
chart/values.yaml | 6
...ync_triggerer.py => test_git_ssh_key_secret.py} | 17 +-
helm_tests/other/test_git_sync_scheduler.py| 30 ++
helm_tests/other/test_git_sync_triggerer.py| 36 ++
helm_tests/other/test_git_sync_webserver.py| 20
helm_tests/other/test_git_sync_worker.py | 36 ++
13 files changed, 160 insertions(+), 26 deletions(-)
copy chart/templates/secrets/{kerberos-keytab-secret.yaml =>
git-ssh-key-secret.yaml} (76%)
copy helm_tests/other/{test_git_sync_triggerer.py =>
test_git_ssh_key_secret.py} (69%)
(airflow-site) branch announce_helm_1.14.0 created (now e3c13bcfba)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch announce_helm_1.14.0 in repository https://gitbox.apache.org/repos/asf/airflow-site.git at e3c13bcfba Announce helm chart 1.14.0 This branch includes the following new commits: new e3c13bcfba Announce helm chart 1.14.0 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(airflow-site) 01/01: Announce helm chart 1.14.0
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch announce_helm_1.14.0 in repository https://gitbox.apache.org/repos/asf/airflow-site.git commit e3c13bcfba9954872eb60f0b77b446d9865e218a Author: Jed Cunningham AuthorDate: Tue Jun 18 19:58:06 2024 -0600 Announce helm chart 1.14.0 --- landing-pages/site/content/en/announcements/_index.md | 9 + 1 file changed, 9 insertions(+) diff --git a/landing-pages/site/content/en/announcements/_index.md b/landing-pages/site/content/en/announcements/_index.md index 83ff8be17c..8bdbf5be42 100644 --- a/landing-pages/site/content/en/announcements/_index.md +++ b/landing-pages/site/content/en/announcements/_index.md @@ -13,6 +13,15 @@ menu: **Note:** Follow [@ApacheAirflow](https://twitter.com/ApacheAirflow) on Twitter for the latest news and announcements! +# June 18, 2024 + +We've just released Apache **Airflow Helm chart 1.14.0**. + + ArtifactHub: https://artifacthub.io/packages/helm/apache-airflow/airflow \ + Docs: https://airflow.apache.org/docs/helm-chart/1.14.0/ \ +️ Release Notes: https://airflow.apache.org/docs/helm-chart/1.14.0/release_notes.html \ +襁 Sources: https://airflow.apache.org/docs/helm-chart/1.14.0/installing-helm-chart-from-sources.html + # June 10, 2024 We’ve just released Apache **Airflow 2.9.2**.
svn commit: r69848 - /release/airflow/helm-chart/1.13.0/
Author: jedcunningham Date: Wed Jun 19 00:55:37 2024 New Revision: 69848 Log: Remove old Helm Chart release: 1.13.0 Removed: release/airflow/helm-chart/1.13.0/
(airflow-site) branch helm-1.14.0-docs deleted (was c85a6a2987)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch helm-1.14.0-docs in repository https://gitbox.apache.org/repos/asf/airflow-site.git was c85a6a2987 Add documentation for Apache Airflow Helm Chart 1.14.0 The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
(airflow-site) branch helm-1.14.0-docs created (now c85a6a2987)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch helm-1.14.0-docs in repository https://gitbox.apache.org/repos/asf/airflow-site.git at c85a6a2987 Add documentation for Apache Airflow Helm Chart 1.14.0 This branch includes the following new commits: new c85a6a2987 Add documentation for Apache Airflow Helm Chart 1.14.0 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(airflow) annotated tag helm-chart/1.14.0 updated (8eebe2b4c8 -> 5bef465d6b)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to annotated tag helm-chart/1.14.0 in repository https://gitbox.apache.org/repos/asf/airflow.git *** WARNING: tag helm-chart/1.14.0 was modified! *** from 8eebe2b4c8 (commit) to 5bef465d6b (tag) tagging 8eebe2b4c8e1271d26a796ec599b43b46941125c (commit) replaces providers-amazon/8.24.0 by Jed Cunningham on Tue Jun 18 18:13:20 2024 -0600 - Log - Apache Airflow Helm Chart 1.14.0 -BEGIN PGP SIGNATURE- iQIzBAABCAAdFiEE4aHphPVbjygL2cuiC7cWOJKi5I4FAmZyIqAACgkQC7cWOJKi 5I6Ekg/7BR8AjKjpTFi1kzLkTIKdwM3bteJns8wkxAPVNwJyZ3mPHsPOK4g6h6qU Ovz1KvPqzE07EkVnbbESJEd+YSbKBZJQYM4e7td17oNhJ3xZ7j+Ol1G7YYQQVzdJ XD0ZudIfPK/vflML+8tCWbvtSuHQUgK67WalG94nuIwuuQLzSMA9hPBS45DRUU9c xg8U3tTd8l0LENQNbpYaDm5rTV4Qbcc4QHE93sj3bswG9C5wjrShYnR+Kq5zhrKF acWG9zrsX2xFEncAurV2w0y1bimDUzOHWRTquMV6EQmlRxs+2FXCevzdFvgjlS7X bKVrqf5Cpgd0Y9rY0gu58qdUVipfwu8iIGL2iA0bUOd3qTXREeEJ+V9LH0Ak4iL0 uwCgWyzw0OD8Jn0NSTBz0vwlvPYQ3dR8CMnB3UvJG/hSI8E4YJYSPNe5OxXlSrXj Y7wmve+dlAv8DwkvKLgTjdzN8SFLNGGJjHVwEkIdEo/5blMxtY/81uKeavIcTOPu vbcmwlJk9cojfaUm6OyMBj+rvMl84QS9phrORFmkPVfYbWC5Yx59sGZaWX3JHZuj eu82gwqm8nC+rSrj8FDEpGQvcknx1eed4pEnEQHiD2fteQeZfEBliTQoAByGE22+ 3guAyRFYVz7zTVxEtIj2pR96AeqomjD4sKsoqeeyAd5hywzA0mM= =1cVZ -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
svn commit: r69847 - /release/airflow/helm-chart/1.14.0/
Author: jedcunningham Date: Wed Jun 19 00:12:41 2024 New Revision: 69847 Log: Release Airflow Helm Chart Check 1.14.0 from 1.14.0rc1 Added: release/airflow/helm-chart/1.14.0/ release/airflow/helm-chart/1.14.0/airflow-1.14.0.tgz - copied unchanged from r69748, dev/airflow/helm-chart/1.14.0rc1/airflow-1.14.0.tgz release/airflow/helm-chart/1.14.0/airflow-1.14.0.tgz.asc - copied unchanged from r69748, dev/airflow/helm-chart/1.14.0rc1/airflow-1.14.0.tgz.asc release/airflow/helm-chart/1.14.0/airflow-1.14.0.tgz.prov - copied unchanged from r69748, dev/airflow/helm-chart/1.14.0rc1/airflow-1.14.0.tgz.prov release/airflow/helm-chart/1.14.0/airflow-1.14.0.tgz.sha512 - copied unchanged from r69748, dev/airflow/helm-chart/1.14.0rc1/airflow-1.14.0.tgz.sha512 release/airflow/helm-chart/1.14.0/airflow-chart-1.14.0-source.tar.gz - copied unchanged from r69748, dev/airflow/helm-chart/1.14.0rc1/airflow-chart-1.14.0-source.tar.gz release/airflow/helm-chart/1.14.0/airflow-chart-1.14.0-source.tar.gz.asc - copied unchanged from r69748, dev/airflow/helm-chart/1.14.0rc1/airflow-chart-1.14.0-source.tar.gz.asc release/airflow/helm-chart/1.14.0/airflow-chart-1.14.0-source.tar.gz.sha512 - copied unchanged from r69748, dev/airflow/helm-chart/1.14.0rc1/airflow-chart-1.14.0-source.tar.gz.sha512
(airflow) annotated tag helm-chart/1.14.0rc1 updated (8eebe2b4c8 -> 3604ad3023)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to annotated tag helm-chart/1.14.0rc1 in repository https://gitbox.apache.org/repos/asf/airflow.git *** WARNING: tag helm-chart/1.14.0rc1 was modified! *** from 8eebe2b4c8 (commit) to 3604ad3023 (tag) tagging 8eebe2b4c8e1271d26a796ec599b43b46941125c (commit) replaces providers-amazon/8.24.0 by Jed Cunningham on Fri Jun 14 14:39:07 2024 -0600 - Log - Apache Airflow Helm Chart 1.14.0rc1 -BEGIN PGP SIGNATURE- iQIzBAABCAAdFiEE4aHphPVbjygL2cuiC7cWOJKi5I4FAmZsqmwACgkQC7cWOJKi 5I7JUw/9EtpFQYdmVDzNkAER4haTO9Isi+V1yS1hqvLUwEIYIoYXQPzjRq9rlO/g 8GIiTIEgO+7zzZTYU8AdvdXqZ3BvusNM6OSCxkQF0eO1OHQ8N+ohSFazK/K+g+1Q 0fEWn1rXqv7cQrz0OMS5dH1sVpTXUv+SIB07oJ4ntHgnDn8zWZ4suMVRK44hHlu4 sPgniG8l7QBebqKoGTnX8rRKyfr6WiXUwstqrWadW8ezRjEQ1BO+mMCX+lab5JuE IwdZGlujj1+kH/fa0dS0yC80edQRbraMdLAyCGs3stBvN/+xbl9kZeisRc40LD/p hEzlCpjTCAhhBimfdBr86AgHXevBUaUVU/hWsyPupaJvAFCFwuvFw0a0Idz9EFRZ QeL7vEzXeG5hqM+eJhTRf43M5vnMjerJmL7eHYlVB3HQnu9iTe7fPYF++XwC6HFw REFR+6peimTO1ZnaQcbBGlPTLpN6KhNWqBy+9djhtN5FhaQR7GT3B8CHStCmF1kw ddaW5QTz2DHL4zOS2j7c7QNBwRMtJT7nMYTKiSBsjWGkvIqmDn2R9eZvPs+EOVyC D5Oc/SvoxfKeUdHksKeXzW5g0Ie4YUYEf7TI4tNDqhWr9lfFp4x/FjFeGWNslCjp mdoYPV0ljypwB3yDV10wue9M5WffPp/O2jWhzF9NnXt57mAEptY= =ANsb -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
svn commit: r69749 - /dev/airflow/helm-chart/1.13.1rc1/
Author: jedcunningham Date: Fri Jun 14 20:45:28 2024 New Revision: 69749 Log: Remove old Helm Chart release: 1.13.1rc1 Removed: dev/airflow/helm-chart/1.13.1rc1/
svn commit: r69748 [1/2] - /dev/airflow/helm-chart/1.14.0rc1/
Author: jedcunningham Date: Fri Jun 14 20:44:51 2024 New Revision: 69748 Log: Add artifacts for Helm Chart 1.14.0rc1 Added: dev/airflow/helm-chart/1.14.0rc1/ dev/airflow/helm-chart/1.14.0rc1/airflow-1.14.0.tgz (with props) dev/airflow/helm-chart/1.14.0rc1/airflow-1.14.0.tgz.asc dev/airflow/helm-chart/1.14.0rc1/airflow-1.14.0.tgz.prov dev/airflow/helm-chart/1.14.0rc1/airflow-1.14.0.tgz.sha512 dev/airflow/helm-chart/1.14.0rc1/airflow-chart-1.14.0-source.tar.gz (with props) dev/airflow/helm-chart/1.14.0rc1/airflow-chart-1.14.0-source.tar.gz.asc dev/airflow/helm-chart/1.14.0rc1/airflow-chart-1.14.0-source.tar.gz.sha512 dev/airflow/helm-chart/1.14.0rc1/index.yaml Added: dev/airflow/helm-chart/1.14.0rc1/airflow-1.14.0.tgz == Binary file - no diff available. Propchange: dev/airflow/helm-chart/1.14.0rc1/airflow-1.14.0.tgz -- svn:mime-type = application/octet-stream Added: dev/airflow/helm-chart/1.14.0rc1/airflow-1.14.0.tgz.asc == --- dev/airflow/helm-chart/1.14.0rc1/airflow-1.14.0.tgz.asc (added) +++ dev/airflow/helm-chart/1.14.0rc1/airflow-1.14.0.tgz.asc Fri Jun 14 20:44:51 2024 @@ -0,0 +1,17 @@ +-BEGIN PGP SIGNATURE- + +iQJNBAABCAA3FiEE4aHphPVbjygL2cuiC7cWOJKi5I4FAmZsqwMZHGplZGN1bm5p +bmdoYW1AYXBhY2hlLm9yZwAKCRALtxY4kqLkjvsoD/48To5Uf1VZYlkNXYyTtrp5 +/VHZYDYLo67AaF0jZsahId8HJvAwubIaYJz9+yrKAUQ/b0/wqIHnyjHA8s06ubNy +zMYMI5ZtOVwyiLEYvX/xT4JEP1Utn7/UvZPMFg0qDIuezaS3wB5o9Sa5hv9HcsFk +P59ruoJR5diIhocRALl2nAJkb/R3Ty6rvJm++IwM11gwCGs91cbDgJ4Aik/v7wJK +4WwluC9i2TgGBlU0TvW1tWI6/NaR8Inl0MBqI6s3fbn9HTHjVksmaoYCqZkff2rX +3MQevAk+zTyO9Qei88TMxUZm2Cc6PkQYCFko8DVvYna0OGhzCbP8F6vQH/20ndI5 +JafTfsPIvlYYKwFvASUHYvBLuwdUGAPesPFCr4N++8sPBA3XQSYJmjbb7ccIHGs/ +o/NIg7bpyH3UjMgVAydbGgZ6Zy7kMop6qI5kiF9lcqelThLgmA3ndwyiF2zaayzj +km0oSF6Iua/sXujxOD1dPt8a7IC3gI3c4ue0D562tEkgJHtdKJjxFVsD7741IGF3 +0g5+QHEPe7DebjI6j+EaTY4UfKI4UrEwNNscKuc31BEHgMSmahreuV3EhXxcziI+ +c6OLPsUwZK9Z58AZ6YyA4Ug/9Y1hlOAHTxZfpNxjPFKDUhq4fckfg5aW8X8RtR3a +kAxjKL+KXG5Vh8yg+SEnHg== +=3Nhr +-END PGP SIGNATURE- Added: dev/airflow/helm-chart/1.14.0rc1/airflow-1.14.0.tgz.prov == --- dev/airflow/helm-chart/1.14.0rc1/airflow-1.14.0.tgz.prov (added) +++ dev/airflow/helm-chart/1.14.0rc1/airflow-1.14.0.tgz.prov Fri Jun 14 20:44:51 2024 @@ -0,0 +1,173 @@ +-BEGIN PGP SIGNED MESSAGE- +Hash: SHA256 + +annotations: + artifacthub.io/changes: | +- description: Enable MySQL KEDA support for triggerer + kind: added + links: + - name: '#37365' +url: https://github.com/apache/airflow/pull/37365 +- description: Allow AWS Executors + kind: added + links: + - name: '#38524' +url: https://github.com/apache/airflow/pull/38524 +- description: Allow ``valueFrom`` in env config of components + kind: changed + links: + - name: '#40135' +url: https://github.com/apache/airflow/pull/40135 +- description: Enable templating in ``extraContainers`` and ``extraInitContainers`` + kind: changed + links: + - name: '#38507' +url: https://github.com/apache/airflow/pull/38507 +- description: Add safe-to-evict annotation to pod-template-file + kind: changed + links: + - name: '#37352' +url: https://github.com/apache/airflow/pull/37352 +- description: Support ``workers.command`` for KubernetesExecutor + kind: changed + links: + - name: '#39132' +url: https://github.com/apache/airflow/pull/39132 +- description: Add ``priorityClassName`` to Jobs + kind: changed + links: + - name: '#39133' +url: https://github.com/apache/airflow/pull/39133 +- description: Add Kerberos sidecar to pod-template-file + kind: changed + links: + - name: '#38815' +url: https://github.com/apache/airflow/pull/38815 +- description: Add templated field support for extra containers + kind: changed + links: + - name: '#38510' +url: https://github.com/apache/airflow/pull/38510 +- description: Set ``workers.safeToEvict`` default to False + kind: fixed + links: + - name: '#40229' +url: https://github.com/apache/airflow/pull/40229 +- description: 'Docs: Document ``extraContainers`` and ``extraInitContainers`` that +are templated' + kind: changed + links: + - name: '#40033' +url: https://github.com/apache/airflow/pull/40033 +- description: 'Docs: Fix typo in HorizontalPodAutoscaling documentation' + kind: changed + links: + - name: '#39307' +url: https://github.com/apache/airflow/pull/39307 +- description: 'Docs: Fix supported k8s versions in docs
svn commit: r69748 [2/2] - /dev/airflow/helm-chart/1.14.0rc1/
Added: dev/airflow/helm-chart/1.14.0rc1/index.yaml == --- dev/airflow/helm-chart/1.14.0rc1/index.yaml (added) +++ dev/airflow/helm-chart/1.14.0rc1/index.yaml Fri Jun 14 20:44:51 2024 @@ -0,0 +1,2117 @@ +apiVersion: v1 +entries: + airflow: + - annotations: + artifacthub.io/changes: | +- description: Enable MySQL KEDA support for triggerer + kind: added + links: + - name: '#37365' +url: https://github.com/apache/airflow/pull/37365 +- description: Allow AWS Executors + kind: added + links: + - name: '#38524' +url: https://github.com/apache/airflow/pull/38524 +- description: Allow ``valueFrom`` in env config of components + kind: changed + links: + - name: '#40135' +url: https://github.com/apache/airflow/pull/40135 +- description: Enable templating in ``extraContainers`` and ``extraInitContainers`` + kind: changed + links: + - name: '#38507' +url: https://github.com/apache/airflow/pull/38507 +- description: Add safe-to-evict annotation to pod-template-file + kind: changed + links: + - name: '#37352' +url: https://github.com/apache/airflow/pull/37352 +- description: Support ``workers.command`` for KubernetesExecutor + kind: changed + links: + - name: '#39132' +url: https://github.com/apache/airflow/pull/39132 +- description: Add ``priorityClassName`` to Jobs + kind: changed + links: + - name: '#39133' +url: https://github.com/apache/airflow/pull/39133 +- description: Add Kerberos sidecar to pod-template-file + kind: changed + links: + - name: '#38815' +url: https://github.com/apache/airflow/pull/38815 +- description: Add templated field support for extra containers + kind: changed + links: + - name: '#38510' +url: https://github.com/apache/airflow/pull/38510 +- description: Set ``workers.safeToEvict`` default to False + kind: fixed + links: + - name: '#40229' +url: https://github.com/apache/airflow/pull/40229 +- description: 'Docs: Document ``extraContainers`` and ``extraInitContainers`` that +are templated' + kind: changed + links: + - name: '#40033' +url: https://github.com/apache/airflow/pull/40033 +- description: 'Docs: Fix typo in HorizontalPodAutoscaling documentation' + kind: changed + links: + - name: '#39307' +url: https://github.com/apache/airflow/pull/39307 +- description: 'Docs: Fix supported k8s versions in docs' + kind: changed + links: + - name: '#39172' +url: https://github.com/apache/airflow/pull/39172 +- description: 'Docs: Fix typo in YAML path for ``brokerUrlSecretName``' + kind: changed + links: + - name: '#39115' +url: https://github.com/apache/airflow/pull/39115 +- description: 'Misc: Default Airflow version to 2.9.2' + kind: changed + links: + - name: '#40160' +url: https://github.com/apache/airflow/pull/40160 +- description: 'Misc: Limit Redis image to 7.2' + kind: changed + links: + - name: '#38928' +url: https://github.com/apache/airflow/pull/38928 +- description: 'Misc: Build Helm values schemas with Kubernetes 1.29 resources' + kind: changed + links: + - name: '#38460' +url: https://github.com/apache/airflow/pull/38460 +- description: 'Misc: Add missing containers to resources docs' + kind: changed + links: + - name: '#38534' +url: https://github.com/apache/airflow/pull/38534 +- description: 'Misc: Upgrade StatsD Exporter image to 0.26.1' + kind: changed + links: + - name: '#38416' +url: https://github.com/apache/airflow/pull/38416 +- description: 'Misc: Remove K8S 1.25 support' + kind: changed + links: + - name: '#38367' +url: https://github.com/apache/airflow/pull/38367 + artifacthub.io/links: | +- name: Documentation + url: https://airflow.apache.org/docs/helm-chart/1.14.0/ + artifacthub.io/screenshots: | +- title: DAGs View + url: https://airflow.apache.org/docs/apache-airflow/2.9.2/_images/dags.png +- title: Datasets View + url: https://airflow.apache.org/docs/apache-airflow/2.9.2/_images/datasets.png +- title: Grid View + url:
(airflow) branch main updated: Chart: Release notes for 1.14.0 (#40246)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new 8eebe2b4c8 Chart: Release notes for 1.14.0 (#40246) 8eebe2b4c8 is described below commit 8eebe2b4c8e1271d26a796ec599b43b46941125c Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com> AuthorDate: Fri Jun 14 14:32:43 2024 -0600 Chart: Release notes for 1.14.0 (#40246) --- chart/Chart.yaml | 107 +++--- chart/RELEASE_NOTES.rst | 75 + chart/newsfragments/37197.significant.rst | 8 --- chart/newsfragments/40160.significant.rst | 3 - chart/newsfragments/40229.significant.rst | 6 -- chart/reproducible_build.yaml | 4 +- 6 files changed, 173 insertions(+), 30 deletions(-) diff --git a/chart/Chart.yaml b/chart/Chart.yaml index a468bf0ae6..f44bc7ed87 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -65,19 +65,104 @@ annotations: - title: Code View url: https://airflow.apache.org/docs/apache-airflow/2.9.2/_images/code.png artifacthub.io/changes: | -- description: Don't overwrite ``.Values.airflowPodAnnotations`` - kind: fixed +- description: Enable MySQL KEDA support for triggerer + kind: added + links: + - name: '#37365' +url: https://github.com/apache/airflow/pull/37365 +- description: Allow AWS Executors + kind: added links: - - name: '#37917' -url: https://github.com/apache/airflow/pull/37917 -- description: Fix cluster-wide RBAC naming clash when using multiple ``multiNamespace`` -releases with the same name + - name: '#38524' +url: https://github.com/apache/airflow/pull/38524 +- description: Allow ``valueFrom`` in env config of components + kind: changed + links: + - name: '#40135' +url: https://github.com/apache/airflow/pull/40135 +- description: Enable templating in ``extraContainers`` and ``extraInitContainers`` + kind: changed + links: + - name: '#38507' +url: https://github.com/apache/airflow/pull/38507 +- description: Add safe-to-evict annotation to pod-template-file + kind: changed + links: + - name: '#37352' +url: https://github.com/apache/airflow/pull/37352 +- description: Support ``workers.command`` for KubernetesExecutor + kind: changed + links: + - name: '#39132' +url: https://github.com/apache/airflow/pull/39132 +- description: Add ``priorityClassName`` to Jobs + kind: changed + links: + - name: '#39133' +url: https://github.com/apache/airflow/pull/39133 +- description: Add Kerberos sidecar to pod-template-file + kind: changed + links: + - name: '#38815' +url: https://github.com/apache/airflow/pull/38815 +- description: Add templated field support for extra containers + kind: changed + links: + - name: '#38510' +url: https://github.com/apache/airflow/pull/38510 +- description: Set ``workers.safeToEvict`` default to False kind: fixed links: - - name: '#37197' -url: https://github.com/apache/airflow/pull/37197 -- description: 'Misc: Default airflow version to 2.8.3' + - name: '#40229' +url: https://github.com/apache/airflow/pull/40229 +- description: 'Docs: Document ``extraContainers`` and ``extraInitContainers`` that +are templated' + kind: changed + links: + - name: '#40033' +url: https://github.com/apache/airflow/pull/40033 +- description: 'Docs: Fix typo in HorizontalPodAutoscaling documentation' + kind: changed + links: + - name: '#39307' +url: https://github.com/apache/airflow/pull/39307 +- description: 'Docs: Fix supported k8s versions in docs' + kind: changed + links: + - name: '#39172' +url: https://github.com/apache/airflow/pull/39172 +- description: 'Docs: Fix typo in YAML path for ``brokerUrlSecretName``' + kind: changed + links: + - name: '#39115' +url: https://github.com/apache/airflow/pull/39115 +- description: 'Misc: Default Airflow version to 2.9.2' + kind: changed + links: + - name: '#40160' +url: https://github.com/apache/airflow/pull/40160 +- description: 'Misc: Limit Redis image to 7.2' + kind: changed + links: + - name: '#38928' +url: https://github.com/apache/airflow/pull/38928 +- description: 'Misc: Build Helm values schemas with Kubernetes 1.29 resources' + kind: changed + links: + - name: '#38460' +url: https://github.com/apache/airflow/pull/38460 +- description: 'Misc: Add missing containers to resources docs' + kind: c
(airflow) branch main updated: Chart: set `workers.safeToEvict` default to False (#40229)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 4c9f12d2bc Chart: set `workers.safeToEvict` default to False (#40229)
4c9f12d2bc is described below
commit 4c9f12d2bce5af0c629e9b9e9916cd525817931d
Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com>
AuthorDate: Fri Jun 14 12:36:20 2024 -0600
Chart: set `workers.safeToEvict` default to False (#40229)
This is a safer default for our workers.
This can be safe to set to true if you have a long enough
`workers.terminationGracePeriodSeconds` set, but what is
"long enough" is very situational, so I feel its better to
default to not evicting worker pods.
---
chart/newsfragments/40229.significant.rst | 6 ++
chart/values.schema.json | 2 +-
chart/values.yaml | 2 +-
helm_tests/airflow_core/test_worker.py| 2 +-
4 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/chart/newsfragments/40229.significant.rst
b/chart/newsfragments/40229.significant.rst
new file mode 100644
index 00..56187f3f60
--- /dev/null
+++ b/chart/newsfragments/40229.significant.rst
@@ -0,0 +1,6 @@
+``workers.safeToEvict`` default changed to False
+
+The default for ``workers.safeToEvict`` now defaults to False. This is a safer
default
+as it prevents the nodes workers are running on from being scaled down by the
+[K8s Cluster
Autoscaler](https://kubernetes.io/docs/concepts/cluster-administration/cluster-autoscaling/#cluster-autoscaler).
+If you would like to retain the previous behavior, you can set this config to
True.
diff --git a/chart/values.schema.json b/chart/values.schema.json
index c2abb29441..7ee05eecfc 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -1864,7 +1864,7 @@
"safeToEvict": {
"description": "This setting tells Kubernetes that it's ok
to evict when it wants to scale a node down.",
"type": "boolean",
-"default": true
+"default": false
},
"extraContainers": {
"description": "Launch additional containers into workers
(templated). Note, if used with KubernetesExecutor, you are responsible for
signaling sidecars to exit when the main container finishes so Airflow can
continue the worker shutdown process!",
diff --git a/chart/values.yaml b/chart/values.yaml
index 333fa44d4d..f8b03a5ef6 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -676,7 +676,7 @@ workers:
# This setting tells kubernetes that its ok to evict
# when it wants to scale a node down.
- safeToEvict: true
+ safeToEvict: false
# Launch additional containers into worker (templated).
# Note: If used with KubernetesExecutor, you are responsible for signaling
sidecars to exit when the main
diff --git a/helm_tests/airflow_core/test_worker.py
b/helm_tests/airflow_core/test_worker.py
index af370f3d9a..1fcc88caa6 100644
--- a/helm_tests/airflow_core/test_worker.py
+++ b/helm_tests/airflow_core/test_worker.py
@@ -755,7 +755,7 @@ class TestWorker:
@pytest.mark.parametrize(
"globalScope, localScope, precedence",
[
-({}, {}, "true"),
+({}, {}, "false"),
({}, {"safeToEvict": True}, "true"),
({}, {"safeToEvict": False}, "false"),
(
(airflow) branch main updated: Chart: Allow valueFrom in env config of components (#40135)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 279e8b7772 Chart: Allow valueFrom in env config of components (#40135)
279e8b7772 is described below
commit 279e8b7772c483890726734600df89613cca3263
Author: Daniel Gellert
AuthorDate: Tue Jun 11 02:21:07 2024 +0200
Chart: Allow valueFrom in env config of components (#40135)
---
chart/templates/_helpers.yaml| 26 ++
chart/values.schema.json | 320 +--
helm_tests/airflow_aux/test_create_user_job.py | 20 +-
helm_tests/airflow_aux/test_pod_template_file.py | 24 +-
helm_tests/airflow_core/test_dag_processor.py| 20 +-
helm_tests/airflow_core/test_scheduler.py| 20 +-
helm_tests/airflow_core/test_triggerer.py| 20 +-
helm_tests/airflow_core/test_worker.py | 20 +-
helm_tests/other/test_flower.py | 20 +-
helm_tests/webserver/test_webserver.py | 20 +-
10 files changed, 486 insertions(+), 24 deletions(-)
diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml
index aaa86ecbe8..dfa23fba2f 100644
--- a/chart/templates/_helpers.yaml
+++ b/chart/templates/_helpers.yaml
@@ -1025,10 +1025,36 @@ capabilities:
{{- $env := index . 1 -}}
{{- range $i, $config := $env }}
- name: {{ $config.name }}
+{{- if $config.value }}
value: {{ $config.value | quote }}
+{{- else if $config.valueFrom }}
+valueFrom:
+ {{- if $config.valueFrom.secretKeyRef }}
+ secretKeyRef:
+name: {{ $config.valueFrom.secretKeyRef.name }}
+key: {{ $config.valueFrom.secretKeyRef.key }}
+ {{- else if $config.valueFrom.configMapKeyRef }}
+ configMapKeyRef:
+name: {{ $config.valueFrom.configMapKeyRef.name }}
+key: {{ $config.valueFrom.configMapKeyRef.key }}
+ {{- end }}
+{{- end }}
{{- if or (eq $.Values.executor "KubernetesExecutor") (eq
$.Values.executor "LocalKubernetesExecutor") (eq $.Values.executor
"CeleryKubernetesExecutor") }}
- name: AIRFLOW__KUBERNETES_ENVIRONMENT_VARIABLES__{{ $config.name }}
+{{- if $config.value }}
value: {{ $config.value | quote }}
+{{- else if $config.valueFrom }}
+valueFrom:
+ {{- if $config.valueFrom.secretKeyRef }}
+ secretKeyRef:
+name: {{ $config.valueFrom.secretKeyRef.name }}
+key: {{ $config.valueFrom.secretKeyRef.key }}
+ {{- else if $config.valueFrom.configMapKeyRef }}
+ configMapKeyRef:
+name: {{ $config.valueFrom.configMapKeyRef.name }}
+key: {{ $config.valueFrom.configMapKeyRef.key }}
+ {{- end }}
+{{- end }}
{{- end }}
{{- end }}
{{- end }}
diff --git a/chart/values.schema.json b/chart/values.schema.json
index e155719720..c2abb29441 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -2145,11 +2145,47 @@
},
"value": {
"type": "string"
+},
+"valueFrom": {
+"type": "object",
+"properties": {
+"configMapKeyRef": {
+"$ref":
"#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector",
+"description": "Selects a key of a
ConfigMap."
+},
+"secretKeyRef": {
+"$ref":
"#/definitions/io.k8s.api.core.v1.SecretKeySelector",
+"description": "Selects a key of a
secret in the pod's namespace"
+}
+},
+"anyOf": [
+{
+"required": [
+"configMapKeyRef"
+]
+},
+{
+"required": [
+"secretKeyRef"
+]
+}
+]
}
},
"required": [
-"
(airflow) branch main updated (d504bfa5d1 -> b93b94397a)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from d504bfa5d1 Aip-61: Add validation on task `executor` field (#40030) add b93b94397a added Tiqets to the list of companies using airflow (#40096) No new revisions were added by this update. Summary of changes: INTHEWILD.md | 1 + 1 file changed, 1 insertion(+)
(airflow) branch main updated: Add rpc server to helm chart (#38549)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 7a254ff240 Add rpc server to helm chart (#38549)
7a254ff240 is described below
commit 7a254ff240e7f39a8fd08cb3bae18021af29c9c1
Author: Daniel Standish <15932138+dstand...@users.noreply.github.com>
AuthorDate: Sat Jun 8 07:54:19 2024 -0700
Add rpc server to helm chart (#38549)
---
chart/templates/_helpers.yaml | 10 +
.../rpc-server/rpc-server-deployment.yaml | 259 ++
.../rpc-server/rpc-server-networkpolicy.yaml | 58 ++
.../rpc-server/rpc-server-poddisruptionbudget.yaml | 46 +
chart/templates/rpc-server/rpc-server-service.yaml | 59 ++
.../rpc-server/rpc-server-serviceaccount.yaml | 41 +
chart/values.schema.json | 727
chart/values.yaml | 104 +++
helm_tests/airflow_aux/test_basic_helm_chart.py| 105 ++-
helm_tests/airflow_core/test_rpc_server.py | 954 +
helm_tests/security/test_rbac.py | 48 +-
11 files changed, 2341 insertions(+), 70 deletions(-)
diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml
index 59a28fd9b7..aaa86ecbe8 100644
--- a/chart/templates/_helpers.yaml
+++ b/chart/templates/_helpers.yaml
@@ -582,6 +582,16 @@ server_tls_key_file = /etc/pgbouncer/server.key
{{- end }}
{{- end }}
+
+{{/* Create the name of the RPC server service account to use */}}
+{{- define "rpcServer.serviceAccountName" -}}
+ {{- if .Values._rpcServer.serviceAccount.create }}
+{{- default (printf "%s-rpc-server" (include "airflow.serviceAccountName"
.)) .Values._rpcServer.serviceAccount.name }}
+ {{- else }}
+{{- default "default" .Values._rpcServer.serviceAccount.name }}
+ {{- end }}
+{{- end }}
+
{{/* Create the name of the redis service account to use */}}
{{- define "redis.serviceAccountName" -}}
{{- if .Values.redis.serviceAccount.create }}
diff --git a/chart/templates/rpc-server/rpc-server-deployment.yaml
b/chart/templates/rpc-server/rpc-server-deployment.yaml
new file mode 100644
index 00..4a3766c585
--- /dev/null
+++ b/chart/templates/rpc-server/rpc-server-deployment.yaml
@@ -0,0 +1,259 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+
+## Airflow rpc-server Deployment
+#
+{{- if .Values._rpcServer.enabled }}
+{{- $nodeSelector := or .Values._rpcServer.nodeSelector .Values.nodeSelector }}
+{{- $affinity := or .Values._rpcServer.affinity .Values.affinity }}
+{{- $tolerations := or .Values._rpcServer.tolerations .Values.tolerations }}
+{{- $topologySpreadConstraints := or
.Values._rpcServer.topologySpreadConstraints .Values.topologySpreadConstraints
}}
+{{- $revisionHistoryLimit := or .Values._rpcServer.revisionHistoryLimit
.Values.revisionHistoryLimit }}
+{{- $securityContext := include "airflowPodSecurityContext" (list .
.Values._rpcServer) }}
+{{- $containerSecurityContext := include "containerSecurityContext" (list .
.Values._rpcServer) }}
+{{- $containerSecurityContextWaitForMigrations := include
"containerSecurityContext" (list . .Values._rpcServer.waitForMigrations) }}
+{{- $containerLifecycleHooks := or .Values._rpcServer.containerLifecycleHooks
.Values.containerLifecycleHooks }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "airflow.fullname" . }}-rpc-server
+ labels:
+tier: airflow
+component: rpc-server
+release: {{ .Release.Name }}
+chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+heritage: {{ .Release.Service }}
+{{- with .Values.labels }}
+ {{- toYaml . | nindent 4 }}
+{{- end }}
+ {{- if .Values._rpcServer.annotations }}
+ annotations: {{- toYaml .Values._rpcServer.annotations | nindent 4 }}
+ {{- end }}
+spec:
+ replicas: {{ .Values._rpcServer.replicas }}
+ {{- if $revisionHistoryLimit }}
+ revisionHistoryLi
(airflow) branch main updated: Remove warnings when uploading code coverage (#40139)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new e01877a162 Remove warnings when uploading code coverage (#40139)
e01877a162 is described below
commit e01877a1623948b7953b92d9dfd1df9e9e31df50
Author: Jarek Potiuk
AuthorDate: Sat Jun 8 16:49:05 2024 +0200
Remove warnings when uploading code coverage (#40139)
We are not running coverage for Python 3.12 due to slow coverage
for some libraries. Therefore uploading coverage generates
warning when attempting to run post-test-success for Python 3.12.
This change avoids attempting to upload coverage for Python 3.12
and adds extra env variable passed to docker with ENABLE_COVERAGE
status (not necessary but makes it easier to see variables when
running tests).
---
.github/actions/post_tests_success/action.yml| 12
.github/workflows/integration-tests.yml | 1 +
.github/workflows/run-unit-tests.yml | 1 +
dev/breeze/src/airflow_breeze/params/shell_params.py | 1 +
4 files changed, 11 insertions(+), 4 deletions(-)
diff --git a/.github/actions/post_tests_success/action.yml
b/.github/actions/post_tests_success/action.yml
index e8cdfa4602..37b51154d3 100644
--- a/.github/actions/post_tests_success/action.yml
+++ b/.github/actions/post_tests_success/action.yml
@@ -23,6 +23,10 @@ inputs:
description: 'Codecov token'
required: true
default: ''
+ python-version:
+description: 'Python version'
+required: true
+default: ''
runs:
using: "composite"
steps:
@@ -37,14 +41,14 @@ runs:
if: env.ENABLE_COVERAGE == 'true' && env.TEST_TYPES != 'Helm'
shell: bash
run: |
-mkdir ./files/coverage-reposts
-mv ./files/coverage*.xml ./files/coverage-reposts/ || true
+mkdir ./files/coverage-reports
+mv ./files/coverage*.xml ./files/coverage-reports/ || true
- name: "Upload all coverage reports to codecov"
uses: codecov/codecov-action@v4
env:
CODECOV_TOKEN: ${{ inputs.codecov-token }}
- if: env.ENABLE_COVERAGE == 'true' && env.TEST_TYPES != 'Helm'
+ if: env.ENABLE_COVERAGE == 'true' && env.TEST_TYPES != 'Helm' &&
inputs.python-version != '3.12'
with:
name: coverage-${{env.JOB_ID}}
flags:
python-${{env.PYTHON_MAJOR_MINOR_VERSION}},${{env.BACKEND}}-${{env.BACKEND_VERSION}}
-directory: "./files/coverage-reposts/"
+directory: "./files/coverage-reports/"
diff --git a/.github/workflows/integration-tests.yml
b/.github/workflows/integration-tests.yml
index e10447a01a..e831350f5b 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -96,6 +96,7 @@ jobs:
uses: ./.github/actions/post_tests_success
with:
codecov-token: ${{ secrets.CODECOV_TOKEN }}
+ python-version: ${{ inputs.default-python-version }}
- name: "Post Tests failure: Integration Tests ${{ matrix.integration }}"
uses: ./.github/actions/post_tests_failure
if: failure()
diff --git a/.github/workflows/run-unit-tests.yml
b/.github/workflows/run-unit-tests.yml
index ae108e7981..7828e50ed7 100644
--- a/.github/workflows/run-unit-tests.yml
+++ b/.github/workflows/run-unit-tests.yml
@@ -197,6 +197,7 @@ jobs:
uses: ./.github/actions/post_tests_success
with:
codecov-token: ${{ secrets.CODECOV_TOKEN }}
+ python-version: ${{ matrix.python-version }}
if: success()
- name: "Post Tests failure"
uses: ./.github/actions/post_tests_failure
diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py
b/dev/breeze/src/airflow_breeze/params/shell_params.py
index 99679ce5ad..f67aa99131 100644
--- a/dev/breeze/src/airflow_breeze/params/shell_params.py
+++ b/dev/breeze/src/airflow_breeze/params/shell_params.py
@@ -509,6 +509,7 @@ class ShellParams:
_set_var(_env, "DOWNGRADE_SQLALCHEMY", self.downgrade_sqlalchemy)
_set_var(_env, "DOWNGRADE_PENDULUM", self.downgrade_pendulum)
_set_var(_env, "ENABLED_SYSTEMS", None, "")
+_set_var(_env, "ENABLE_COVERAGE", self.enable_coverage)
_set_var(_env, "FLOWER_HOST_PORT", None, FLOWER_HOST_PORT)
_set_var(_env, "FORCE_LOWEST_DEPENDENCIES",
self.force_lowest_dependencies)
_set_var(_env, "SQLALCHEMY_WARN_20", self.force_sa_warnings)
(airflow) branch main updated: Add warning to deprecated api docs that access control isn't applied (#40129)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new 85be1867a6 Add warning to deprecated api docs that access control isn't applied (#40129) 85be1867a6 is described below commit 85be1867a690726b4556a58656597a0606ba09dc Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com> AuthorDate: Fri Jun 7 14:04:52 2024 -0600 Add warning to deprecated api docs that access control isn't applied (#40129) --- airflow/config_templates/config.yml | 4 ++-- docs/apache-airflow/deprecated-rest-api-ref.rst | 5 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index 1ad8c1b4cb..b1e2881907 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -1293,8 +1293,8 @@ api: options: enable_experimental_api: description: | -Enables the deprecated experimental API. Please note that these APIs do not have access control. -The authenticated user has full access. +Enables the deprecated experimental API. Please note that these API endpoints do not have +access control. An authenticated user has full access. .. warning:: diff --git a/docs/apache-airflow/deprecated-rest-api-ref.rst b/docs/apache-airflow/deprecated-rest-api-ref.rst index 276175f97f..cb13323885 100644 --- a/docs/apache-airflow/deprecated-rest-api-ref.rst +++ b/docs/apache-airflow/deprecated-rest-api-ref.rst @@ -23,6 +23,11 @@ Deprecated REST API This REST API is deprecated since version 2.0. Please consider using the :doc:`stable REST API `. For more information on migration, see `UPDATING.md <https://airflow.apache.org/docs/apache-airflow/stable/howto/upgrading-from-1-10/>`_ +.. warning:: + + Please note that these API endpoints do not have access control. An authenticated user has full access. + + Before Airflow 2.0 this REST API was known as the "experimental" API, but now that the :doc:`stable REST API ` is available, it has been renamed. The endpoints for this API are available at ``/api/experimental/``.
(airflow) branch main updated: Chart: enable mysql keda support for triggerer (#37365)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 44a5300594 Chart: enable mysql keda support for triggerer (#37365)
44a5300594 is described below
commit 44a53005948a0e96b94714e71ce4aa2b5f1f9d6b
Author: Forlot Romain
AuthorDate: Fri Jun 7 17:52:31 2024 +0200
Chart: enable mysql keda support for triggerer (#37365)
---
chart/templates/triggerer/triggerer-kedaautoscaler.yaml | 12
chart/templates/workers/worker-kedaautoscaler.yaml | 1 -
helm_tests/airflow_core/test_triggerer.py | 17 +
helm_tests/airflow_core/test_worker.py | 16
4 files changed, 45 insertions(+), 1 deletion(-)
diff --git a/chart/templates/triggerer/triggerer-kedaautoscaler.yaml
b/chart/templates/triggerer/triggerer-kedaautoscaler.yaml
index a83834a838..f719281c30 100644
--- a/chart/templates/triggerer/triggerer-kedaautoscaler.yaml
+++ b/chart/templates/triggerer/triggerer-kedaautoscaler.yaml
@@ -48,10 +48,22 @@ spec:
advanced: {{- toYaml .Values.triggerer.keda.advanced | nindent 4 }}
{{- end }}
triggers:
+{{- if eq .Values.data.metadataConnection.protocol "mysql" }}
+- type: "mysql"
+ metadata:
+queryValue: "1"
+connectionStringFromEnv: KEDA_DB_CONN
+query: {{ tpl .Values.triggerer.keda.query . | quote }}
+{{- else }}
- type: postgresql
metadata:
targetQueryValue: "1"
+{{- if and .Values.pgbouncer.enabled (not
.Values.triggerer.keda.usePgbouncer) }}
+connectionFromEnv: KEDA_DB_CONN
+{{- else }}
connectionFromEnv: AIRFLOW_CONN_AIRFLOW_DB
+{{- end }}
query: {{ tpl .Values.triggerer.keda.query . | quote }}
+ {{- end }}
{{- end }}
{{- end }}
diff --git a/chart/templates/workers/worker-kedaautoscaler.yaml
b/chart/templates/workers/worker-kedaautoscaler.yaml
index 95adb3e126..74af09b174 100644
--- a/chart/templates/workers/worker-kedaautoscaler.yaml
+++ b/chart/templates/workers/worker-kedaautoscaler.yaml
@@ -64,5 +64,4 @@ spec:
{{- end }}
query: {{ tpl .Values.workers.keda.query . | quote }}
{{- end }}
-
{{- end }}
diff --git a/helm_tests/airflow_core/test_triggerer.py
b/helm_tests/airflow_core/test_triggerer.py
index c92683e80d..a684e0f334 100644
--- a/helm_tests/airflow_core/test_triggerer.py
+++ b/helm_tests/airflow_core/test_triggerer.py
@@ -738,3 +738,20 @@ class TestTriggererKedaAutoScaler:
show_only=["templates/triggerer/triggerer-kedaautoscaler.yaml"],
)
assert expected_query ==
jmespath.search("spec.triggers[0].metadata.query", docs[0])
+
+def test_mysql_db_backend_keda(self):
+docs = render_chart(
+values={
+"data": {"metadataConnection": {"protocol": "mysql"}},
+"triggerer": {
+"enabled": True,
+"keda": {"enabled": True},
+},
+},
+show_only=["templates/triggerer/triggerer-kedaautoscaler.yaml"],
+)
+assert "1" == jmespath.search("spec.triggers[0].metadata.queryValue",
docs[0])
+assert jmespath.search("spec.triggers[0].metadata.targetQueryValue",
docs[0]) is None
+
+assert "KEDA_DB_CONN" ==
jmespath.search("spec.triggers[0].metadata.connectionStringFromEnv", docs[0])
+assert jmespath.search("spec.triggers[0].metadata.connectionFromEnv",
docs[0]) is None
diff --git a/helm_tests/airflow_core/test_worker.py
b/helm_tests/airflow_core/test_worker.py
index 100d53c28c..bbfea0be68 100644
--- a/helm_tests/airflow_core/test_worker.py
+++ b/helm_tests/airflow_core/test_worker.py
@@ -1030,6 +1030,22 @@ class TestWorkerKedaAutoScaler:
)
assert expected_query ==
jmespath.search("spec.triggers[0].metadata.query", docs[0])
+def test_mysql_db_backend_keda(self):
+docs = render_chart(
+values={
+"data": {"metadataConnection": {"protocol": "mysql"}},
+"workers": {
+"keda": {"enabled": True},
+},
+},
+show_only=["templates/workers/worker-kedaautoscaler.yaml"],
+)
+assert "1" == jmespath.search("spec.triggers[0].metadata.queryValue",
docs[0])
+assert jmespath.search("spec.triggers[0].metadata.targetQueryValue",
docs[0]) is None
+
+assert "KEDA_DB_CONN" ==
jmespath.search("spec.triggers[0].metadata.connectionStringFromEnv", docs[0])
+assert jmespath.search("spec.triggers[0].metadata.connectionFromEnv",
docs[0]) is None
+
class TestWorkerHPAAutoScaler:
"""Tests worker HPA auto scaler."""
(airflow) branch main updated: Fix param order in PythonOperator docstring (#40122)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new b49db97b8e Fix param order in PythonOperator docstring (#40122) b49db97b8e is described below commit b49db97b8e2fd5fbeaef469cec2a1343e21a411b Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com> AuthorDate: Fri Jun 7 09:35:14 2024 -0600 Fix param order in PythonOperator docstring (#40122) --- airflow/operators/python.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/operators/python.py b/airflow/operators/python.py index fe9923529a..2faeea2100 100644 --- a/airflow/operators/python.py +++ b/airflow/operators/python.py @@ -173,10 +173,10 @@ class PythonOperator(BaseOperator): :param python_callable: A reference to an object that is callable -:param op_kwargs: a dictionary of keyword arguments that will get unpacked -in your function :param op_args: a list of positional arguments that will get unpacked when calling your callable +:param op_kwargs: a dictionary of keyword arguments that will get unpacked +in your function :param templates_dict: a dictionary where the values are templates that will get templated by the Airflow engine sometime between ``__init__`` and ``execute`` takes place and are made available
(airflow) branch main updated: Exit instead of return in case of Providers[fab] test is ignored (#40089)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 0568e9a604 Exit instead of return in case of Providers[fab] test is
ignored (#40089)
0568e9a604 is described below
commit 0568e9a604a12f29e79319f12ee6d86675eb4a0f
Author: Jarek Potiuk
AuthorDate: Thu Jun 6 18:26:39 2024 +0200
Exit instead of return in case of Providers[fab] test is ignored (#40089)
The #40059 added an exception for fab tests not to run on old
airflow releases but it was wrong doing return instead of exit
---
Dockerfile.ci | 2 +-
scripts/docker/entrypoint_ci.sh | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 873ec62f25..c2ebddc2f1 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -989,7 +989,7 @@ function determine_airflow_to_use() {
echo
echo "${COLOR_YELLOW}Skipping FAB tests on Airflow 2.7 and 2.8
because of FAB incompatibility with them${COLOR_RESET}"
echo
-return
+exit 0
fi
python "${IN_CONTAINER_DIR}/install_airflow_and_providers.py"
# Some packages might leave legacy typing module which causes test
issues
diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh
index 6b3130adba..9847cfdbb1 100755
--- a/scripts/docker/entrypoint_ci.sh
+++ b/scripts/docker/entrypoint_ci.sh
@@ -208,7 +208,7 @@ function determine_airflow_to_use() {
echo
echo "${COLOR_YELLOW}Skipping FAB tests on Airflow 2.7 and 2.8
because of FAB incompatibility with them${COLOR_RESET}"
echo
-return
+exit 0
fi
python "${IN_CONTAINER_DIR}/install_airflow_and_providers.py"
# Some packages might leave legacy typing module which causes test
issues
(airflow) branch main updated (f28ee5a694 -> 45bf7b9721)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from f28ee5a694 Resolve core operators deprecations in tests (#39992) add 45bf7b9721 Resolve dagbag deprecations in tests (#39989) No new revisions were added by this update. Summary of changes: tests/deprecations_ignore.yml | 45 --- tests/models/test_dag.py | 267 -- tests/models/test_dagbag.py | 8 +- 3 files changed, 210 insertions(+), 110 deletions(-)
(airflow) branch main updated (16e0182025 -> f28ee5a694)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 16e0182025 Docs: Simpler command to check local scheduler is alive (#40074) add f28ee5a694 Resolve core operators deprecations in tests (#39992) No new revisions were added by this update. Summary of changes: tests/deprecations_ignore.yml| 14 -- tests/operators/test_bash.py | 4 +++- tests/operators/test_branch_operator.py | 4 tests/operators/test_datetime.py | 7 ++- tests/operators/test_latest_only_operator.py | 6 +- 5 files changed, 18 insertions(+), 17 deletions(-)
(airflow) branch main updated (fcb9f87c55 -> 16e0182025)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from fcb9f87c55 fix typo and path to licences folder (#40067) add 16e0182025 Docs: Simpler command to check local scheduler is alive (#40074) No new revisions were added by this update. Summary of changes: .../administration-and-deployment/logging-monitoring/check-health.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(airflow) branch main updated (4dea367047 -> fcb9f87c55)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 4dea367047 Add unit test to cover back compat case in celery (#40035) add fcb9f87c55 fix typo and path to licences folder (#40067) No new revisions were added by this update. Summary of changes: LICENSE| 8 +++ airflow/_vendor/README.md | 7 +++--- dev/breeze/doc/images/output_build-docs.svg| 28 +++--- dev/breeze/doc/images/output_build-docs.txt| 2 +- ...elease-management_prepare-provider-packages.svg | 26 ++-- ...elease-management_prepare-provider-packages.txt | 2 +- .../output_release-management_publish-docs.svg | 28 +++--- .../output_release-management_publish-docs.txt | 2 +- .../airflow_breeze/commands/developer_commands.py | 2 +- .../commands/release_management_commands.py| 4 ++-- 10 files changed, 54 insertions(+), 55 deletions(-)
(airflow) branch main updated (ac2a1ef81d -> ef1c9a2a2b)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from ac2a1ef81d Expllicitly skip API tests for compatiblity tests on 2.7 and 2.8 (#40059) add ef1c9a2a2b Chart: Allow AWS Executors (#38524) No new revisions were added by this update. Summary of changes: chart/README.md | 3 +++ chart/values.schema.json| 4 +++- helm_tests/airflow_aux/test_basic_helm_chart.py | 24 +++- 3 files changed, 29 insertions(+), 2 deletions(-)
(airflow) branch main updated (9bcf665570 -> ac2a1ef81d)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 9bcf665570 Fix typos in overview.rst (#40061) add ac2a1ef81d Expllicitly skip API tests for compatiblity tests on 2.7 and 2.8 (#40059) No new revisions were added by this update. Summary of changes: Dockerfile.ci | 6 ++ scripts/docker/entrypoint_ci.sh | 6 ++ 2 files changed, 12 insertions(+)
(airflow) branch main updated: Fix typos in overview.rst (#40061)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new 9bcf665570 Fix typos in overview.rst (#40061) 9bcf665570 is described below commit 9bcf665570f3b87eeb5932500c1ae10391ef067d Author: bangjiehan AuthorDate: Wed Jun 5 23:10:02 2024 +0800 Fix typos in overview.rst (#40061) --- docs/apache-airflow/core-concepts/overview.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/apache-airflow/core-concepts/overview.rst b/docs/apache-airflow/core-concepts/overview.rst index 1b1eb8d10f..767b7e8990 100644 --- a/docs/apache-airflow/core-concepts/overview.rst +++ b/docs/apache-airflow/core-concepts/overview.rst @@ -52,7 +52,7 @@ A minimal Airflow installation consists of the following components: * A *webserver*, which presents a handy user interface to inspect, trigger and debug the behaviour of DAGs and tasks. -* A folder of *DAG files* is read by the *scheduler* to figure out what tasks to run and when and to +* A folder of *DAG files*, which is read by the *scheduler* to figure out what tasks to run and when to run them. * A *metadata database*, which airflow components use to store state of workflows and tasks. @@ -163,7 +163,7 @@ DAGs and tasks, but cannot author DAGs. The *DAG files* need to be synchronized between all the components that use them - *scheduler*, *triggerer* and *workers*. The *DAG files* can be synchronized by various mechanisms - typical -ways how DAGs can be synchronized are described in :doc:`helm-chart:manage-dags-files` ot our +ways how DAGs can be synchronized are described in :doc:`helm-chart:manage-dags-files` of our Helm Chart documentation. Helm chart is one of the ways how to deploy Airflow in K8S cluster. .. image:: ../img/diagram_distributed_airflow_architecture.png
(airflow) branch main updated (8a32b940ce -> 8b99ad0fbe)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 8a32b940ce Bedrock system test adjustment (#40032) add 8b99ad0fbe Document `extraContainers` and `extraInitContainers` that are templated (#40033) No new revisions were added by this update. Summary of changes: chart/values.schema.json | 20 ++-- chart/values.yaml| 20 ++-- 2 files changed, 20 insertions(+), 20 deletions(-)
(airflow) branch main updated: Warn on mini scheduler failures instead of debug (#39760)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new 38e003c2c9 Warn on mini scheduler failures instead of debug (#39760) 38e003c2c9 is described below commit 38e003c2c9af48499971c694730d47b8973be1d6 Author: Ryan Hatter <25823361+rnh...@users.noreply.github.com> AuthorDate: Sun Jun 2 09:30:18 2024 -0400 Warn on mini scheduler failures instead of debug (#39760) Mini scheduler issues are rare and hard to debug. Chances are the problem won't be reproducible after enabling debug logging. --- airflow/models/taskinstance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index 39af664631..1637c06e58 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -3703,7 +3703,7 @@ class TaskInstance(Base, LoggingMixin): except OperationalError as e: # Any kind of DB error here is _non fatal_ as this block is just an optimisation. -cls.logger().debug( +cls.logger().warning( "Skipping mini scheduling run due to exception: %s", e.statement, exc_info=True,
(airflow) branch main updated (482c0be434 -> 1734b15e4e)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 482c0be434 breeze k8s version upgrade (#39785) add 1734b15e4e Add typing for SqlSensor (#39773) No new revisions were added by this update. Summary of changes: airflow/providers/common/sql/sensors/sql.py| 40 +++ airflow/providers/common/sql/sensors/sql.pyi | 17 +- tests/providers/common/sql/sensors/test_sql.py | 44 +- 3 files changed, 51 insertions(+), 50 deletions(-)
(airflow) branch main updated (4d525aa32d -> f509b0a924)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 4d525aa32d Determine needs_expansion at time of serialization (#39604) add f509b0a924 More typing in TimeSensor and TimeSensorAsync (#39696) No new revisions were added by this update. Summary of changes: airflow/sensors/time_sensor.py | 8 1 file changed, 4 insertions(+), 4 deletions(-)
(airflow) branch main updated: Fix spelling satiny -> sanity (#39594)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new 4a59276187 Fix spelling satiny -> sanity (#39594) 4a59276187 is described below commit 4a5927618781bf53a17c4dc57a6951c3f0e428fb Author: Daniel Standish <15932138+dstand...@users.noreply.github.com> AuthorDate: Mon May 13 15:13:05 2024 -0700 Fix spelling satiny -> sanity (#39594) --- tests/serialization/test_dag_serialization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/serialization/test_dag_serialization.py b/tests/serialization/test_dag_serialization.py index 623b2e6731..62a79faeaa 100644 --- a/tests/serialization/test_dag_serialization.py +++ b/tests/serialization/test_dag_serialization.py @@ -697,7 +697,7 @@ class TestStringifiedDAGs: if isinstance(task, MappedOperator): # MappedOperator.operator_class holds a backup of the serialized # data; checking its entirety basically duplicates this validation -# function, so we just do some satiny checks. +# function, so we just do some sanity checks. serialized_task.operator_class["_task_type"] == type(task).__name__ if isinstance(serialized_task.operator_class, DecoratedOperator): serialized_task.operator_class["_operator_name"] == task._operator_name
(airflow) branch main updated: Remove compat code for 2.7.0 - its now the min Airflow version (#39591)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new e3897dcbed Remove compat code for 2.7.0 - its now the min Airflow
version (#39591)
e3897dcbed is described below
commit e3897dcbed0262b0cab7a357f8d7fbbb6c4f4eeb
Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com>
AuthorDate: Mon May 13 16:01:44 2024 -0600
Remove compat code for 2.7.0 - its now the min Airflow version (#39591)
---
.../providers/celery/executors/celery_executor.py | 44 ++---
.../kubernetes/executors/kubernetes_executor.py| 46 +++---
.../executors/kubernetes_executor_utils.py | 19 +
3 files changed, 29 insertions(+), 80 deletions(-)
diff --git a/airflow/providers/celery/executors/celery_executor.py
b/airflow/providers/celery/executors/celery_executor.py
index d81e57a222..9f1948db52 100644
--- a/airflow/providers/celery/executors/celery_executor.py
+++ b/airflow/providers/celery/executors/celery_executor.py
@@ -37,37 +37,19 @@ from celery import states as celery_states
from packaging.version import Version
from airflow import __version__ as airflow_version
-
-try:
-from airflow.cli.cli_config import (
-ARG_DAEMON,
-ARG_LOG_FILE,
-ARG_PID,
-ARG_SKIP_SERVE_LOGS,
-ARG_STDERR,
-ARG_STDOUT,
-ARG_VERBOSE,
-ActionCommand,
-Arg,
-GroupCommand,
-lazy_load_command,
-)
-except ImportError:
-import packaging.version
-
-from airflow.exceptions import AirflowOptionalProviderFeatureException
-
-base_version = packaging.version.parse(airflow_version).base_version
-
-if packaging.version.parse(base_version) <
packaging.version.parse("2.7.0"):
-raise AirflowOptionalProviderFeatureException(
-"Celery Executor from Celery Provider should only be used with
Airflow 2.7.0+.\n"
-f"This is Airflow {airflow_version} and Celery and
CeleryKubernetesExecutor are "
-f"available in the 'airflow.executors' package. You should not use
"
-f"the provider's executors in this version of Airflow."
-)
-raise
-
+from airflow.cli.cli_config import (
+ARG_DAEMON,
+ARG_LOG_FILE,
+ARG_PID,
+ARG_SKIP_SERVE_LOGS,
+ARG_STDERR,
+ARG_STDOUT,
+ARG_VERBOSE,
+ActionCommand,
+Arg,
+GroupCommand,
+lazy_load_command,
+)
from airflow.configuration import conf
from airflow.exceptions import AirflowTaskTimeout
from airflow.executors.base_executor import BaseExecutor
diff --git a/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py
b/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py
index f62f021fd2..43cefeb9c4 100644
--- a/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py
+++ b/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py
@@ -38,38 +38,18 @@ from typing import TYPE_CHECKING, Any, Sequence
from kubernetes.dynamic import DynamicClient
from sqlalchemy import select, update
-from airflow.providers.cncf.kubernetes.pod_generator import
PodMutationHookException, PodReconciliationError
-from airflow.stats import Stats
-
-try:
-from airflow.cli.cli_config import (
-ARG_DAG_ID,
-ARG_EXECUTION_DATE,
-ARG_OUTPUT_PATH,
-ARG_SUBDIR,
-ARG_VERBOSE,
-ActionCommand,
-Arg,
-GroupCommand,
-lazy_load_command,
-positive_int,
-)
-except ImportError:
-import packaging.version
-
-from airflow import __version__ as airflow_version
-from airflow.exceptions import AirflowOptionalProviderFeatureException
-
-base_version = packaging.version.parse(airflow_version).base_version
-
-if packaging.version.parse(base_version) <
packaging.version.parse("2.7.0"):
-raise AirflowOptionalProviderFeatureException(
-"Kubernetes Executor from CNCF Provider should only be used with
Airflow 2.7.0+.\n"
-f"This is Airflow {airflow_version} and Kubernetes and
CeleryKubernetesExecutor are "
-f"available in the 'airflow.executors' package. You should not use
"
-f"the provider's executors in this version of Airflow."
-)
-raise
+from airflow.cli.cli_config import (
+ARG_DAG_ID,
+ARG_EXECUTION_DATE,
+ARG_OUTPUT_PATH,
+ARG_SUBDIR,
+ARG_VERBOSE,
+ActionCommand,
+Arg,
+GroupCommand,
+lazy_load_command,
+positive_int,
+)
from airflow.configuration import conf
from airflow.executors.base_executor import BaseExecutor
from airflow.providers.cncf.kubernetes.executors.kubernetes_executor_types
impor
(airflow) branch main updated: Faster `airflow_version` imports (#39552)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new 2c05187b07 Faster `airflow_version` imports (#39552) 2c05187b07 is described below commit 2c05187b07baf7c41a32b18fabdbb3833acc08eb Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com> AuthorDate: Fri May 10 16:54:54 2024 -0600 Faster `airflow_version` imports (#39552) Turns out `importlib.metadata.version` is much slower than a simple import, so let's switch back. It's not "slow", but might as well be faster :) --- airflow/providers/airbyte/__init__.py | 6 ++ airflow/providers/alibaba/__init__.py | 6 ++ airflow/providers/amazon/__init__.py | 6 ++ airflow/providers/apache/beam/__init__.py | 6 ++ airflow/providers/apache/cassandra/__init__.py | 6 ++ airflow/providers/apache/drill/__init__.py | 6 ++ airflow/providers/apache/druid/__init__.py | 6 ++ airflow/providers/apache/flink/__init__.py | 6 ++ airflow/providers/apache/hdfs/__init__.py | 6 ++ airflow/providers/apache/hive/__init__.py | 6 ++ airflow/providers/apache/impala/__init__.py| 6 ++ airflow/providers/apache/kafka/__init__.py | 6 ++ airflow/providers/apache/kylin/__init__.py | 6 ++ airflow/providers/apache/livy/__init__.py | 6 ++ airflow/providers/apache/pig/__init__.py | 6 ++ airflow/providers/apache/pinot/__init__.py | 6 ++ airflow/providers/apache/spark/__init__.py | 6 ++ airflow/providers/apprise/__init__.py | 6 ++ airflow/providers/arangodb/__init__.py | 6 ++ airflow/providers/asana/__init__.py| 6 ++ airflow/providers/atlassian/jira/__init__.py | 6 ++ airflow/providers/celery/__init__.py | 6 ++ airflow/providers/celery/executors/__init__.py | 4 +--- airflow/providers/celery/executors/celery_executor.py | 10 +++--- airflow/providers/cloudant/__init__.py | 6 ++ airflow/providers/cncf/kubernetes/__init__.py | 6 ++ .../providers/cncf/kubernetes/executors/kubernetes_executor.py | 4 +--- airflow/providers/cohere/__init__.py | 6 ++ airflow/providers/common/io/__init__.py| 6 ++ airflow/providers/common/io/xcom/__init__.py | 5 + airflow/providers/common/sql/__init__.py | 6 ++ airflow/providers/databricks/__init__.py | 6 ++ airflow/providers/datadog/__init__.py | 6 ++ airflow/providers/dbt/cloud/__init__.py| 6 ++ airflow/providers/dingding/__init__.py | 6 ++ airflow/providers/discord/__init__.py | 6 ++ airflow/providers/docker/__init__.py | 6 ++ airflow/providers/elasticsearch/__init__.py| 6 ++ airflow/providers/exasol/__init__.py | 6 ++ airflow/providers/fab/__init__.py | 6 ++ airflow/providers/facebook/__init__.py | 6 ++ airflow/providers/ftp/__init__.py | 6 ++ airflow/providers/github/__init__.py | 6 ++ airflow/providers/google/__init__.py | 6 ++ airflow/providers/grpc/__init__.py | 6 ++ airflow/providers/hashicorp/__init__.py| 6 ++ airflow/providers/http/__init__.py | 6 ++ airflow/providers/imap/__init__.py | 6 ++ airflow/providers/influxdb/__init__.py | 6 ++ airflow/providers/jdbc/__init__.py | 6 ++ airflow/providers/jenkins/__init__.py | 6 ++ airflow/providers/microsoft/azure/__init__.py | 6 ++ airflow/providers/microsoft/mssql/__init__.py | 6 ++ airflow/providers/microsoft/psrp/__init__.py | 6 ++ airflow/providers/microsoft/winrm/__init__.py | 6 ++ airflow/providers/mongo/__init__.py| 6 ++ airflow
(airflow) branch main updated: Drop `xmlsec` pin (#39534)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new c9f4618be4 Drop `xmlsec` pin (#39534) c9f4618be4 is described below commit c9f4618be4a88d8287cf579230380dda8edaeafd Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com> AuthorDate: Thu May 9 17:35:30 2024 -0600 Drop `xmlsec` pin (#39534) --- airflow/providers/amazon/provider.yaml | 4 generated/provider_dependencies.json | 3 +-- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index 97fdb9fd8f..44cad7e104 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -99,10 +99,6 @@ dependencies: - sqlalchemy_redshift>=0.8.6 - asgiref - PyAthena>=3.0.10 - # XML sec 1.3.14 breaks Amazon's authentication with `lxml & xmlsec libxml2 library version mismatch` - # We should investigate if we can upgrade to a newer version of lxml and xmlsec - # Tracked in https://github.com/apache/airflow/issues/39103 - - xmlsec<1.3.14 - jmespath additional-extras: diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 113fb78219..c1d3842142 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -37,8 +37,7 @@ "jsonpath_ng>=1.5.3", "redshift_connector>=2.0.918", "sqlalchemy_redshift>=0.8.6", - "watchtower>=2.0.1,<4", - "xmlsec<1.3.14" + "watchtower>=2.0.1,<4" ], "devel-deps": [ "aiobotocore>=2.7.0",
(airflow) branch main updated (e794d6cce4 -> 73918925ed)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from e794d6cce4 Add pre-commit to correct mismatching revision IDs in migration file (#39480) add 73918925ed Simplify `airflow_version` imports (#39497) No new revisions were added by this update. Summary of changes: airflow/providers/airbyte/__init__.py | 7 +++ airflow/providers/alibaba/__init__.py | 7 +++ airflow/providers/amazon/__init__.py | 7 +++ airflow/providers/apache/beam/__init__.py | 7 +++ airflow/providers/apache/cassandra/__init__.py| 7 +++ airflow/providers/apache/drill/__init__.py| 7 +++ airflow/providers/apache/druid/__init__.py| 7 +++ airflow/providers/apache/flink/__init__.py| 7 +++ airflow/providers/apache/hdfs/__init__.py | 7 +++ airflow/providers/apache/hive/__init__.py | 7 +++ airflow/providers/apache/impala/__init__.py | 7 +++ airflow/providers/apache/kafka/__init__.py| 7 +++ airflow/providers/apache/kylin/__init__.py| 7 +++ airflow/providers/apache/livy/__init__.py | 7 +++ airflow/providers/apache/pig/__init__.py | 7 +++ airflow/providers/apache/pinot/__init__.py| 7 +++ airflow/providers/apache/spark/__init__.py| 7 +++ airflow/providers/apprise/__init__.py | 7 +++ airflow/providers/arangodb/__init__.py| 7 +++ airflow/providers/asana/__init__.py | 7 +++ airflow/providers/atlassian/jira/__init__.py | 7 +++ airflow/providers/celery/__init__.py | 7 +++ airflow/providers/celery/executors/__init__.py| 8 +++- airflow/providers/celery/executors/celery_executor.py | 6 ++ airflow/providers/cloudant/__init__.py| 7 +++ airflow/providers/cncf/kubernetes/__init__.py | 7 +++ .../providers/cncf/kubernetes/executors/kubernetes_executor.py| 6 ++ airflow/providers/cohere/__init__.py | 7 +++ airflow/providers/common/io/__init__.py | 7 +++ airflow/providers/common/io/xcom/__init__.py | 7 +++ airflow/providers/common/sql/__init__.py | 7 +++ airflow/providers/databricks/__init__.py | 7 +++ airflow/providers/datadog/__init__.py | 7 +++ airflow/providers/dbt/cloud/__init__.py | 7 +++ airflow/providers/dingding/__init__.py| 7 +++ airflow/providers/discord/__init__.py | 7 +++ airflow/providers/docker/__init__.py | 7 +++ airflow/providers/elasticsearch/__init__.py | 7 +++ airflow/providers/exasol/__init__.py | 7 +++ airflow/providers/fab/__init__.py | 7 +++ airflow/providers/facebook/__init__.py| 7 +++ airflow/providers/ftp/__init__.py | 7 +++ airflow/providers/github/__init__.py | 7 +++ airflow/providers/google/__init__.py | 7 +++ airflow/providers/grpc/__init__.py| 7 +++ airflow/providers/hashicorp/__init__.py | 7 +++ airflow/providers/http/__init__.py| 7 +++ airflow/providers/imap/__init__.py| 7 +++ airflow/providers/influxdb/__init__.py| 7 +++ airflow/providers/jdbc/__init__.py| 7 +++ airflow/providers/jenkins/__init__.py | 7 +++ airflow/providers/microsoft/azure/__init__.py | 7 +++ airflow/providers/microsoft/mssql/__init__.py | 7 +++ airflow/providers/microsoft/psrp/__init__.py | 7 +++ airflow/providers/microsoft/winrm/__init__.py | 7 +++ airflow/providers/mongo/__init__.py | 7 +++ airflow/providers/mysql/__init__.py | 7 +++ airflow/providers/neo4j/__init__.py | 7 +++ airflow
(airflow) branch main updated: Fix typo in deferrable docs (#39494)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new 58509f52a3 Fix typo in deferrable docs (#39494) 58509f52a3 is described below commit 58509f52a3f41a564b068867290767993c929031 Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com> AuthorDate: Wed May 8 10:22:26 2024 -0600 Fix typo in deferrable docs (#39494) --- docs/apache-airflow/authoring-and-scheduling/deferring.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-airflow/authoring-and-scheduling/deferring.rst b/docs/apache-airflow/authoring-and-scheduling/deferring.rst index 752ddc5f25..084a08f0ac 100644 --- a/docs/apache-airflow/authoring-and-scheduling/deferring.rst +++ b/docs/apache-airflow/authoring-and-scheduling/deferring.rst @@ -143,7 +143,7 @@ The ``self.defer`` call raises the ``TaskDeferred`` exception, so it can work an Triggering Deferral from Start ~~~ -If you want to defer your task directly to the triggerer without going into the worker, you can add the class level attributes ``start_trigger`` and ``_next_method`` to your deferrable operator. +If you want to defer your task directly to the triggerer without going into the worker, you can add the class level attributes ``start_trigger`` and ``next_method`` to your deferrable operator. * ``start_trigger``: An instance of a trigger you want to defer to. It will be serialized into the database. * ``next_method``: The method name on your operator that you want Airflow to call when it resumes.
(airflow) branch main updated (a65a837187 -> 0c159bfa90)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from a65a837187 Increase volume size for ARM instance (#39472) add 0c159bfa90 Remove stale comment about TI index (#39470) No new revisions were added by this update. Summary of changes: airflow/models/taskinstance.py | 8 1 file changed, 8 deletions(-)
(airflow) branch main updated: Minor fixup for custom FAB permission consistency warning (#39469)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new 4a1d040973 Minor fixup for custom FAB permission consistency warning (#39469) 4a1d040973 is described below commit 4a1d04097348d73cc3399e86c3b44a21b098bead Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com> AuthorDate: Tue May 7 16:00:25 2024 -0400 Minor fixup for custom FAB permission consistency warning (#39469) --- .../auth-manager/access-control.rst | 20 ++-- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/apache-airflow-providers-fab/auth-manager/access-control.rst b/docs/apache-airflow-providers-fab/auth-manager/access-control.rst index d7cca5c985..4d5637d40d 100644 --- a/docs/apache-airflow-providers-fab/auth-manager/access-control.rst +++ b/docs/apache-airflow-providers-fab/auth-manager/access-control.rst @@ -110,16 +110,16 @@ Permissions .. warning:: - Airflow allows to define custom Roles with fine-grained RBAC permissions for the users. However, - there is no mechanism to make sure that the set of permissions assigned is fully consistent and not all - combinations of permissions are fully consistent. There are a number of cases where permissions for - particular resources are overlapping. Good examples are menu access permissions - lack of menu access - does not automatically disable access to functionality they are pointing at. Another examples is access to - Role views - which allows to access User information even if the user does not have "user view" access. - It is simply inconsistent to add access to Roles where you have no access to users. - - When you decide to use custom set of resource-based permissions, the Deployment Manager should carefully - review if the final set of permissions granted to the roles is what they expect. + Airflow allows you to define custom Roles with fine-grained RBAC permissions for users. However, not all + combinations of permissions are fully consistent, and there is no mechanism to make sure that the set of + permissions assigned is fully consistent. There are a number of cases where permissions for + particular resources are overlapping. A good example is menu access permissions - a lack of menu access + does not automatically disable access to the functionality the menu is pointing at. Another example is access + to the Role view, which allows access to User information even if the user does not have "user view" access. + It is simply inconsistent to add access to Roles when you have no access to users. + + When you decide to use a custom set of resource-based permissions, the Deployment Manager should carefully + review if the final set of permissions granted to roles is what they expect. Resource-Based permissions
(airflow) branch main updated (a74b5f0694 -> 9a0d9c570b)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from a74b5f0694 ECS Executor: Set tasks to RUNNING state once active (#39212) add 9a0d9c570b Simpler error message when trying to offline migrate with sqlite (#39441) No new revisions were added by this update. Summary of changes: airflow/utils/db.py| 2 +- tests/utils/test_db.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-)
(airflow) branch main updated (495814e908 -> b9fc1694cd)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 495814e908 Update python client version in README (#39384) add b9fc1694cd Update python client release doc (#39383) No new revisions were added by this update. Summary of changes: dev/README_RELEASE_PYTHON_CLIENT.md | 53 - 1 file changed, 46 insertions(+), 7 deletions(-)
(airflow-client-python) 01/01: Update README for 2.9.0
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch fix_readme
in repository https://gitbox.apache.org/repos/asf/airflow-client-python.git
commit 4e2362a85d46a50206cec382891acfe2695bcb78
Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com>
AuthorDate: Thu May 2 16:09:52 2024 -0600
Update README for 2.9.0
These were missed during the initial release
---
README.md| 5 +-
airflow_client/README.md | 166 ---
2 files changed, 116 insertions(+), 55 deletions(-)
diff --git a/README.md b/README.md
index dc8fd40..f0964d5 100644
--- a/README.md
+++ b/README.md
@@ -240,9 +240,10 @@ fulfilling the request.
This Python package is automatically generated by the [OpenAPI
Generator](https://openapi-generator.tech) project:
-- API version: 2.8.0
-- Package version: 2.8.0
+- API version: 2.9.0
+- Package version: 2.9.0
- Build package: org.openapitools.codegen.languages.PythonClientCodegen
+
For more information, please visit
[https://airflow.apache.org](https://airflow.apache.org)
## Requirements.
diff --git a/airflow_client/README.md b/airflow_client/README.md
index 3ff2491..f0964d5 100644
--- a/airflow_client/README.md
+++ b/airflow_client/README.md
@@ -18,6 +18,7 @@
-->
# Apache Airflow Python Client
+
# Overview
To facilitate management, Apache Airflow supports a range of REST API
endpoints across its
@@ -26,6 +27,7 @@ This section provides an overview of the API design, methods,
and supported use
Most of the endpoints accept `JSON` as input and return `JSON` responses.
This means that you must usually add the following headers to your request:
+
```
Content-type: application/json
Accept: application/json
@@ -41,7 +43,7 @@ Resource names are used as part of endpoint URLs, as well as
in API parameters a
## CRUD Operations
-The platform supports **C**reate, **R**ead, **U**pdate, and **D**elete
operations on most resources.
+The platform supports **Create**, **Read**, **Update**, and **Delete**
operations on most resources.
You can review the standards for these operations and their standard
parameters below.
Some endpoints have special behavior as exceptions.
@@ -66,6 +68,7 @@ The response usually returns a `200 OK` response code upon
success, with an obje
of resources' metadata in the response body.
When reading resources, some common query parameters are usually available.
e.g.:
+
```
v1/connections?limit=25=25
```
@@ -84,7 +87,7 @@ resource in the response body.
### Delete
-Deleting a resource requires the resource `id` and is typically executed via
an HTTP `DELETE` request.
+Deleting a resource requires the resource `id` and is typically executing via
an HTTP `DELETE` request.
The response usually returns a `204 No Content` response code upon success.
## Conventions
@@ -93,16 +96,15 @@ The response usually returns a `204 No Content` response
code upon success.
- Names are consistent between URL parameter name and field name.
- Field names are in snake_case.
+
```json
{
-\"description\": \"string\",
\"name\": \"string\",
+\"slots\": 0,
\"occupied_slots\": 0,
-\"open_slots\": 0
+\"used_slots\": 0,
\"queued_slots\": 0,
-\"running_slots\": 0,
-\"scheduled_slots\": 0,
-\"slots\": 0,
+\"open_slots\": 0
}
```
@@ -115,10 +117,13 @@ The update request ignores any fields that aren't
specified in the field mask, l
their current values.
Example:
-```
- resource = request.get('/resource/my-id').json()
- resource['my_field'] = 'new-value'
- request.patch('/resource/my-id?update_mask=my_field',
data=json.dumps(resource))
+
+```python
+import requests
+
+resource = requests.get("/resource/my-id").json()
+resource["my_field"] = "new-value"
+requests.patch("/resource/my-id?update_mask=my_field",
data=json.dumps(resource))
```
## Versioning and Endpoint Lifecycle
@@ -136,6 +141,7 @@ the Apache Airflow API.
Note that you will need to pass credentials data.
For e.g., here is how to pause a DAG with [curl](https://curl.haxx.se/), when
basic authorization is used:
+
```bash
curl -X PATCH 'https://example.com/api/v1/dags/{dag_id}?update_mask=is_paused'
\\
-H 'Content-Type: application/json' \\
@@ -148,8 +154,9 @@ curl -X PATCH
'https://example.com/api/v1/dags/{dag_id}?update_mask=is_paused' \
Using a graphical tool such as [Postman](https://www.postman.com/) or
[Insomnia](https://insomnia.rest/),
it is possible to import the API specifications directly:
-1. Download the API specification by clicking the **Download** button at the
top of this document
+1. Download the API specification by clicking the **Download** button at top
of this document.
2. Impor
(airflow-client-python) branch fix_readme created (now 4e2362a)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch fix_readme in repository https://gitbox.apache.org/repos/asf/airflow-client-python.git at 4e2362a Update README for 2.9.0 This branch includes the following new commits: new 4e2362a Update README for 2.9.0 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(airflow) branch main updated: Update python client version in README (#39384)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new 495814e908 Update python client version in README (#39384) 495814e908 is described below commit 495814e908890763d9429df7ad9f4f33908fef98 Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com> AuthorDate: Thu May 2 18:07:28 2024 -0400 Update python client version in README (#39384) --- clients/python/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/clients/python/README.md b/clients/python/README.md index dc8fd402ae..f0964d5552 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -240,9 +240,10 @@ fulfilling the request. This Python package is automatically generated by the [OpenAPI Generator](https://openapi-generator.tech) project: -- API version: 2.8.0 -- Package version: 2.8.0 +- API version: 2.9.0 +- Package version: 2.9.0 - Build package: org.openapitools.codegen.languages.PythonClientCodegen + For more information, please visit [https://airflow.apache.org](https://airflow.apache.org) ## Requirements.
(airflow-client-python) annotated tag 2.9.0 updated (564f532 -> 9645a79)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to annotated tag 2.9.0 in repository https://gitbox.apache.org/repos/asf/airflow-client-python.git *** WARNING: tag 2.9.0 was modified! *** from 564f532 (commit) to 9645a79 (tag) tagging 564f5329d20331baa4c691c48ef0214763fa4b43 (commit) replaces 2.8.0rc1 by Jed Cunningham on Thu May 2 13:11:25 2024 -0600 - Log - 2.9.0 -BEGIN PGP SIGNATURE- iQIzBAABCAAdFiEE4aHphPVbjygL2cuiC7cWOJKi5I4FAmYz5V0ACgkQC7cWOJKi 5I41Bw/+LtsXEKDAOY50qC/N/djrjcFbO5NoC/8X1ElQW49F8IP+a17ApdOMqLq6 WcQoXkTISsc4AsF3Tn8KNknAf8sbvML3tV780pXywiRDiH5GsLsO/4otJ9gpHk0z ki9eA3Jhyrv+5MyYd12HGTGVTLWyiaQ3to8RS7+pmQ07DVHrROMYvC3eRGogxXwN wAC5Hn/XwUZwG7KMNlSNgj9w4WuME35f5664UJhp9doI15OnIGXOYWNKhH0bOzQN /h2Fgrkv5SHw3ZlciJY61QWsUvtx4yA7xS5w/7RBDjsgoQURMt10wJFL4VjxYBR1 2zOxcHfiOLwMiOEkF5DdB6L7bqTfH3ogKeCV5mAn8fJVSOQpvjb2zbbiuVIzdqLM UZujH7NFvzt0GIgtcy7AsJLJkkMee1y3OJyxM3HGEq3cmvNFOC2OE3XTua8ftF18 tcPU7lOfiXCXRzmB/POs1ZhSjPGQbVeNmP8noDe+OhVdLt2vqNpyLZrk6sRkYlBc i13CqSC5PKdt37fiogbb/ozAK2kvRkDSlxbz3O5btYRlsUi+LvfZVhTGI5wWn4cx LpZOsHjdtuE9hDJMXN4gTsseyOuBCywZ20s2Ekces9eo1zKJ8x9KcAO480Jn5nE3 +Qt92z/zfHYPTHKWTE8/75hcwT4inuoJi/1o/3ckeVY8HsfVRgU= =C7y3 -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
(airflow) annotated tag python-client-2.9.0 updated (e61cb8fa41 -> 67db47872f)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to annotated tag python-client-2.9.0 in repository https://gitbox.apache.org/repos/asf/airflow.git *** WARNING: tag python-client-2.9.0 was modified! *** from e61cb8fa41 (commit) to 67db47872f (tag) tagging e61cb8fa41f34bc5e3140a2c22b24dd110b4c421 (commit) replaces 2.9.0 by Jed Cunningham on Thu May 2 13:06:55 2024 -0600 - Log - Airflow Python Client 2.9.0 -BEGIN PGP SIGNATURE- iQIzBAABCAAdFiEE4aHphPVbjygL2cuiC7cWOJKi5I4FAmYz5E8ACgkQC7cWOJKi 5I4ahA/+PU1BLq41FwIe69ST6AGK7a7I6zfqURsf0aYMeQIt9vyL2OZ2V69m2SwD /jE+ShyqRMV7H7FHVwO3vVkNAks/UozdMnpjBkgHns0tI3KN1Faw8g6SAJps0AXz KaT8ifiHnAq8dcwzDu+5pl7ced9InWXxYEgjaHddbmL3w4m3TMK04P9VXGoGJfdF 5GGjbwTEVndLAMFzE+RbjzT7L/6BmoeQyZp8DMkTY55Wi+36/VPY5kgtq/2o77Hf H+E63JM4AuhKFtYpIkYgw/+KaDkgF4T71/9pVHpHFWkA7l6S+WW+zQcvekxMz1lQ dGBMJm8faihp9i1Xf85M4oZhCaKNZI51FMf/1AFkd+47J8JTKUNNkMu9UAVpP1il zJ8YQ+/ju7x0EYG+4cLBAVxifaDjfyjsPP/D7g7obe+19rXsemti8Jr7ymDvJHS5 5vIlHTI9dKq0CHB0OeRUDv0WgNI6S9jDzttFAkgnJexvIuKKMZOAvqIFpP0mCZv2 BEIKUpGva4yDZ8YtxMcf1BA2W0a7PGyemIeeY3tFK9RWLfZUxFPyqyN7ljaEE7YJ jaB1C1oEYeW0PmgHM7rE3mNNWEpmR9ZL+NFO92A904wAICROZOHmRkJ7ATGgUOOB eWfsFJD+2VprINfEB/Zmaq2FGA/GN476dqjh4tXKnNucoe5lj70= =FjnW -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
svn commit: r68924 - in /release/airflow/clients/python: 2.8.0/ 2.9.0/
Author: jedcunningham Date: Thu May 2 18:48:24 2024 New Revision: 68924 Log: Release Apache Airflow Python Client 2.9.0 from 0 Added: release/airflow/clients/python/2.9.0/ release/airflow/clients/python/2.9.0/apache_airflow_client-2.9.0-py3-none-any.whl - copied unchanged from r68923, dev/airflow/clients/python/2.9.0rc1/apache_airflow_client-2.9.0-py3-none-any.whl release/airflow/clients/python/2.9.0/apache_airflow_client-2.9.0-py3-none-any.whl.asc - copied unchanged from r68923, dev/airflow/clients/python/2.9.0rc1/apache_airflow_client-2.9.0-py3-none-any.whl.asc release/airflow/clients/python/2.9.0/apache_airflow_client-2.9.0-py3-none-any.whl.sha512 - copied unchanged from r68923, dev/airflow/clients/python/2.9.0rc1/apache_airflow_client-2.9.0-py3-none-any.whl.sha512 release/airflow/clients/python/2.9.0/apache_airflow_client-2.9.0.tar.gz - copied unchanged from r68923, dev/airflow/clients/python/2.9.0rc1/apache_airflow_client-2.9.0.tar.gz release/airflow/clients/python/2.9.0/apache_airflow_client-2.9.0.tar.gz.asc - copied unchanged from r68923, dev/airflow/clients/python/2.9.0rc1/apache_airflow_client-2.9.0.tar.gz.asc release/airflow/clients/python/2.9.0/apache_airflow_client-2.9.0.tar.gz.sha512 - copied unchanged from r68923, dev/airflow/clients/python/2.9.0rc1/apache_airflow_client-2.9.0.tar.gz.sha512 Removed: release/airflow/clients/python/2.8.0/
(airflow) branch main updated (29a9a24a5f -> 4910c0439b)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 29a9a24a5f Refactor tests re sensor reschedule mode and try_number (#39351) add 4910c0439b Add `jmespath` as an explicit dependency (#39350) No new revisions were added by this update. Summary of changes: airflow/providers/amazon/provider.yaml | 1 + airflow/providers/fab/provider.yaml| 1 + dev/breeze/tests/test_packages.py | 3 +++ generated/provider_dependencies.json | 4 +++- hatch_build.py | 1 + 5 files changed, 9 insertions(+), 1 deletion(-)
(airflow) branch main updated: docs: fix environment variable names when section name has dot in it (#39312)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f6fb4ccf2d docs: fix environment variable names when section name has
dot in it (#39312)
f6fb4ccf2d is described below
commit f6fb4ccf2dd048eb0024caec3baf44abc14a9307
Author: Kalle Ahlström <71292737+kahls...@users.noreply.github.com>
AuthorDate: Wed May 1 05:40:50 2024 +0300
docs: fix environment variable names when section name has dot in it
(#39312)
---
docs/exts/includes/sections-and-options.rst | 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/docs/exts/includes/sections-and-options.rst
b/docs/exts/includes/sections-and-options.rst
index 33ea64231f..f191cf10d5 100644
--- a/docs/exts/includes/sections-and-options.rst
+++ b/docs/exts/includes/sections-and-options.rst
@@ -63,13 +63,13 @@
:Default: ``{{ "''" if option["default"] == "" else option["default"] }}``
{% if option.get("sensitive") %}
:Environment Variables:
- ``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}``
+ ``AIRFLOW__{{ section_name | replace(".", "_") | upper }}__{{
option_name | upper }}``
- ``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}_CMD``
+ ``AIRFLOW__{{ section_name | replace(".", "_") | upper }}__{{
option_name | upper }}_CMD``
- ``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}_SECRET``
+ ``AIRFLOW__{{ section_name | replace(".", "_") | upper }}__{{
option_name | upper }}_SECRET``
{% else %}
-:Environment Variable: ``AIRFLOW__{{ section_name | upper }}__{{
option_name | upper }}``
+:Environment Variable: ``AIRFLOW__{{ section_name | replace(".", "_") |
upper }}__{{ option_name | upper }}``
{% endif %}
{% if option["example"] %}
:Example:
(airflow) branch main updated: Remove unused index `idx_last_scheduling_decision` on `dag_run` table (#39275)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 92ffab6004 Remove unused index `idx_last_scheduling_decision` on
`dag_run` table (#39275)
92ffab6004 is described below
commit 92ffab60049723a4a82fce7f256892fcd6a980d5
Author: Pankaj Koti
AuthorDate: Wed May 1 01:27:41 2024 +0530
Remove unused index `idx_last_scheduling_decision` on `dag_run` table
(#39275)
---
..._2_9_2_remove_idx_last_scheduling_decision_.py} | 24 +++---
...=> 0142_2_10_0_add_new_executor_field_to_db.py} | 6 +++---
airflow/models/dagrun.py | 1 -
airflow/utils/db.py| 2 ++
docs/apache-airflow/img/airflow_erd.sha256 | 2 +-
docs/apache-airflow/migrations-ref.rst | 5 -
6 files changed, 22 insertions(+), 18 deletions(-)
diff --git
a/airflow/migrations/versions/0141_2_10_0_add_new_executor_field_to_db.py
b/airflow/migrations/versions/0141_2_9_2_remove_idx_last_scheduling_decision_.py
similarity index 58%
copy from
airflow/migrations/versions/0141_2_10_0_add_new_executor_field_to_db.py
copy to
airflow/migrations/versions/0141_2_9_2_remove_idx_last_scheduling_decision_.py
index 67e463f1cc..70923a4780 100644
--- a/airflow/migrations/versions/0141_2_10_0_add_new_executor_field_to_db.py
+++
b/airflow/migrations/versions/0141_2_9_2_remove_idx_last_scheduling_decision_.py
@@ -16,31 +16,31 @@
# specific language governing permissions and limitations
# under the License.
-"""add new executor field to db
+"""Remove ``idx_last_scheduling_decision`` index on last_scheduling_decision
in dag_run table
-Revision ID: 677fdbb7fc54
-Revises: 1949afb29106
-Create Date: 2024-04-01 15:26:59.186579
+Revision ID: bff083ad727d
+Revises: 677fdbb7fc54
+Create Date: 2024-04-26 12:58:00.594762
"""
-import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
-revision = '677fdbb7fc54'
-down_revision = '1949afb29106'
+revision = "bff083ad727d"
+down_revision = "1949afb29106"
branch_labels = None
depends_on = None
-airflow_version = '2.10.0'
+airflow_version = "2.9.2"
def upgrade():
-"""Apply add executor field to task instance"""
-op.add_column('task_instance', sa.Column('executor',
sa.String(length=1000), default=None))
+"""Apply Remove idx_last_scheduling_decision index on
last_scheduling_decision in dag_run table"""
+op.drop_index("idx_last_scheduling_decision", table_name="dag_run")
def downgrade():
-"""Unapply add executor field to task instance"""
-op.drop_column('task_instance', 'executor')
+"""Unapply Remove idx_last_scheduling_decision index on
last_scheduling_decision in dag_run table"""
+op.create_index("idx_last_scheduling_decision", "dag_run",
["last_scheduling_decision"],
+unique=False)
diff --git
a/airflow/migrations/versions/0141_2_10_0_add_new_executor_field_to_db.py
b/airflow/migrations/versions/0142_2_10_0_add_new_executor_field_to_db.py
similarity index 94%
rename from
airflow/migrations/versions/0141_2_10_0_add_new_executor_field_to_db.py
rename to
airflow/migrations/versions/0142_2_10_0_add_new_executor_field_to_db.py
index 67e463f1cc..0a6b64460e 100644
--- a/airflow/migrations/versions/0141_2_10_0_add_new_executor_field_to_db.py
+++ b/airflow/migrations/versions/0142_2_10_0_add_new_executor_field_to_db.py
@@ -29,11 +29,11 @@ from alembic import op
# revision identifiers, used by Alembic.
-revision = '677fdbb7fc54'
-down_revision = '1949afb29106'
+revision = "677fdbb7fc54"
+down_revision = "bff083ad727d"
branch_labels = None
depends_on = None
-airflow_version = '2.10.0'
+airflow_version = "2.10.0"
def upgrade():
diff --git a/airflow/models/dagrun.py b/airflow/models/dagrun.py
index 774f8d0983..77679f8d9a 100644
--- a/airflow/models/dagrun.py
+++ b/airflow/models/dagrun.py
@@ -159,7 +159,6 @@ class DagRun(Base, LoggingMixin):
Index("dag_id_state", dag_id, _state),
UniqueConstraint("dag_id", "execution_date",
name="dag_run_dag_id_execution_date_key"),
UniqueConstraint("dag_id", "run_id", name="dag_run_dag_id_run_id_key"),
-Index("idx_last_scheduling_decision", last_scheduling_decision),
Index("idx_dag_run_dag_id", dag_id),
Index(
"idx_dag_run_running_dags",
diff --git a/airflow/utils/db.py b/airflow/utils/db.py
index b7997498bb..af53c241ed 100644
--- a/airfl
(airflow) branch v2-9-test updated (44a1fe781b -> 07f7e8f747)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch v2-9-test in repository https://gitbox.apache.org/repos/asf/airflow.git from 44a1fe781b Avoid to use `functools.lru_cache` in class methods in dev (#38627) add 07f7e8f747 Update RELEASE_NOTES.rst No new revisions were added by this update. Summary of changes: RELEASE_NOTES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(airflow) branch v2-9-test updated (fe7dd31836 -> 44a1fe781b)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch v2-9-test in repository https://gitbox.apache.org/repos/asf/airflow.git from fe7dd31836 Fix reproducible build hash add 44a1fe781b Avoid to use `functools.lru_cache` in class methods in dev (#38627) No new revisions were added by this update. Summary of changes: .../src/airflow_breeze/utils/selective_checks.py | 60 -- 1 file changed, 34 insertions(+), 26 deletions(-)
(airflow) branch v2-9-test updated: Copy menu_item href for nav bar (#39282)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch v2-9-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v2-9-test by this push:
new cddaf23e59 Copy menu_item href for nav bar (#39282)
cddaf23e59 is described below
commit cddaf23e594aa6999421a39ba8cf718f884de9fe
Author: Brent Bovenzi
AuthorDate: Sat Apr 27 12:21:24 2024 -0400
Copy menu_item href for nav bar (#39282)
Co-authored-by: Jed Cunningham
(cherry picked from commit 25f901a963001377621abe0ac0a1ff121a042bcd)
---
airflow/auth/managers/base_auth_manager.py| 10 --
tests/auth/managers/test_base_auth_manager.py | 16 +++-
2 files changed, 19 insertions(+), 7 deletions(-)
diff --git a/airflow/auth/managers/base_auth_manager.py
b/airflow/auth/managers/base_auth_manager.py
index 44fc53a66e..86f0ebd6dc 100644
--- a/airflow/auth/managers/base_auth_manager.py
+++ b/airflow/auth/managers/base_auth_manager.py
@@ -398,12 +398,10 @@ class BaseAuthManager(LoggingMixin):
accessible_items = []
for menu_item in items:
menu_item_copy = MenuItem(
-name=menu_item.name,
-icon=menu_item.icon,
-label=menu_item.label,
-childs=[],
-baseview=menu_item.baseview,
-cond=menu_item.cond,
+**{
+**menu_item.__dict__,
+"childs": [],
+}
)
if menu_item.childs:
accessible_children = []
diff --git a/tests/auth/managers/test_base_auth_manager.py
b/tests/auth/managers/test_base_auth_manager.py
index a39b60787c..7628924ad6 100644
--- a/tests/auth/managers/test_base_auth_manager.py
+++ b/tests/auth/managers/test_base_auth_manager.py
@@ -300,7 +300,15 @@ class TestBaseAuthManager:
mock_security_manager.has_access.side_effect = [True, False, True,
True, False]
menu = Menu()
-menu.add_link("item1")
+menu.add_link(
+# These may not all be valid types, but it does let us check each
attr is copied
+name="item1",
+href="h1",
+icon="i1",
+label="l1",
+baseview="b1",
+cond="c1",
+)
menu.add_link("item2")
menu.add_link("item3")
menu.add_link("item3.1", category="item3")
@@ -313,6 +321,12 @@ class TestBaseAuthManager:
assert result[1].name == "item3"
assert len(result[1].childs) == 1
assert result[1].childs[0].name == "item3.1"
+# check we've copied every attr
+assert result[0].href == "h1"
+assert result[0].icon == "i1"
+assert result[0].label == "l1"
+assert result[0].baseview == "b1"
+assert result[0].cond == "c1"
@patch.object(EmptyAuthManager, "security_manager")
def test_filter_permitted_menu_items_twice(self, mock_security_manager,
auth_manager):
(airflow) branch main updated: Copy menu_item href for nav bar (#39282)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 25f901a963 Copy menu_item href for nav bar (#39282)
25f901a963 is described below
commit 25f901a963001377621abe0ac0a1ff121a042bcd
Author: Brent Bovenzi
AuthorDate: Sat Apr 27 12:21:24 2024 -0400
Copy menu_item href for nav bar (#39282)
Co-authored-by: Jed Cunningham
---
airflow/auth/managers/base_auth_manager.py| 10 --
tests/auth/managers/test_base_auth_manager.py | 16 +++-
2 files changed, 19 insertions(+), 7 deletions(-)
diff --git a/airflow/auth/managers/base_auth_manager.py
b/airflow/auth/managers/base_auth_manager.py
index 44fc53a66e..86f0ebd6dc 100644
--- a/airflow/auth/managers/base_auth_manager.py
+++ b/airflow/auth/managers/base_auth_manager.py
@@ -398,12 +398,10 @@ class BaseAuthManager(LoggingMixin):
accessible_items = []
for menu_item in items:
menu_item_copy = MenuItem(
-name=menu_item.name,
-icon=menu_item.icon,
-label=menu_item.label,
-childs=[],
-baseview=menu_item.baseview,
-cond=menu_item.cond,
+**{
+**menu_item.__dict__,
+"childs": [],
+}
)
if menu_item.childs:
accessible_children = []
diff --git a/tests/auth/managers/test_base_auth_manager.py
b/tests/auth/managers/test_base_auth_manager.py
index a39b60787c..7628924ad6 100644
--- a/tests/auth/managers/test_base_auth_manager.py
+++ b/tests/auth/managers/test_base_auth_manager.py
@@ -300,7 +300,15 @@ class TestBaseAuthManager:
mock_security_manager.has_access.side_effect = [True, False, True,
True, False]
menu = Menu()
-menu.add_link("item1")
+menu.add_link(
+# These may not all be valid types, but it does let us check each
attr is copied
+name="item1",
+href="h1",
+icon="i1",
+label="l1",
+baseview="b1",
+cond="c1",
+)
menu.add_link("item2")
menu.add_link("item3")
menu.add_link("item3.1", category="item3")
@@ -313,6 +321,12 @@ class TestBaseAuthManager:
assert result[1].name == "item3"
assert len(result[1].childs) == 1
assert result[1].childs[0].name == "item3.1"
+# check we've copied every attr
+assert result[0].href == "h1"
+assert result[0].icon == "i1"
+assert result[0].label == "l1"
+assert result[0].baseview == "b1"
+assert result[0].cond == "c1"
@patch.object(EmptyAuthManager, "security_manager")
def test_filter_permitted_menu_items_twice(self, mock_security_manager,
auth_manager):
(airflow) branch v2-9-test updated: Move significant note for past release to release notes (#39283)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch v2-9-test in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/v2-9-test by this push: new 368ff13c13 Move significant note for past release to release notes (#39283) 368ff13c13 is described below commit 368ff13c13d33854e0da8940f1b9d77ff21c9a9c Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com> AuthorDate: Sat Apr 27 09:56:48 2024 -0400 Move significant note for past release to release notes (#39283) Once the release is out, these should be added directly to the release notes - newsfragments are all about future releases! (cherry picked from commit 8dfdc3a0a8aa4a12f0e0e3f6e0a6ff925646c201) --- RELEASE_NOTES.rst | 8 airflow/reproducible_build.yaml | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 5533fad32c..1f323919ea 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -111,6 +111,14 @@ Xcom table column ``value`` type has changed from ``blob`` to ``longblob``. This To downgrade from revision: ``b4078ac230a1``, ensure that you don't have Xcom values larger than 65,535 bytes. Otherwise, you'll need to clean those rows or run ``airflow db clean xcom`` to clean the Xcom table. +Stronger validation for key parameter defaults in taskflow context variables (#38015) +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +As for the taskflow implementation in conjunction with context variable defaults invalid parameter orders can be +generated, it is now not accepted anymore (and validated) that taskflow functions are defined with defaults +other than ``None``. If you have done this before you most likely will see a broken DAG and a error message like +``Error message: Context key parameter my_param can't have a default other than None``. + New Features """""""""""" - Allow users to write dag_id and task_id in their national characters, added display name for dag / task (v2) (#38446) diff --git a/airflow/reproducible_build.yaml b/airflow/reproducible_build.yaml index fc6a3933e8..c6683aa2c0 100644 --- a/airflow/reproducible_build.yaml +++ b/airflow/reproducible_build.yaml @@ -1,2 +1,2 @@ -release-notes-hash: 416d01241f2b6ed259e8d991fb7ac1f8 -source-date-epoch: 1712672348 +release-notes-hash: aad86522e49984ce17db1b8647cfb54a +source-date-epoch: 1714165337
(airflow) branch main updated (60b17bb78a -> 8dfdc3a0a8)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 60b17bb78a Improve helm chart annotation building script (#39286) add 8dfdc3a0a8 Move significant note for past release to release notes (#39283) No new revisions were added by this update. Summary of changes: RELEASE_NOTES.rst | 8 airflow/reproducible_build.yaml | 4 ++-- newsfragments/38015.significant.rst | 6 -- 3 files changed, 10 insertions(+), 8 deletions(-) delete mode 100644 newsfragments/38015.significant.rst
(airflow) branch main updated (48c98bc1d4 -> 60b17bb78a)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 48c98bc1d4 Update Hashicorp AWS assume role auth docs (#39287) add 60b17bb78a Improve helm chart annotation building script (#39286) No new revisions were added by this update. Summary of changes: dev/chart/build_changelog_annotations.py | 11 --- 1 file changed, 8 insertions(+), 3 deletions(-)
(airflow) branch main updated (a7e5510dbb -> 49865acf30)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from a7e5510dbb Make audit log before/after filterable (#39120) add 49865acf30 Fix supported version of python/k8s in main README (#39284) No new revisions were added by this update. Summary of changes: README.md| 4 ++-- generated/PYPI_README.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-)
(airflow) branch v2-9-test updated (c9cc72608d -> eb11360a9c)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a change to branch v2-9-test in repository https://gitbox.apache.org/repos/asf/airflow.git from c9cc72608d Fix trigger kwarg encryption migration (#39246) new 0a40b34c17 Fix supported k8s version in docs (#39172) new eb11360a9c (img/edge_label_example.png): changed imaged to new label view (#38802) The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: docs/apache-airflow/img/edge_label_example.png | Bin 33270 -> 37438 bytes docs/apache-airflow/installation/prerequisites.rst | 2 +- docs/helm-chart/index.rst | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-)
(airflow) 01/02: Fix supported k8s version in docs (#39172)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch v2-9-test in repository https://gitbox.apache.org/repos/asf/airflow.git commit 0a40b34c17d0a2ed91560620496238ba5557acd3 Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com> AuthorDate: Mon Apr 22 13:03:20 2024 -0400 Fix supported k8s version in docs (#39172) These were missed when support for 1.25 was dropped. (cherry picked from commit acc75c959becd6957ba690b6e2afe0877a298f91) --- docs/apache-airflow/installation/prerequisites.rst | 2 +- docs/helm-chart/index.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/apache-airflow/installation/prerequisites.rst b/docs/apache-airflow/installation/prerequisites.rst index c253217a29..98e20988ec 100644 --- a/docs/apache-airflow/installation/prerequisites.rst +++ b/docs/apache-airflow/installation/prerequisites.rst @@ -28,7 +28,7 @@ Airflow™ is tested with: * MySQL: 8.0, `Innovation <https://dev.mysql.com/blog-archive/introducing-mysql-innovation-and-long-term-support-lts-versions>`_ * SQLite: 3.15.0+ -* Kubernetes: 1.25, 1.26, 1.27, 1.28, 1.29 +* Kubernetes: 1.26, 1.27, 1.28, 1.29 The minimum memory required we recommend Airflow to run with is 4GB, but the actual requirements depend wildly on the deployment options you have diff --git a/docs/helm-chart/index.rst b/docs/helm-chart/index.rst index e86fa71dce..d6194049d4 100644 --- a/docs/helm-chart/index.rst +++ b/docs/helm-chart/index.rst @@ -59,7 +59,7 @@ deployment on a `Kubernetes <http://kubernetes.io>`__ cluster using the Requirements -- Kubernetes 1.25+ cluster +- Kubernetes 1.26+ cluster - Helm 3.0+ - PV provisioner support in the underlying infrastructure (optionally)
(airflow) 02/02: (img/edge_label_example.png): changed imaged to new label view (#38802)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch v2-9-test in repository https://gitbox.apache.org/repos/asf/airflow.git commit eb11360a9c49a6f8d3c0b2ba73129638426bd6d4 Author: Idan Tepper <65827206+idantep...@users.noreply.github.com> AuthorDate: Sun Apr 7 19:17:30 2024 +0300 (img/edge_label_example.png): changed imaged to new label view (#38802) Co-authored-by: idantep...@gmail.com (cherry picked from commit 6004175d555b648c0de92a11dfd551985237834d) --- docs/apache-airflow/img/edge_label_example.png | Bin 33270 -> 37438 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/docs/apache-airflow/img/edge_label_example.png b/docs/apache-airflow/img/edge_label_example.png index 8d0b64650e..52c280fcb1 100644 Binary files a/docs/apache-airflow/img/edge_label_example.png and b/docs/apache-airflow/img/edge_label_example.png differ
(airflow) 37/48: Add k8s 1.29 to supported version in docs (#39168)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch v2-9-test in repository https://gitbox.apache.org/repos/asf/airflow.git commit fd0226e068d6ade76f558d37b8e75f218fe9ea44 Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com> AuthorDate: Mon Apr 22 11:56:28 2024 -0400 Add k8s 1.29 to supported version in docs (#39168) (cherry picked from commit 897f00d3cafb8a7ef9c1444997784d693c09611f) --- docs/apache-airflow/installation/prerequisites.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-airflow/installation/prerequisites.rst b/docs/apache-airflow/installation/prerequisites.rst index 07ccc903c7..c253217a29 100644 --- a/docs/apache-airflow/installation/prerequisites.rst +++ b/docs/apache-airflow/installation/prerequisites.rst @@ -28,7 +28,7 @@ Airflow™ is tested with: * MySQL: 8.0, `Innovation <https://dev.mysql.com/blog-archive/introducing-mysql-innovation-and-long-term-support-lts-versions>`_ * SQLite: 3.15.0+ -* Kubernetes: 1.25, 1.26, 1.27, 1.28 +* Kubernetes: 1.25, 1.26, 1.27, 1.28, 1.29 The minimum memory required we recommend Airflow to run with is 4GB, but the actual requirements depend wildly on the deployment options you have
(airflow) 38/48: Update `is_authorized_custom_view` from auth manager to handle custom actions (#39167)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch v2-9-test in repository https://gitbox.apache.org/repos/asf/airflow.git commit 092ceaed55fff2d09c5a975b255d33d3d29106b9 Author: Vincent <97131062+vincb...@users.noreply.github.com> AuthorDate: Mon Apr 22 12:36:26 2024 -0400 Update `is_authorized_custom_view` from auth manager to handle custom actions (#39167) (cherry picked from commit f7a2f6032544defa8a00d1f7fa90e91d27eb3a8e) --- airflow/auth/managers/base_auth_manager.py | 7 +-- airflow/providers/amazon/aws/auth_manager/avp/entities.py | 2 +- airflow/providers/amazon/aws/auth_manager/avp/facade.py | 7 +-- .../providers/amazon/aws/auth_manager/aws_auth_manager.py | 2 +- airflow/providers/fab/auth_manager/fab_auth_manager.py | 7 +-- tests/auth/managers/test_base_auth_manager.py | 2 +- tests/providers/fab/auth_manager/test_fab_auth_manager.py | 13 - 7 files changed, 30 insertions(+), 10 deletions(-) diff --git a/airflow/auth/managers/base_auth_manager.py b/airflow/auth/managers/base_auth_manager.py index 4d5c249235..7bb4e92889 100644 --- a/airflow/auth/managers/base_auth_manager.py +++ b/airflow/auth/managers/base_auth_manager.py @@ -237,7 +237,7 @@ class BaseAuthManager(LoggingMixin): @abstractmethod def is_authorized_custom_view( -self, *, method: ResourceMethod, resource_name: str, user: BaseUser | None = None +self, *, method: ResourceMethod | str, resource_name: str, user: BaseUser | None = None ): """ Return whether the user is authorized to perform a given action on a custom view. @@ -246,7 +246,10 @@ class BaseAuthManager(LoggingMixin): the auth manager is used as part of the environment. It can also be a view defined as part of a plugin defined by a user. -:param method: the method to perform +:param method: the method to perform. +The method can also be a string if the action has been defined in a plugin. +In that case, the action can be anything (e.g. can_do). +See https://github.com/apache/airflow/issues/39144 :param resource_name: the name of the resource :param user: the user to perform the action on. If not provided (or None), it uses the current user """ diff --git a/airflow/providers/amazon/aws/auth_manager/avp/entities.py b/airflow/providers/amazon/aws/auth_manager/avp/entities.py index f2c6376729..8c2e8855b8 100644 --- a/airflow/providers/amazon/aws/auth_manager/avp/entities.py +++ b/airflow/providers/amazon/aws/auth_manager/avp/entities.py @@ -55,7 +55,7 @@ def get_entity_type(resource_type: AvpEntities) -> str: return AVP_PREFIX_ENTITIES + resource_type.value -def get_action_id(resource_type: AvpEntities, method: ResourceMethod): +def get_action_id(resource_type: AvpEntities, method: ResourceMethod | str): """ Return action id. diff --git a/airflow/providers/amazon/aws/auth_manager/avp/facade.py b/airflow/providers/amazon/aws/auth_manager/avp/facade.py index 010531155e..4bb9515004 100644 --- a/airflow/providers/amazon/aws/auth_manager/avp/facade.py +++ b/airflow/providers/amazon/aws/auth_manager/avp/facade.py @@ -75,7 +75,7 @@ class AwsAuthManagerAmazonVerifiedPermissionsFacade(LoggingMixin): def is_authorized( self, *, -method: ResourceMethod, +method: ResourceMethod | str, entity_type: AvpEntities, user: AwsAuthManagerUser | None, entity_id: str | None = None, @@ -86,7 +86,10 @@ class AwsAuthManagerAmazonVerifiedPermissionsFacade(LoggingMixin): Check whether the user has permissions to access given resource. -:param method: the method to perform +:param method: the method to perform. +The method can also be a string if the action has been defined in a plugin. +In that case, the action can be anything (e.g. can_do). +See https://github.com/apache/airflow/issues/39144 :param entity_type: the entity type the user accesses :param user: the user :param entity_id: the entity ID the user accesses. If not provided, all entities of the type will be diff --git a/airflow/providers/amazon/aws/auth_manager/aws_auth_manager.py b/airflow/providers/amazon/aws/auth_manager/aws_auth_manager.py index eb202acfca..1d2bcf1b19 100644 --- a/airflow/providers/amazon/aws/auth_manager/aws_auth_manager.py +++ b/airflow/providers/amazon/aws/auth_manager/aws_auth_manager.py @@ -207,7 +207,7 @@ class AwsAuthManager(BaseAuthManager): ) def is_authorized_custom_view( -self, *, method: ResourceMethod, resource_name: str, user: BaseUser | None = None +self, *, method: ResourceMethod | str, resource_name: str, user: BaseUser | None = None
(airflow) 44/48: Get served logs when remote or executor logs not available for non-running task try (#39177)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch v2-9-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 505ecd95381610ed073a3bb5cfd9d773335b8a2b
Author: Kalle Ahlström <71292737+kahls...@users.noreply.github.com>
AuthorDate: Thu Apr 25 16:19:37 2024 +0300
Get served logs when remote or executor logs not available for non-running
task try (#39177)
(cherry picked from commit eca077b3b994813a09942497704ec61e35efd7d5)
---
airflow/utils/log/file_task_handler.py | 12 ++--
tests/utils/test_log_handlers.py | 14 --
2 files changed, 18 insertions(+), 8 deletions(-)
diff --git a/airflow/utils/log/file_task_handler.py
b/airflow/utils/log/file_task_handler.py
index a5a2da6062..6d35b230b4 100644
--- a/airflow/utils/log/file_task_handler.py
+++ b/airflow/utils/log/file_task_handler.py
@@ -343,10 +343,7 @@ class FileTaskHandler(logging.Handler):
executor_messages: list[str] = []
executor_logs: list[str] = []
served_logs: list[str] = []
-is_running = ti.try_number == try_number and ti.state in (
-TaskInstanceState.RUNNING,
-TaskInstanceState.DEFERRED,
-)
+is_in_running_or_deferred = ti.state in (TaskInstanceState.RUNNING,
TaskInstanceState.DEFERRED)
with suppress(NotImplementedError):
remote_messages, remote_logs = self._read_remote_logs(ti,
try_number, metadata)
messages_list.extend(remote_messages)
@@ -361,7 +358,9 @@ class FileTaskHandler(logging.Handler):
worker_log_full_path = Path(self.local_base, worker_log_rel_path)
local_messages, local_logs =
self._read_from_local(worker_log_full_path)
messages_list.extend(local_messages)
-if is_running and not executor_messages:
+if is_in_running_or_deferred and not executor_messages and not
remote_logs:
+# While task instance is still running and we don't have either
executor nor remote logs, look for served logs
+# This is for cases when users have not setup remote logging nor
shared drive for logs
served_messages, served_logs = self._read_from_logs_server(ti,
worker_log_rel_path)
messages_list.extend(served_messages)
elif ti.state not in State.unfinished and not (local_logs or
remote_logs):
@@ -381,11 +380,12 @@ class FileTaskHandler(logging.Handler):
)
log_pos = len(logs)
messages = "".join([f"*** {x}\n" for x in messages_list])
+end_of_log = ti.try_number != try_number or not
is_in_running_or_deferred
if metadata and "log_pos" in metadata:
previous_chars = metadata["log_pos"]
logs = logs[previous_chars:] # Cut off previously passed log test
as new tail
out_message = logs if "log_pos" in (metadata or {}) else messages +
logs
-return out_message, {"end_of_log": not is_running, "log_pos": log_pos}
+return out_message, {"end_of_log": end_of_log, "log_pos": log_pos}
@staticmethod
def _get_pod_namespace(ti: TaskInstance):
diff --git a/tests/utils/test_log_handlers.py b/tests/utils/test_log_handlers.py
index 7f84b6d9f1..13f1e4de0e 100644
--- a/tests/utils/test_log_handlers.py
+++ b/tests/utils/test_log_handlers.py
@@ -304,7 +304,7 @@ class TestFileTaskLogHandler:
def test__read_for_celery_executor_fallbacks_to_worker(self,
create_task_instance):
"""Test for executors which do not have `get_task_log` method, it
fallbacks to reading
-log from worker. But it happens only for the latest try_number."""
+log from worker if and only if remote logs aren't found"""
executor_name = "CeleryExecutor"
ti = create_task_instance(
@@ -325,7 +325,17 @@ class TestFileTaskLogHandler:
fth._read_from_logs_server.assert_called_once()
assert actual == ("*** this message\nthis\nlog\ncontent",
{"end_of_log": False, "log_pos": 16})
-# Previous try_number is from remote logs without reaching worker
server
+# Previous try_number should return served logs when remote logs
aren't implemented
+fth._read_from_logs_server = mock.Mock()
+fth._read_from_logs_server.return_value = ["served logs
try_number=1"], ["this\nlog\ncontent"]
+actual = fth._read(ti=ti, try_number=1)
+fth._read_from_logs_server.assert_called_once()
+assert actual == (
+"*** served logs try_number=1\nthis\nlog\ncontent",
+{"end_of_log": True, "log_pos": 16},
+)
+
+# When remote_logs is implemented, previous try_number is f
(airflow) 36/48: Improve task filtering UX (#39119)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch v2-9-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 20910721674cf3c7fdef159a48a0661a5f91b708
Author: Brent Bovenzi
AuthorDate: Mon Apr 22 10:15:07 2024 -0400
Improve task filtering UX (#39119)
* Improve filtering grid/graph/gantt filtering by task UX
* only show map index if greater than -1
(cherry picked from commit 596e9ea10c1e7359d63532b74f3b7f12b42599e5)
---
airflow/www/static/js/dag/details/FilterTasks.tsx | 76 +++
airflow/www/static/js/dag/details/Header.tsx | 2 +-
airflow/www/static/js/dag/details/index.tsx | 2 +-
3 files changed, 52 insertions(+), 28 deletions(-)
diff --git a/airflow/www/static/js/dag/details/FilterTasks.tsx
b/airflow/www/static/js/dag/details/FilterTasks.tsx
index 33d830b74d..7bc9b976b9 100644
--- a/airflow/www/static/js/dag/details/FilterTasks.tsx
+++ b/airflow/www/static/js/dag/details/FilterTasks.tsx
@@ -28,9 +28,11 @@ import {
} from "@chakra-ui/react";
import useFilters from "src/dag/useFilters";
import { MdArrowDropDown } from "react-icons/md";
+import { useGridData } from "src/api";
+import { getTask } from "src/utils";
interface Props {
- taskId: string;
+ taskId: string | null;
}
const FilterTasks = ({ taskId }: Props) => {
@@ -40,34 +42,48 @@ const FilterTasks = ({ taskId }: Props) => {
resetRoot,
} = useFilters();
- const onFilterUpstream = () =>
-onFilterTasksChange({
- root: taskId,
- filterUpstream: true,
- filterDownstream: false,
-});
+ const {
+data: { groups },
+ } = useGridData();
- const onFilterDownstream = () =>
-onFilterTasksChange({
- root: taskId,
- filterUpstream: false,
- filterDownstream: true,
-});
+ const label = "Filter upstream and/or downstream of a task";
- const onFilterAll = () =>
-onFilterTasksChange({
- root: taskId,
- filterUpstream: true,
- filterDownstream: true,
-});
+ if (!root && !taskId) return null;
- const label = "Filter upstream and/or downstream of a task";
+ const onFilterUpstream = () => {
+if (taskId)
+ onFilterTasksChange({
+root: taskId,
+filterUpstream: true,
+filterDownstream: false,
+ });
+ };
+
+ const onFilterDownstream = () => {
+if (taskId)
+ onFilterTasksChange({
+root: taskId,
+filterUpstream: false,
+filterDownstream: true,
+ });
+ };
+
+ const onFilterAll = () => {
+if (taskId)
+ onFilterTasksChange({
+root: taskId,
+filterUpstream: true,
+filterDownstream: true,
+ });
+ };
+
+ const task = root ? getTask({ taskId: root, task: groups }) : undefined;
return (
{
mt={2}
>
- {!root ? "Filter Tasks " : "Clear Task Filter "}
+ {root
+? `Filtered on ${task?.label || "unknown"}`
+: "Filter DAG by task"}
-Filter Upstream
-Filter Downstream
-Filter Upstream & Downstream
-{!!root && Reset Root}
+{!!root && Reset}
+{!!taskId && (
+ <>
+Only upstream
+Only downstream
+
+ Both upstream & downstream
+
+
+)}
);
diff --git a/airflow/www/static/js/dag/details/Header.tsx
b/airflow/www/static/js/dag/details/Header.tsx
index d13dfa6c81..78f30f3620 100644
--- a/airflow/www/static/js/dag/details/Header.tsx
+++ b/airflow/www/static/js/dag/details/Header.tsx
@@ -124,7 +124,7 @@ const Header = ({ mapIndex }: Props) => {
)}
- {mapIndex !== undefined && (
+ {mapIndex !== undefined && mapIndex !== -1 && (
)}
- {taskId && runId && }
+
(airflow) 23/48: Add `inherit_cache` attribute for `CreateTableAs` custom SA Clause (#38985)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch v2-9-test in repository https://gitbox.apache.org/repos/asf/airflow.git commit 82a1b6abdeeb225b75f53fcfa9d74ba258a4aee9 Author: Andrey Anshin AuthorDate: Sun Apr 14 17:19:17 2024 +0400 Add `inherit_cache` attribute for `CreateTableAs` custom SA Clause (#38985) (cherry picked from commit bdb8f749ee462d508bc9da43b16d18a599ff0e7b) --- airflow/utils/db_cleanup.py | 2 ++ pyproject.toml | 1 + 2 files changed, 3 insertions(+) diff --git a/airflow/utils/db_cleanup.py b/airflow/utils/db_cleanup.py index 4475209b77..7fe158e605 100644 --- a/airflow/utils/db_cleanup.py +++ b/airflow/utils/db_cleanup.py @@ -218,6 +218,8 @@ def _subquery_keep_last(*, recency_column, keep_last_filters, group_by_columns, class CreateTableAs(Executable, ClauseElement): """Custom sqlalchemy clause element for CTAS operations.""" +inherit_cache = False + def __init__(self, name, query): self.name = name self.query = query diff --git a/pyproject.toml b/pyproject.toml index fbc7b7a9d7..4107396af3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -482,6 +482,7 @@ filterwarnings = [ # Avoid building cartesian product which might impact performance "error:SELECT statement has a cartesian product between FROM:sqlalchemy.exc.SAWarning:airflow", 'error:Coercing Subquery object into a select\(\) for use in IN\(\):sqlalchemy.exc.SAWarning:airflow', +'error:Class.*will not make use of SQL compilation caching', "ignore::DeprecationWarning:flask_appbuilder.filemanager", "ignore::DeprecationWarning:flask_appbuilder.widgets", # https://github.com/dpgaspar/Flask-AppBuilder/pull/1940
(airflow) 45/48: Force to use Airflow Deprecation warnings categories on `@deprecated` decorator (#39205)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch v2-9-test in repository https://gitbox.apache.org/repos/asf/airflow.git commit d2ae8c97550eaeaefca309d4b5878546d66895c9 Author: Andrey Anshin AuthorDate: Thu Apr 25 17:44:22 2024 +0400 Force to use Airflow Deprecation warnings categories on `@deprecated` decorator (#39205) * Force to use Airflow Deprecation warnings categories on @deprecated decorator * Catch warning in Experimental API test (cherry picked from commit c2ef1daea65885db04610eaa7e0200aec0e0) --- .pre-commit-config.yaml| 7 + airflow/api/common/experimental/get_code.py| 6 +- .../api/common/experimental/get_dag_run_state.py | 3 +- airflow/api/common/experimental/get_task.py| 3 +- .../api/common/experimental/get_task_instance.py | 4 +- airflow/api/common/experimental/pool.py| 10 +- airflow/auth/managers/fab/fab_auth_manager.py | 5 +- .../auth/managers/fab/security_manager/override.py | 4 +- airflow/www/security.py| 4 +- contributing-docs/08_static_code_checks.rst| 2 + dev/breeze/doc/images/output_static-checks.svg | 152 dev/breeze/doc/images/output_static-checks.txt | 2 +- dev/breeze/src/airflow_breeze/pre_commit_ids.py| 1 + scripts/ci/pre_commit/check_deprecations.py| 194 + .../auth/backend/test_basic_auth.py| 6 +- 15 files changed, 313 insertions(+), 90 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b2f998afa9..308d63a615 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -987,6 +987,13 @@ repos: pass_filenames: true files: ^airflow/.*\.py$ exclude: ^.*/.*_vendor/ + - id: check-code-deprecations +name: Check deprecations categories in decorators +entry: ./scripts/ci/pre_commit/check_deprecations.py +language: python +pass_filenames: true +files: ^airflow/.*\.py$ +exclude: ^.*/.*_vendor/ - id: lint-chart-schema name: Lint chart/values.schema.json file entry: ./scripts/ci/pre_commit/chart_schema.py diff --git a/airflow/api/common/experimental/get_code.py b/airflow/api/common/experimental/get_code.py index f618690a92..96aadc441b 100644 --- a/airflow/api/common/experimental/get_code.py +++ b/airflow/api/common/experimental/get_code.py @@ -22,11 +22,13 @@ from __future__ import annotations from deprecated import deprecated from airflow.api.common.experimental import check_and_get_dag -from airflow.exceptions import AirflowException, DagCodeNotFound +from airflow.exceptions import AirflowException, DagCodeNotFound, RemovedInAirflow3Warning from airflow.models.dagcode import DagCode -@deprecated(reason="Use DagCode().get_code_by_fileloc() instead", version="2.2.4") +@deprecated( +reason="Use DagCode().get_code_by_fileloc() instead", version="2.2.4", category=RemovedInAirflow3Warning +) def get_code(dag_id: str) -> str: """Return python code of a given dag_id. diff --git a/airflow/api/common/experimental/get_dag_run_state.py b/airflow/api/common/experimental/get_dag_run_state.py index 8905aa4f04..6c7cd96c2b 100644 --- a/airflow/api/common/experimental/get_dag_run_state.py +++ b/airflow/api/common/experimental/get_dag_run_state.py @@ -24,12 +24,13 @@ from typing import TYPE_CHECKING from deprecated import deprecated from airflow.api.common.experimental import check_and_get_dag, check_and_get_dagrun +from airflow.exceptions import RemovedInAirflow3Warning if TYPE_CHECKING: from datetime import datetime -@deprecated(reason="Use DagRun().get_state() instead", version="2.2.4") +@deprecated(reason="Use DagRun().get_state() instead", version="2.2.4", category=RemovedInAirflow3Warning) def get_dag_run_state(dag_id: str, execution_date: datetime) -> dict[str, str]: """Return the Dag Run state identified by the given dag_id and execution_date. diff --git a/airflow/api/common/experimental/get_task.py b/airflow/api/common/experimental/get_task.py index 6bc58d0328..7e25ff4ffd 100644 --- a/airflow/api/common/experimental/get_task.py +++ b/airflow/api/common/experimental/get_task.py @@ -24,12 +24,13 @@ from typing import TYPE_CHECKING from deprecated import deprecated from airflow.api.common.experimental import check_and_get_dag +from airflow.exceptions import RemovedInAirflow3Warning if TYPE_CHECKING: from airflow.models import TaskInstance -@deprecated(reason="Use DAG().get_task", version="2.2.4") +@deprecated(reason="Use DAG().get_task", version="2.2.4", category=RemovedInAirflow3Warning) def get_task(dag_id: str, task_id:
(airflow) 30/48: [docs] update `DagBag` class docstring to include all params (#38814)
This is an automated email from the ASF dual-hosted git repository. jedcunningham pushed a commit to branch v2-9-test in repository https://gitbox.apache.org/repos/asf/airflow.git commit 672d47ebf062b9b152ff407289db9959c9579db5 Author: Kalyan AuthorDate: Tue Apr 16 17:52:57 2024 +0530 [docs] update `DagBag` class docstring to include all params (#38814) * update docstring for DagBag class * break long line * fix space Signed-off-by: kalyanr - Signed-off-by: kalyanr (cherry picked from commit b00274156b28cce6a68dac20e667d21d00310016) --- airflow/models/dagbag.py | 5 + 1 file changed, 5 insertions(+) diff --git a/airflow/models/dagbag.py b/airflow/models/dagbag.py index ce9bf5587b..1902c985be 100644 --- a/airflow/models/dagbag.py +++ b/airflow/models/dagbag.py @@ -89,11 +89,16 @@ class DagBag(LoggingMixin): :param dag_folder: the folder to scan to find DAGs :param include_examples: whether to include the examples that ship with airflow or not +:param safe_mode: when ``False``, scans all python modules for dags. +When ``True`` uses heuristics (files containing ``DAG`` and ``airflow`` strings) +to filter python modules to scan for dags. :param read_dags_from_db: Read DAGs from DB if ``True`` is passed. If ``False`` DAGs are read from python files. +:param store_serialized_dags: deprecated parameter, same effect as `read_dags_from_db` :param load_op_links: Should the extra operator link be loaded via plugins when de-serializing the DAG? This flag is set to False in Scheduler so that Extra Operator links are not loaded to not run User code in Scheduler. +:param collect_dags: when True, collects dags during class initialization. """ def __init__(
(airflow) 48/48: Fix trigger kwarg encryption migration (#39246)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch v2-9-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit c9cc72608d66b1481ef1029499ce54e30b178b12
Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com>
AuthorDate: Thu Apr 25 15:20:40 2024 -0400
Fix trigger kwarg encryption migration (#39246)
Do the encryption in the migration itself, and fix support for offline
migrations as well.
The offline up migration won't actually encrypt the trigger kwargs as there
isn't a safe way to accomplish that, so the decryption processes checks
and short circuits if it isn't encrypted.
The offline down migration will now print out a warning that the offline
migration will fail if there are any running triggers. I think this is
the best we can do for that scenario (and folks willing to do offline
migrations will hopefully be able to understand the situation).
This also solves the "encrypting the already encrypted kwargs" bug in
2.9.0.
(cherry picked from commit adeb7f7cba2ab2b16be2e006c17e140fe91fdf77)
---
.../0140_2_9_0_update_trigger_kwargs_type.py | 46 +++---
airflow/models/trigger.py | 10 -
airflow/utils/db.py| 39 --
docs/apache-airflow/img/airflow_erd.sha256 | 2 +-
docs/apache-airflow/img/airflow_erd.svg| 4 +-
docs/apache-airflow/migrations-ref.rst | 2 +-
tests/models/test_trigger.py | 17
7 files changed, 70 insertions(+), 50 deletions(-)
diff --git
a/airflow/migrations/versions/0140_2_9_0_update_trigger_kwargs_type.py
b/airflow/migrations/versions/0140_2_9_0_update_trigger_kwargs_type.py
index dbde1201e4..2d57686e43 100644
--- a/airflow/migrations/versions/0140_2_9_0_update_trigger_kwargs_type.py
+++ b/airflow/migrations/versions/0140_2_9_0_update_trigger_kwargs_type.py
@@ -16,18 +16,22 @@
# specific language governing permissions and limitations
# under the License.
-"""update trigger kwargs type
+"""update trigger kwargs type and encrypt
Revision ID: 1949afb29106
Revises: ee1467d4aa35
Create Date: 2024-03-17 22:09:09.406395
"""
+import json
+from textwrap import dedent
+
+from alembic import context, op
import sqlalchemy as sa
+from sqlalchemy.orm import lazyload
+from airflow.serialization.serialized_objects import BaseSerialization
from airflow.models.trigger import Trigger
-from alembic import op
-
from airflow.utils.sqlalchemy import ExtendedJSON
# revision identifiers, used by Alembic.
@@ -38,13 +42,43 @@ depends_on = None
airflow_version = "2.9.0"
+def get_session() -> sa.orm.Session:
+conn = op.get_bind()
+sessionmaker = sa.orm.sessionmaker()
+return sessionmaker(bind=conn)
+
def upgrade():
-"""Update trigger kwargs type to string"""
+"""Update trigger kwargs type to string and encrypt"""
with op.batch_alter_table("trigger") as batch_op:
batch_op.alter_column("kwargs", type_=sa.Text(), )
+if not context.is_offline_mode():
+session = get_session()
+try:
+for trigger in
session.query(Trigger).options(lazyload(Trigger.task_instance)):
+trigger.kwargs = trigger.kwargs
+session.commit()
+finally:
+session.close()
+
def downgrade():
-"""Unapply update trigger kwargs type to string"""
+"""Unapply update trigger kwargs type to string and encrypt"""
+if context.is_offline_mode():
+print(dedent("""
+
+-- WARNING: Unable to decrypt trigger kwargs automatically in offline
mode!
+-- If any trigger rows exist when you do an offline downgrade, the
migration will fail.
+
+"""))
+else:
+session = get_session()
+try:
+for trigger in
session.query(Trigger).options(lazyload(Trigger.task_instance)):
+trigger.encrypted_kwargs =
json.dumps(BaseSerialization.serialize(trigger.kwargs))
+session.commit()
+finally:
+session.close()
+
with op.batch_alter_table("trigger") as batch_op:
-batch_op.alter_column("kwargs", type_=ExtendedJSON(),
postgresql_using="kwargs::json")
+batch_op.alter_column("kwargs", type_=ExtendedJSON(),
postgresql_using='kwargs::json')
diff --git a/airflow/models/trigger.py b/airflow/models/trigger.py
index 32fec3b343..cde76ca241 100644
--- a/airflow/models/trigger.py
+++ b/airflow/models/trigger.py
@@ -116,7 +116,15 @@ class Trigger(Base):
fro
(airflow) 41/48: Use grid view for Task Instance's `log_url` (#39183)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch v2-9-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 7cd311dc78a82c057b971912fdb4d70457e06638
Author: Madison Swain-Bowden
AuthorDate: Wed Apr 24 09:41:43 2024 -0700
Use grid view for Task Instance's `log_url` (#39183)
* Use grid view for Task Instance's `log_url`
* Update other tests
(cherry picked from commit 6db6fef357f2adadf94956238fd14eb34aebfe34)
---
airflow/models/taskinstance.py | 10 ++
tests/models/test_taskinstance.py | 9 +
tests/providers/smtp/notifications/test_smtp.py | 2 +-
3 files changed, 12 insertions(+), 9 deletions(-)
diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py
index 88a495e9eb..42824fa586 100644
--- a/airflow/models/taskinstance.py
+++ b/airflow/models/taskinstance.py
@@ -1719,15 +1719,17 @@ class TaskInstance(Base, LoggingMixin):
@property
def log_url(self) -> str:
"""Log URL for TaskInstance."""
-iso = quote(self.execution_date.isoformat())
+run_id = quote(self.run_id)
base_url = conf.get_mandatory_value("webserver", "BASE_URL")
return (
f"{base_url}"
-"/log"
-f"?execution_date={iso}"
+f"/dags"
+f"/{self.dag_id}"
+f"/grid"
+f"?dag_run_id={run_id}"
f"_id={self.task_id}"
-f"_id={self.dag_id}"
f"_index={self.map_index}"
+"=logs"
)
@property
diff --git a/tests/models/test_taskinstance.py
b/tests/models/test_taskinstance.py
index 120856dbbf..d1cb500dec 100644
--- a/tests/models/test_taskinstance.py
+++ b/tests/models/test_taskinstance.py
@@ -1981,14 +1981,15 @@ class TestTaskInstance:
assert 1 == tis2[("task_3",
0)].get_num_running_task_instances(session=session, same_dagrun=True)
def test_log_url(self, create_task_instance):
-ti = create_task_instance(dag_id="dag", task_id="op",
execution_date=timezone.datetime(2018, 1, 1))
+ti = create_task_instance(dag_id="my_dag", task_id="op",
execution_date=timezone.datetime(2018, 1, 1))
expected_url = (
-"http://localhost:8080/log?;
-"execution_date=2018-01-01T00%3A00%3A00%2B00%3A00"
+"http://localhost:8080;
+"/dags/my_dag/grid"
+"?dag_run_id=test"
"_id=op"
-"_id=dag"
"_index=-1"
+"=logs"
)
assert ti.log_url == expected_url
diff --git a/tests/providers/smtp/notifications/test_smtp.py
b/tests/providers/smtp/notifications/test_smtp.py
index 9878d99786..f1a71e2b53 100644
--- a/tests/providers/smtp/notifications/test_smtp.py
+++ b/tests/providers/smtp/notifications/test_smtp.py
@@ -129,7 +129,7 @@ class TestSmtpNotifier:
from_email=conf.get("smtp", "smtp_mail_from"),
to="test_reci...@test.com",
subject="DAG dag - Task op - Run ID test in State None",
-html_content="""\n\n\n\n\n\n\n\n\n\nRun ID:\n
test\n\n\nTry:\n
1 of 1\n\n\nTask
State:\n [...]
+html_content="""\n\n\n\n\n\n\n\n\n\nRun ID:\n
test\n\n\nTry:\n
1 of 1\n\n\nTask
State:\n [...]
smtp_conn_id="smtp_default",
files=None,
cc=None,
