[airflow] 17/20: Ensure execution_timeout as timedelta (#23655)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 07d0bee056cb925531dc6d4ad73215fb183d0a40
Author: Ping Zhang
AuthorDate: Mon May 16 11:53:58 2022 -0700
Ensure execution_timeout as timedelta (#23655)
(cherry picked from commit 6f82fc70ec91b493924249f062306330ee929728)
---
airflow/models/baseoperator.py| 6 ++
tests/models/test_baseoperator.py | 11 +++
2 files changed, 17 insertions(+)
diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py
index 4b3ccdde8b..2fea760716 100644
--- a/airflow/models/baseoperator.py
+++ b/airflow/models/baseoperator.py
@@ -767,7 +767,13 @@ class BaseOperator(AbstractOperator,
metaclass=BaseOperatorMeta):
self.email = email
self.email_on_retry = email_on_retry
self.email_on_failure = email_on_failure
+
+if execution_timeout is not None and not isinstance(execution_timeout,
timedelta):
+raise ValueError(
+f'execution_timeout must be timedelta object but passed as
type: {type(execution_timeout)}'
+)
self.execution_timeout = execution_timeout
+
self.on_execute_callback = on_execute_callback
self.on_failure_callback = on_failure_callback
self.on_success_callback = on_success_callback
diff --git a/tests/models/test_baseoperator.py
b/tests/models/test_baseoperator.py
index 1d81104547..8cb8d96e81 100644
--- a/tests/models/test_baseoperator.py
+++ b/tests/models/test_baseoperator.py
@@ -119,6 +119,17 @@ class TestBaseOperator:
with pytest.raises(AirflowException, match="missing keyword argument
'test_sub_param'"):
DummySubClass(default_args=default_args)
+def test_execution_timeout_type(self):
+with pytest.raises(
+ValueError, match="execution_timeout must be timedelta object but
passed as type: "
+):
+BaseOperator(task_id='test', execution_timeout="1")
+
+with pytest.raises(
+ValueError, match="execution_timeout must be timedelta object but
passed as type: "
+):
+BaseOperator(task_id='test', execution_timeout=1)
+
def test_incorrect_default_args(self):
default_args = {'test_param': True, 'extra_param': True}
dummy_class = DummyClass(default_args=default_args)
[airflow] 08/20: Prevent KubernetesJobWatcher getting stuck on resource too old (#23521)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 3c997781d576deda679489d2dc7df9f0bbd38816
Author: Ruben Laguna
AuthorDate: Wed May 11 08:25:49 2022 +0200
Prevent KubernetesJobWatcher getting stuck on resource too old (#23521)
* Prevent KubernetesJobWatcher getting stuck on resource too old
If the watch fails because "resource too old" the
KubernetesJobWatcher should not retry with the same resource version
as that will end up in loop where there is no progress.
* Reset ResourceVersion().resource_version to 0
(cherry picked from commit dee05b2ebca6ab66f1b447837e11fe204f98b2df)
---
airflow/executors/kubernetes_executor.py| 3 +++
tests/executors/test_kubernetes_executor.py | 34 +
2 files changed, 37 insertions(+)
diff --git a/airflow/executors/kubernetes_executor.py
b/airflow/executors/kubernetes_executor.py
index 9b9de71681..c76cf58f41 100644
--- a/airflow/executors/kubernetes_executor.py
+++ b/airflow/executors/kubernetes_executor.py
@@ -109,6 +109,8 @@ class KubernetesJobWatcher(multiprocessing.Process,
LoggingMixin):
time.sleep(1)
except Exception:
self.log.exception('Unknown error in KubernetesJobWatcher.
Failing')
+self.resource_version = "0"
+ResourceVersion().resource_version = "0"
raise
else:
self.log.warning(
@@ -288,6 +290,7 @@ class AirflowKubernetesScheduler(LoggingMixin):
self.log.error(
'Error while health checking kube watcher process. Process
died for unknown reasons'
)
+ResourceVersion().resource_version = "0"
self.kube_watcher = self._make_kube_watcher()
def run_next(self, next_job: KubernetesJobType) -> None:
diff --git a/tests/executors/test_kubernetes_executor.py
b/tests/executors/test_kubernetes_executor.py
index 1a4160d9ce..954f4f0600 100644
--- a/tests/executors/test_kubernetes_executor.py
+++ b/tests/executors/test_kubernetes_executor.py
@@ -39,6 +39,7 @@ try:
AirflowKubernetesScheduler,
KubernetesExecutor,
KubernetesJobWatcher,
+ResourceVersion,
create_pod_id,
get_base_pod_from_template,
)
@@ -957,3 +958,36 @@ class TestKubernetesJobWatcher(unittest.TestCase):
f"Kubernetes failure for {raw_object['reason']} "
f"with code {raw_object['code']} and message:
{raw_object['message']}"
)
+
+def test_recover_from_resource_too_old(self):
+# too old resource
+mock_underscore_run = mock.MagicMock()
+
+def effect():
+yield '500'
+while True:
+yield Exception('sentinel')
+
+mock_underscore_run.side_effect = effect()
+
+self.watcher._run = mock_underscore_run
+
+with
mock.patch('airflow.executors.kubernetes_executor.get_kube_client'):
+try:
+# self.watcher._run() is mocked and return "500" as last
resource_version
+self.watcher.run()
+except Exception as e:
+assert e.args == ('sentinel',)
+
+# both resource_version should be 0 after _run raises and
exception
+assert self.watcher.resource_version == '0'
+assert ResourceVersion().resource_version == '0'
+
+# check that in the next run, _run is invoked with
resource_version = 0
+mock_underscore_run.reset_mock()
+try:
+self.watcher.run()
+except Exception as e:
+assert e.args == ('sentinel',)
+
+mock_underscore_run.assert_called_once_with(mock.ANY, '0',
mock.ANY, mock.ANY)
[airflow] 10/20: Fix `PythonVirtualenvOperator` templated_fields (#23559)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 38b9c7c19b7dadef03f4ef0b6a24baf78273f20c
Author: eladkal <45845474+elad...@users.noreply.github.com>
AuthorDate: Mon May 9 18:17:34 2022 +0300
Fix `PythonVirtualenvOperator` templated_fields (#23559)
* Fix `PythonVirtualenvOperator` templated_fields
The `PythonVirtualenvOperator` templated_fields override `PythonOperator`
templated_fields which caused functionality not to work as expected.
fixes: https://github.com/apache/airflow/issues/23557
(cherry picked from commit 1657bd2827a3299a91ae0abbbfe4f6b80bd4cdc0)
---
airflow/operators/python.py| 3 ++-
tests/operators/test_python.py | 3 +++
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/airflow/operators/python.py b/airflow/operators/python.py
index 1995992809..920a75d08f 100644
--- a/airflow/operators/python.py
+++ b/airflow/operators/python.py
@@ -327,7 +327,8 @@ class PythonVirtualenvOperator(PythonOperator):
processing templated fields, for examples ``['.sql', '.hql']``
"""
-template_fields: Sequence[str] = ('requirements',)
+template_fields: Sequence[str] = tuple({'requirements'} |
set(PythonOperator.template_fields))
+
template_ext: Sequence[str] = ('.txt',)
BASE_SERIALIZABLE_CONTEXT_KEYS = {
'ds',
diff --git a/tests/operators/test_python.py b/tests/operators/test_python.py
index bf001c0edf..e58b424de6 100644
--- a/tests/operators/test_python.py
+++ b/tests/operators/test_python.py
@@ -868,6 +868,9 @@ class TestPythonVirtualenvOperator(unittest.TestCase):
task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
return task
+def test_template_fields(self):
+assert
set(PythonOperator.template_fields).issubset(PythonVirtualenvOperator.template_fields)
+
def test_add_dill(self):
def f():
"""Ensure dill is correctly installed."""
[airflow] branch v2-3-test updated (5e0c9896d2 -> 1c37a96fb5)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a change to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
from 5e0c9896d2 Add typing for airflow/configuration.py (#23716)
new 71a0f0d973 Move around overflow, position and padding (#23044)
new af22a53393 Pools with negative open slots should not block other pools
(#23143)
new 540beb6f2b Handle invalid date parsing in webserver views. (#23161)
new 9e3a9ec222 Fix: Exception when parsing log #20966 (#23301)
new 25ff87d904 Move dag_processing.processor_timeouts to counters section
(#23393)
new b369085409 Fix broken dagrun links when many runs start at the same
time (#23462)
new cceccf27b2 Fix scheduler crash when expanding with mapped task that
returned none (#23486)
new 3c997781d5 Prevent KubernetesJobWatcher getting stuck on resource too
old (#23521)
new e6b41e0198 Apply specific ID collation to root_dag_id too (#23536)
new 38b9c7c19b Fix `PythonVirtualenvOperator` templated_fields (#23559)
new 624c401b2e Update dags.rst (#23579)
new 6d003ddbbb Add slim images to docker-stack docs index (#23601)
new 8e35daec67 Implement send_callback method for CeleryKubernetesExecutor
and LocalKubernetesExecutor (#23617)
new acad83ca8c Add index for event column in log table (#23625)
new f62101bd41 Fix typo issue (#23633)
new d836cd141d Don't run pre-migration checks for downgrade (#23634)
new 07d0bee056 Ensure execution_timeout as timedelta (#23655)
new e66e813cfe remove `--` in `./breeze build-docs` command (#23671)
new c0b9d26b9e Fix grid details header text overlap (#23728)
new 1c37a96fb5 Remove titles from link buttons (#23736)
The 20 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
airflow/executors/celery_kubernetes_executor.py| 12 +++
airflow/executors/kubernetes_executor.py | 3 +
airflow/executors/local_kubernetes_executor.py | 12 +++
airflow/hooks/subprocess.py| 2 +-
airflow/jobs/scheduler_job.py | 2 +-
.../0045_b3b105409875_add_root_dag_id_to_dag.py| 4 +-
...109_1de7bc13c950_add_index_for_event_in_log.py} | 23 +++---
airflow/models/baseoperator.py | 6 ++
airflow/models/log.py | 5 +-
airflow/models/pool.py | 6 +-
airflow/models/taskinstance.py | 4 +-
airflow/operators/python.py| 3 +-
.../providers/cncf/kubernetes/utils/pod_manager.py | 5 +-
airflow/utils/db.py| 10 ---
airflow/www/static/css/graph.css | 19 +
airflow/www/static/js/graph.js | 2 +-
airflow/www/static/js/grid/details/Header.jsx | 8 +-
airflow/www/templates/airflow/dag.html | 4 +-
airflow/www/views.py | 58 +
docs/README.rst| 8 +-
docs/apache-airflow/concepts/dags.rst | 2 +-
docs/apache-airflow/howto/set-up-database.rst | 2 +-
docs/apache-airflow/logging-monitoring/metrics.rst | 2 +-
docs/apache-airflow/migrations-ref.rst | 4 +-
docs/docker-stack/index.rst| 11 ++-
tests/executors/test_celery_kubernetes_executor.py | 12 +++
tests/executors/test_kubernetes_executor.py| 34
tests/executors/test_local_kubernetes_executor.py | 12 +++
tests/hooks/test_subprocess.py | 6 ++
tests/jobs/test_scheduler_job.py | 36
tests/models/test_baseoperator.py | 11 +++
tests/models/test_taskinstance.py | 18
tests/operators/test_python.py | 3 +
tests/www/views/test_views.py | 57 +
tests/www/views/test_views_graph_gantt.py | 96 +-
35 files changed, 433 insertions(+), 69 deletions(-)
copy
airflow/migrations/versions/{0021_5e7d17757c7a_add_pid_field_to_taskinstance.py
=> 0109_1de7bc13c950_add_index_for_event_in_log.py} (66%)
[airflow] 01/20: Move around overflow, position and padding (#23044)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 71a0f0d97316607e872c22b521c6f8f5dc61f3bf
Author: Brent Bovenzi
AuthorDate: Thu May 12 15:47:24 2022 -0400
Move around overflow, position and padding (#23044)
(cherry picked from commit 028087b5a6e94fd98542d0e681d947979eb1011f)
---
airflow/www/static/css/graph.css | 19 +++
airflow/www/static/js/graph.js | 2 +-
2 files changed, 20 insertions(+), 1 deletion(-)
diff --git a/airflow/www/static/css/graph.css b/airflow/www/static/css/graph.css
index ccf353b53e..5d30cb43e4 100644
--- a/airflow/www/static/css/graph.css
+++ b/airflow/www/static/css/graph.css
@@ -125,6 +125,25 @@ g.node text {
border-radius: 4px;
background-color: #f0f0f0;
cursor: move;
+ overflow: hidden;
+ height: 100%;
+ width: 100%;
+}
+
+#graph-svg {
+ overflow: visible;
+}
+
+.refresh-actions {
+ justify-content: flex-end;
+ min-width: 225px;
+ display: inline-flex;
+ align-items: center;
+ right: 20px;
+ margin-top: 10px;
+ margin-bottom: 15px;
+ position: absolute;
+ background-color: #f0f0f0ee; /* the last two chars apply an opacity to the
background color */
}
.legend-item.dag {
diff --git a/airflow/www/static/js/graph.js b/airflow/www/static/js/graph.js
index a205593467..d45557a3f6 100644
--- a/airflow/www/static/js/graph.js
+++ b/airflow/www/static/js/graph.js
@@ -246,7 +246,7 @@ function setUpZoomSupport() {
const graphWidth = g.graph().width;
const graphHeight = g.graph().height;
// Get SVG dimensions
- const padding = 20;
+ const padding = 80;
const svgBb = svg.node().getBoundingClientRect();
const width = svgBb.width - padding * 2;
const height = svgBb.height - padding; // we are not centering the dag
vertically
[airflow] 05/20: Move dag_processing.processor_timeouts to counters section (#23393)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git commit 25ff87d904b2ab519e77ce83439b47d9a2a71711 Author: Yeachan Park AuthorDate: Sun May 8 23:11:51 2022 +0200 Move dag_processing.processor_timeouts to counters section (#23393) (cherry picked from commit fcfaa8307ac410283f1270a0df9e557570e5ffd3) --- docs/apache-airflow/logging-monitoring/metrics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-airflow/logging-monitoring/metrics.rst b/docs/apache-airflow/logging-monitoring/metrics.rst index f0bc75bb49..454f08507f 100644 --- a/docs/apache-airflow/logging-monitoring/metrics.rst +++ b/docs/apache-airflow/logging-monitoring/metrics.rst @@ -92,6 +92,7 @@ NameDescription ``zombies_killed`` Zombie tasks killed ``scheduler_heartbeat`` Scheduler heartbeats ``dag_processing.processes``Number of currently running DAG parsing processes +``dag_processing.processor_timeouts`` Number of file processors that have been killed due to taking too long ``dag_processing.manager_stalls`` Number of stalled ``DagFileProcessorManager`` ``dag_file_refresh_error`` Number of failures loading any DAG files ``scheduler.tasks.killed_externally`` Number of tasks killed externally @@ -129,7 +130,6 @@ Name Description ``dag_processing.import_errors``Number of errors from trying to parse DAG files ``dag_processing.total_parse_time`` Seconds taken to scan and import all DAG files once ``dag_processing.last_run.seconds_ago.`` Seconds since was last processed -``dag_processing.processor_timeouts`` Number of file processors that have been killed due to taking too long ``scheduler.tasks.running`` Number of tasks running in executor ``scheduler.tasks.starving``Number of tasks that cannot be scheduled because of no open slot in pool ``scheduler.tasks.executable`` Number of tasks that are ready for execution (set to queued)
[airflow] 11/20: Update dags.rst (#23579)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git commit 624c401b2eea186422ec51ffd96f7858e8eb637e Author: mthakare-onshape <56147818+mthakare-onsh...@users.noreply.github.com> AuthorDate: Mon May 9 13:47:12 2022 +0530 Update dags.rst (#23579) Update missing bracket (cherry picked from commit 827bfda59b7a0db6ada697ccd01c739d37430b9a) --- docs/apache-airflow/concepts/dags.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-airflow/concepts/dags.rst b/docs/apache-airflow/concepts/dags.rst index 222b3847b7..3f1d9cc64b 100644 --- a/docs/apache-airflow/concepts/dags.rst +++ b/docs/apache-airflow/concepts/dags.rst @@ -791,7 +791,7 @@ via UI and API. Paused DAG is not scheduled by the Scheduler, but you can trigge manual runs. In the UI, you can see Paused DAGs (in ``Paused`` tab). The DAGs that are un-paused can be found in the ``Active`` tab. -Dag can be deactivated (do not confuse it with ``Active`` tag in the UI by removing them from the +Dag can be deactivated (do not confuse it with ``Active`` tag in the UI) by removing them from the ``DAGS_FOLDER``. When scheduler parses the ``DAGS_FOLDER`` and misses the DAG that it had seen before and stored in the database it will set is as deactivated. The metadata and history of the DAG` is kept for deactivated DAGs and when the DAG is re-added to the ``DAGS_FOLDER`` it will be again
[airflow] 07/20: Fix scheduler crash when expanding with mapped task that returned none (#23486)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit cceccf27b281f9da9d8701ddad78ac8aeba6ab55
Author: Ephraim Anierobi
AuthorDate: Mon May 9 13:44:35 2022 +0100
Fix scheduler crash when expanding with mapped task that returned none
(#23486)
When task is expanded from a mapped task that returned no value, it
crashes the scheduler. This PR fixes it by first checking if there's
a return value from the mapped task, if no returned value, then error
in the task itself instead of crashing the scheduler
(cherry picked from commit 7813f996ab79b9e6ef07090194f1e621e4f90e17)
---
airflow/models/taskinstance.py| 4 +++-
tests/models/test_taskinstance.py | 18 ++
2 files changed, 21 insertions(+), 1 deletion(-)
diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py
index de85b4cf8c..e1e87ec8e8 100644
--- a/airflow/models/taskinstance.py
+++ b/airflow/models/taskinstance.py
@@ -2329,10 +2329,12 @@ class TaskInstance(Base, LoggingMixin):
# currently possible for a downstream to depend on one individual
mapped
# task instance, only a task as a whole. This will change in AIP-42
# Phase 2, and we'll need to further analyze the mapped task case.
-if task.is_mapped or next(task.iter_mapped_dependants(), None) is None:
+if next(task.iter_mapped_dependants(), None) is None:
return
if value is None:
raise XComForMappingNotPushed()
+if task.is_mapped:
+return
if not isinstance(value, collections.abc.Collection) or
isinstance(value, (bytes, str)):
raise UnmappableXComTypePushed(value)
task_map = TaskMap.from_task_instance_xcom(self, value)
diff --git a/tests/models/test_taskinstance.py
b/tests/models/test_taskinstance.py
index dac987e431..57355a3ad5 100644
--- a/tests/models/test_taskinstance.py
+++ b/tests/models/test_taskinstance.py
@@ -2831,3 +2831,21 @@ def test_ti_mapped_depends_on_mapped_xcom_arg(dag_maker,
session):
query = XCom.get_many(run_id=dagrun.run_id, task_ids=["add_one__1"],
session=session)
assert [x.value for x in query.order_by(None).order_by(XCom.map_index)] ==
[3, 4, 5]
+
+
+def test_ti_mapped_depends_on_mapped_xcom_arg_XXX(dag_maker, session):
+with dag_maker(session=session) as dag:
+
+@dag.task
+def add_one(x):
+x + 1
+
+two_three_four = add_one.expand(x=[1, 2, 3])
+add_one.expand(x=two_three_four)
+
+dagrun = dag_maker.create_dagrun()
+for map_index in range(3):
+ti = dagrun.get_task_instance("add_one", map_index=map_index)
+ti.refresh_from_task(dag.get_task("add_one"))
+with pytest.raises(XComForMappingNotPushed):
+ti.run()
[airflow] 20/20: Remove titles from link buttons (#23736)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git commit 1c37a96fb5b64800668a28ab944c6a77e79a09e6 Author: Brent Bovenzi AuthorDate: Mon May 16 22:58:56 2022 -0400 Remove titles from link buttons (#23736) (cherry picked from commit 239a9dce5b97d45620862b42fd9018fdc9d6d505) --- airflow/www/templates/airflow/dag.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/www/templates/airflow/dag.html b/airflow/www/templates/airflow/dag.html index ee7dd64994..c43af805c4 100644 --- a/airflow/www/templates/airflow/dag.html +++ b/airflow/www/templates/airflow/dag.html @@ -247,10 +247,10 @@ XCom - + List Instances, current run - + List Instances, all runs
[airflow] 04/20: Fix: Exception when parsing log #20966 (#23301)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 9e3a9ec222dd2e1c61d2862b82d6122f1ac0e49f
Author: Jakub Novák
AuthorDate: Tue May 10 22:43:25 2022 +0200
Fix: Exception when parsing log #20966 (#23301)
* UnicodeDecodeError: 'utf-8' codec can't decode byte 0xXX in position X:
invalid start byte
File
"/opt/work/python395/lib/python3.9/site-packages/airflow/hooks/subprocess.py",
line 89, in run_command
line = raw_line.decode(output_encoding).rstrip()# raw_line
== b'\x00\x00\x00\x11\xa9\x01\n'
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xa9 in position 4:
invalid start byte
* Update subprocess.py
* Update subprocess.py
* Fix: Exception when parsing log #20966
* Fix: Exception when parsing log #20966
Another alternative is: try-catch it.
e.g.
```
line = ''
for raw_line in iter(self.sub_process.stdout.readline, b''):
try:
line = raw_line.decode(output_encoding).rstrip()
except UnicodeDecodeError as err:
print(err, output_encoding, raw_line)
self.log.info("%s", line)
```
* Create test_subprocess.sh
* Update test_subprocess.py
* Added shell directive and license to test_subprocess.sh
* Distinguish between raw and decoded lines as suggested by @uranusjr
* simplify test
Co-authored-by: muhua
(cherry picked from commit 863b2576423e1a7933750b297a9b4518ae598db9)
---
airflow/hooks/subprocess.py| 2 +-
airflow/providers/cncf/kubernetes/utils/pod_manager.py | 5 +++--
tests/hooks/test_subprocess.py | 6 ++
3 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/airflow/hooks/subprocess.py b/airflow/hooks/subprocess.py
index c814b0528d..fa8c706c69 100644
--- a/airflow/hooks/subprocess.py
+++ b/airflow/hooks/subprocess.py
@@ -88,7 +88,7 @@ class SubprocessHook(BaseHook):
raise RuntimeError("The subprocess should be created here and
is None!")
if self.sub_process.stdout is not None:
for raw_line in iter(self.sub_process.stdout.readline, b''):
-line = raw_line.decode(output_encoding).rstrip()
+line = raw_line.decode(output_encoding,
errors='backslashreplace').rstrip()
self.log.info("%s", line)
self.sub_process.wait()
diff --git a/airflow/providers/cncf/kubernetes/utils/pod_manager.py
b/airflow/providers/cncf/kubernetes/utils/pod_manager.py
index 993ba12e31..4d6fbaa4c0 100644
--- a/airflow/providers/cncf/kubernetes/utils/pod_manager.py
+++ b/airflow/providers/cncf/kubernetes/utils/pod_manager.py
@@ -220,8 +220,9 @@ class PodManager(LoggingMixin):
),
follow=follow,
)
-for line in logs:
-timestamp, message =
self.parse_log_line(line.decode('utf-8'))
+for raw_line in logs:
+line = raw_line.decode('utf-8', errors="backslashreplace")
+timestamp, message = self.parse_log_line(line)
self.log.info(message)
except BaseHTTPError as e:
self.log.warning(
diff --git a/tests/hooks/test_subprocess.py b/tests/hooks/test_subprocess.py
index 642b219c01..10d9b00970 100644
--- a/tests/hooks/test_subprocess.py
+++ b/tests/hooks/test_subprocess.py
@@ -96,3 +96,9 @@ class TestSubprocessHook(unittest.TestCase):
stderr=STDOUT,
stdout=PIPE,
)
+
+def test_task_decode(self):
+hook = SubprocessHook()
+command = ['bash', '-c', 'printf "This will cause a coding error
\\xb1\\xa6\\x01\n"']
+result = hook.run_command(command=command)
+assert result.exit_code == 0
[airflow] 15/20: Fix typo issue (#23633)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git commit f62101bd412128f79fdf86c3476a8aa6102d6b9e Author: humit AuthorDate: Wed May 11 19:58:26 2022 +0900 Fix typo issue (#23633) (cherry picked from commit f313e144f751cc52ea144bf9dd99b1c85f15d469) --- docs/apache-airflow/howto/set-up-database.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-airflow/howto/set-up-database.rst b/docs/apache-airflow/howto/set-up-database.rst index 563d2efc6b..a83948296d 100644 --- a/docs/apache-airflow/howto/set-up-database.rst +++ b/docs/apache-airflow/howto/set-up-database.rst @@ -45,7 +45,7 @@ Database URI Airflow uses SQLAlchemy to connect to the database, which requires you to configure the Database URL. You can do this in option ``sql_alchemy_conn`` in section ``[database]``. It is also common to configure -this option with ``AIRFLOW__DATABE__SQL_ALCHEMY_CONN`` environment variable. +this option with ``AIRFLOW__DATABASE__SQL_ALCHEMY_CONN`` environment variable. .. note:: For more information on setting the configuration, see :doc:`/howto/set-config`.
[airflow] 09/20: Apply specific ID collation to root_dag_id too (#23536)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit e6b41e0198009965506570e793753fae3daaca86
Author: Michael Peteuil
AuthorDate: Mon May 9 13:48:11 2022 -0400
Apply specific ID collation to root_dag_id too (#23536)
In certain databases there is a need to set the collation for ID fields
like dag_id or task_id to something different than the database default.
This is because in MySQL with utf8mb4 the index size becomes too big for
the MySQL limits. In past pull requests this was handled
[#7570](https://github.com/apache/airflow/pull/7570),
[#17729](https://github.com/apache/airflow/pull/17729), but the
root_dag_id field on the dag model was missed. Since this field is used
to join with the dag_id in various other models ([and
self-referentially](https://github.com/apache/airflow/blob/451c7cbc42a83a180c4362693508ed33dd1d1dab/airflow/models/dag.py#L2766)),
it also needs to have the same collation as other ID fields.
This can be seen by running `airflow db reset` before and after applying
this change while also specifying `sql_engine_collation_for_ids` in the
configuration.
Other related PRs
[#19408](https://github.com/apache/airflow/pull/19408)
(cherry picked from commit b7f862760ce4e20a284bb1933c9078e54851c518)
---
.../migrations/versions/0045_b3b105409875_add_root_dag_id_to_dag.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git
a/airflow/migrations/versions/0045_b3b105409875_add_root_dag_id_to_dag.py
b/airflow/migrations/versions/0045_b3b105409875_add_root_dag_id_to_dag.py
index 3d184e6b9b..f879450369 100644
--- a/airflow/migrations/versions/0045_b3b105409875_add_root_dag_id_to_dag.py
+++ b/airflow/migrations/versions/0045_b3b105409875_add_root_dag_id_to_dag.py
@@ -27,6 +27,8 @@ Create Date: 2019-09-28 23:20:01.744775
import sqlalchemy as sa
from alembic import op
+from airflow.migrations.db_types import StringID
+
# revision identifiers, used by Alembic.
revision = 'b3b105409875'
down_revision = 'd38e04c12aa2'
@@ -37,7 +39,7 @@ airflow_version = '1.10.7'
def upgrade():
"""Apply Add ``root_dag_id`` to ``DAG``"""
-op.add_column('dag', sa.Column('root_dag_id', sa.String(length=250),
nullable=True))
+op.add_column('dag', sa.Column('root_dag_id', StringID(), nullable=True))
op.create_index('idx_root_dag_id', 'dag', ['root_dag_id'], unique=False)
[airflow] branch v2-3-test updated (4dd629a66f -> 4ab319672f)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git from 4dd629a66f Update version to 2.3.1 and add release notes add 4ab319672f Apply suggestions from code review No new revisions were added by this update. Summary of changes: RELEASE_NOTES.rst | 4 1 file changed, 4 deletions(-)
[airflow] branch v2-3-test updated (4ab319672f -> e3c1bf4934)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git omit 4ab319672f Apply suggestions from code review omit 4dd629a66f Update version to 2.3.1 and add release notes add e3c1bf4934 Update version to 2.3.1 and add release notes This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (4ab319672f) \ N -- N -- N refs/heads/v2-3-test (e3c1bf4934) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: RELEASE_NOTES.rst | 1 - 1 file changed, 1 deletion(-)
[airflow] branch v2-3-test updated (84f03e2e64 -> 1e3e4dc58b)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git omit 84f03e2e64 Update version to 2.3.1 and add release notes add 1e3e4dc58b Update version to 2.3.1 and add release notes This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (84f03e2e64) \ N -- N -- N refs/heads/v2-3-test (1e3e4dc58b) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: RELEASE_NOTES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[airflow] branch main updated: Fix secrets rendered in UI when task is not executed. (#22754)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new ce8ea66918 Fix secrets rendered in UI when task is not executed.
(#22754)
ce8ea66918 is described below
commit ce8ea6691820140a0e2d9a5dad5254bc05a5a270
Author: Karthikeyan Singaravelan
AuthorDate: Fri May 20 19:37:47 2022 +0530
Fix secrets rendered in UI when task is not executed. (#22754)
---
airflow/models/taskinstance.py | 14 ++-
tests/www/views/test_views_rendered.py | 43 +++---
2 files changed, 53 insertions(+), 4 deletions(-)
diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py
index f08b3ba812..4b2b2f9348 100644
--- a/airflow/models/taskinstance.py
+++ b/airflow/models/taskinstance.py
@@ -2094,8 +2094,18 @@ class TaskInstance(Base, LoggingMixin):
for field_name, rendered_value in
rendered_task_instance_fields.items():
setattr(self.task, field_name, rendered_value)
return
+
try:
-self.render_templates()
+# Task was never executed. Initialize RenderedTaskInstanceFields
+# to render template and mask secrets. Set MASK_SECRETS_IN_LOGS
+# to True to enable masking similar to task run.
+original_value = settings.MASK_SECRETS_IN_LOGS
+settings.MASK_SECRETS_IN_LOGS = True
+rendered_task_instance = RenderedTaskInstanceFields(self)
+rendered_fields = rendered_task_instance.rendered_fields
+if rendered_fields:
+for field_name, rendered_value in rendered_fields.items():
+setattr(self.task, field_name, rendered_value)
except (TemplateAssertionError, UndefinedError) as e:
raise AirflowException(
"Webserver does not have access to User-defined Macros or
Filters "
@@ -2103,6 +2113,8 @@ class TaskInstance(Base, LoggingMixin):
"started running, please use 'airflow tasks render' for
debugging the "
"rendering of template_fields."
) from e
+finally:
+settings.MASK_SECRETS_IN_LOGS = original_value
@provide_session
def get_rendered_k8s_spec(self, session=NEW_SESSION):
diff --git a/tests/www/views/test_views_rendered.py
b/tests/www/views/test_views_rendered.py
index 129baf7c0e..31935221c0 100644
--- a/tests/www/views/test_views_rendered.py
+++ b/tests/www/views/test_views_rendered.py
@@ -20,7 +20,7 @@ from urllib.parse import quote_plus
import pytest
-from airflow.models import DAG, RenderedTaskInstanceFields
+from airflow.models import DAG, RenderedTaskInstanceFields, Variable
from airflow.operators.bash import BashOperator
from airflow.serialization.serialized_objects import SerializedDAG
from airflow.utils import timezone
@@ -61,6 +61,15 @@ def task2(dag):
)
+@pytest.fixture()
+def task_secret(dag):
+return BashOperator(
+task_id='task_secret',
+bash_command='echo {{ var.value.my_secret }} && echo {{ var.value.spam
}}',
+dag=dag,
+)
+
+
@pytest.fixture(scope="module", autouse=True)
def init_blank_db():
"""Make sure there are no runs before we test anything.
@@ -73,7 +82,7 @@ def init_blank_db():
@pytest.fixture(autouse=True)
-def reset_db(dag, task1, task2):
+def reset_db(dag, task1, task2, task_secret):
yield
clear_db_dags()
clear_db_runs()
@@ -81,7 +90,7 @@ def reset_db(dag, task1, task2):
@pytest.fixture()
-def create_dag_run(dag, task1, task2):
+def create_dag_run(dag, task1, task2, task_secret):
def _create_dag_run(*, execution_date, session):
dag_run = dag.create_dagrun(
state=DagRunState.RUNNING,
@@ -94,6 +103,8 @@ def create_dag_run(dag, task1, task2):
ti1.state = TaskInstanceState.SUCCESS
ti2 = dag_run.get_task_instance(task2.task_id, session=session)
ti2.state = TaskInstanceState.SCHEDULED
+ti3 = dag_run.get_task_instance(task_secret.task_id, session=session)
+ti3.state = TaskInstanceState.QUEUED
session.flush()
return dag_run
@@ -168,3 +179,29 @@ def
test_user_defined_filter_and_macros_raise_error(admin_client, create_dag_run
# MarkupSafe changed the exception detail from 'no filter named' to
# 'No filter named' in 2.0 (I think), so we normalize for comparison.
assert "originalerror: no filter named hello" in
resp_html.lower()
+
+
+@pytest.mark.usefixtures("patch_app")
+def test_rendered_template_secret(admin_client, create_dag_run, task_secret):
+"""Test that the Rendered View masks values retrieved from secret
variables."""
+Variable.set("my_s
[airflow] branch v2-3-test updated (003c4d75f0 -> eb95fa4a3c)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git from 003c4d75f0 Fix provider import error matching (#23825) add eb95fa4a3c Fix secrets rendered in UI when task is not executed. (#22754) No new revisions were added by this update. Summary of changes: airflow/models/taskinstance.py | 14 ++- tests/www/views/test_views_rendered.py | 43 +++--- 2 files changed, 53 insertions(+), 4 deletions(-)
[airflow] branch v2-3-test updated (23135c91a2 -> da45511a22)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git omit 23135c91a2 Fix retrieval of deprecated non-config values (#23723) add da45511a22 Fix retrieval of deprecated non-config values (#23723) This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (23135c91a2) \ N -- N -- N refs/heads/v2-3-test (da45511a22) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes:
[airflow] branch v2-3-test updated (da45511a22 -> 84f03e2e64)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a change to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
omit da45511a22 Fix retrieval of deprecated non-config values (#23723)
omit eb95fa4a3c Fix secrets rendered in UI when task is not executed.
(#22754)
omit 003c4d75f0 Fix provider import error matching (#23825)
omit 403c6344c3 Fix regression in ignoring symlinks (#23535)
omit 2f35d6e132 Update version to 2.3.1 and add release notes
add f5f06fa785 Fix regression in ignoring symlinks (#23535)
add 54e5ce24bb Fix provider import error matching (#23825)
add 743f4b9031 Fix secrets rendered in UI when task is not executed.
(#22754)
add d61cde679a Fix retrieval of deprecated non-config values (#23723)
add 66c6c0458b Expand/collapse all groups (#23487)
add 15fe9f61d5 19943 Grid view status filters (#23392)
add abbc0b5a2b Fix expand/collapse all buttons (#23590)
add c8a4e557a6 Add UI tests for /utils and /components (#23456)
add d534bdf8fa Automatically reschedule stalled queued tasks in
CeleryExecutor (v2) (#23690)
add 84f03e2e64 Update version to 2.3.1 and add release notes
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (da45511a22)
\
N -- N -- N refs/heads/v2-3-test (84f03e2e64)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
RELEASE_NOTES.rst | 15 +-
airflow/config_templates/config.yml| 15 +-
airflow/config_templates/default_airflow.cfg | 11 +-
airflow/executors/celery_executor.py | 143 +++---
airflow/settings.py| 2 +-
airflow/utils/state.py | 2 +-
airflow/www/jest-setup.js | 6 +-
airflow/www/static/js/datetime_utils.js| 1 +
.../static/js/grid/AutoRefresh.jsx}| 54 +++---
airflow/www/static/js/grid/FilterBar.jsx | 127 +
airflow/www/static/js/grid/Grid.jsx| 120 +---
airflow/www/static/js/grid/Grid.test.jsx | 197 +++
.../static/js/grid/LegendRow.jsx} | 38 ++--
airflow/www/static/js/grid/Main.jsx| 80
airflow/www/static/js/grid/ToggleGroups.jsx| 77
airflow/www/static/js/grid/api/useGridData.js | 41 ++--
.../www/static/js/grid/api/useGridData.test.jsx| 6 +-
.../static/js/grid/{ => components}/Clipboard.jsx | 2 +-
.../js/grid/components/Clipboard.test.jsx} | 33 ++--
.../js/grid/{ => components}/InstanceTooltip.jsx | 4 +-
.../js/grid/components/InstanceTooltip.test.jsx| 86 +
.../static/js/grid/{ => components}/StatusBox.jsx | 5 +-
.../www/static/js/grid/{ => components}/Table.jsx | 6 +-
.../www/static/js/grid/components/Table.test.jsx | 211 +
.../static/js/grid/{ => components}/TaskName.jsx | 8 +-
.../static/js/grid/components/TaskName.test.jsx| 53 ++
.../www/static/js/grid/{ => components}/Time.jsx | 5 +-
.../www/static/js/grid/components/Time.test.jsx| 83
airflow/www/static/js/grid/context/timezone.jsx| 5 +-
airflow/www/static/js/grid/dagRuns/Bar.jsx | 2 +-
airflow/www/static/js/grid/dagRuns/Tooltip.jsx | 2 +-
airflow/www/static/js/grid/dagRuns/index.test.jsx | 31 ++-
airflow/www/static/js/grid/details/Header.jsx | 12 +-
airflow/www/static/js/grid/details/content/Dag.jsx | 4 +-
.../js/grid/details/content/dagRun/index.jsx | 6 +-
.../grid/details/content/taskInstance/Details.jsx | 6 +-
.../content/taskInstance/MappedInstances.jsx | 6 +-
.../js/grid/details/content/taskInstance/index.jsx | 7 +-
airflow/www/static/js/grid/details/index.jsx | 2 +-
airflow/www/static/js/grid/index.jsx | 5 +-
airflow/www/static/js/grid/renderTaskRows.jsx | 55 +++---
airflow/www/static/js/grid/renderTaskRows.test.jsx | 128 +
.../js/{task.js => grid/utils/gridData.test.js}| 37 ++--
airflow/www/static/js/grid/utils/testUtils.jsx | 12 ++
airflow/www/static/js/grid/utils/useErrorToast.js | 4 +-
.../static/js/grid/utils/useErrorToast.test.jsx| 52 +
airflow/www/sta
[airflow] branch v2-3-test updated (10675af30b -> d07392fe15)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git discard 10675af30b Add 'reschedule' to the serialized fields for the BaseSensorOperator (#23674) new 0ddc90f120 Add 'reschedule' to the serialized fields for the BaseSensorOperator (#23674) new 99202020fc Fix task log is not captured (#23684) new 27762afd9f Fix auto upstream dep when expanding non-templated field (#23771) new d07392fe15 Fix dag-processor fetch metabase config (#23575) This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (10675af30b) \ N -- N -- N refs/heads/v2-3-test (d07392fe15) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: airflow/cli/commands/dag_processor_command.py | 2 +- airflow/models/mappedoperator.py | 3 +-- airflow/models/taskinstance.py| 6 -- tests/cli/commands/test_task_command.py | 2 +- tests/models/test_taskinstance.py | 26 ++ 5 files changed, 33 insertions(+), 6 deletions(-)
[airflow] 02/04: Fix task log is not captured (#23684)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 99202020fcaff1260247fc63f77cd48fb6a7924d
Author: Ping Zhang
AuthorDate: Tue May 17 09:29:52 2022 -0700
Fix task log is not captured (#23684)
when StandardTaskRunner runs tasks with exec
Issue: https://github.com/apache/airflow/issues/23540
(cherry picked from commit e453e68158b65177572a3a1f95d8542c3a38d4e7)
---
airflow/models/taskinstance.py | 6 --
tests/cli/commands/test_task_command.py | 2 +-
tests/models/test_taskinstance.py | 6 ++
3 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py
index e1e87ec8e8..0bc90636ff 100644
--- a/airflow/models/taskinstance.py
+++ b/airflow/models/taskinstance.py
@@ -548,7 +548,8 @@ class TaskInstance(Base, LoggingMixin):
self.task_id = task.task_id
self.map_index = map_index
self.refresh_from_task(task)
-self._log = logging.getLogger("airflow.task")
+# init_on_load will config the log
+self.init_on_load()
if run_id is None and execution_date is not None:
from airflow.models.dagrun import DagRun # Avoid circular import
@@ -591,7 +592,6 @@ class TaskInstance(Base, LoggingMixin):
if state:
self.state = state
self.hostname = ''
-self.init_on_load()
# Is this TaskInstance being currently running within `airflow tasks
run --raw`.
# Not persisted to the database so only valid for the current process
self.raw = False
@@ -622,6 +622,8 @@ class TaskInstance(Base, LoggingMixin):
@reconstructor
def init_on_load(self):
"""Initialize the attributes that aren't stored in the DB"""
+# correctly config the ti log
+self._log = logging.getLogger("airflow.task")
self.test_mode = False # can be changed when calling 'run'
@property
diff --git a/tests/cli/commands/test_task_command.py
b/tests/cli/commands/test_task_command.py
index e539592a64..08ecebf525 100644
--- a/tests/cli/commands/test_task_command.py
+++ b/tests/cli/commands/test_task_command.py
@@ -110,7 +110,7 @@ class TestCliTasks(unittest.TestCase):
args = self.parser.parse_args(["tasks", "test", self.dag_id, task_id,
DEFAULT_DATE.isoformat()])
-with self.assertLogs('airflow.models', level='INFO') as cm:
+with self.assertLogs('airflow.task', level='INFO') as cm:
task_command.task_test(args)
assert any(
[
diff --git a/tests/models/test_taskinstance.py
b/tests/models/test_taskinstance.py
index 57355a3ad5..4bb3b2bae4 100644
--- a/tests/models/test_taskinstance.py
+++ b/tests/models/test_taskinstance.py
@@ -264,6 +264,12 @@ class TestTaskInstance:
assert op2 in op1.downstream_list
assert op2 in op3.downstream_list
+def test_init_on_load(self, create_task_instance):
+ti = create_task_instance()
+# ensure log is correctly created for ORM ti
+assert ti.log.name == 'airflow.task'
+assert not ti.test_mode
+
@patch.object(DAG, 'get_concurrency_reached')
def test_requeue_over_dag_concurrency(self, mock_concurrency_reached,
create_task_instance):
mock_concurrency_reached.return_value = True
[airflow] 03/04: Fix auto upstream dep when expanding non-templated field (#23771)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 27762afd9f01519d0dd53a769eae968b96ac2580
Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com>
AuthorDate: Wed May 18 13:43:16 2022 -0600
Fix auto upstream dep when expanding non-templated field (#23771)
If you tried to expand via xcom into a non-templated field without
explicitly setting the upstream task dependency, the scheduler would
crash because the upstream task dependency wasn't being set
automatically. It was being set only for templated fields, but now we do
it for both.
(cherry picked from commit 3849ebb8d22bbc229d464c4171c9b5ff960cd089)
---
airflow/models/mappedoperator.py | 3 +--
tests/models/test_taskinstance.py | 20
2 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/airflow/models/mappedoperator.py b/airflow/models/mappedoperator.py
index b63e26ec9e..c522cefb2c 100644
--- a/airflow/models/mappedoperator.py
+++ b/airflow/models/mappedoperator.py
@@ -300,8 +300,7 @@ class MappedOperator(AbstractOperator):
if self.dag:
self.dag.add_task(self)
for k, v in self.mapped_kwargs.items():
-if k in self.template_fields:
-XComArg.apply_upstream_relationship(self, v)
+XComArg.apply_upstream_relationship(self, v)
for k, v in self.partial_kwargs.items():
if k in self.template_fields:
XComArg.apply_upstream_relationship(self, v)
diff --git a/tests/models/test_taskinstance.py
b/tests/models/test_taskinstance.py
index 4bb3b2bae4..c520fe70bd 100644
--- a/tests/models/test_taskinstance.py
+++ b/tests/models/test_taskinstance.py
@@ -2855,3 +2855,23 @@ def
test_ti_mapped_depends_on_mapped_xcom_arg_XXX(dag_maker, session):
ti.refresh_from_task(dag.get_task("add_one"))
with pytest.raises(XComForMappingNotPushed):
ti.run()
+
+
+def test_expand_non_templated_field(dag_maker, session):
+"""Test expand on non-templated fields sets upstream deps properly."""
+
+class SimpleBashOperator(BashOperator):
+template_fields = ()
+
+with dag_maker(dag_id="product_same_types", session=session) as dag:
+
+@dag.task
+def get_extra_env():
+return [{"foo": "bar"}, {"foo": "biz"}]
+
+SimpleBashOperator.partial(task_id="echo", bash_command="echo
$FOO").expand(env=get_extra_env())
+
+dag_maker.create_dagrun()
+
+echo_task = dag.get_task("echo")
+assert "get_extra_env" in echo_task.upstream_task_ids
[airflow] 01/04: Add 'reschedule' to the serialized fields for the BaseSensorOperator (#23674)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 0ddc90f12020cd7197189a80ea56a7ec0a81273b
Author: David Caron
AuthorDate: Tue May 17 08:18:29 2022 -0400
Add 'reschedule' to the serialized fields for the BaseSensorOperator
(#23674)
fix #23411
(cherry picked from commit f9e2a3051cd3a5b6fcf33bca4c929d220cf5661e)
---
airflow/sensors/base.py| 4
tests/serialization/test_dag_serialization.py | 1 +
tests/ti_deps/deps/test_ready_to_reschedule_dep.py | 7 ++-
3 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/airflow/sensors/base.py b/airflow/sensors/base.py
index 7f1cd87c3d..f00b3a6761 100644
--- a/airflow/sensors/base.py
+++ b/airflow/sensors/base.py
@@ -339,6 +339,10 @@ class BaseSensorOperator(BaseOperator, SkipMixin):
"""Define mode rescheduled sensors."""
return self.mode == 'reschedule'
+@classmethod
+def get_serialized_fields(cls):
+return super().get_serialized_fields() | {"reschedule"}
+
def poke_mode_only(cls):
"""
diff --git a/tests/serialization/test_dag_serialization.py
b/tests/serialization/test_dag_serialization.py
index 0144501f1a..fe9fc7c7e5 100644
--- a/tests/serialization/test_dag_serialization.py
+++ b/tests/serialization/test_dag_serialization.py
@@ -1462,6 +1462,7 @@ class TestStringifiedDAGs:
assert "deps" in blob
serialized_op = SerializedBaseOperator.deserialize_operator(blob)
+assert serialized_op.reschedule == (mode == "reschedule")
assert op.deps == serialized_op.deps
@pytest.mark.parametrize(
diff --git a/tests/ti_deps/deps/test_ready_to_reschedule_dep.py
b/tests/ti_deps/deps/test_ready_to_reschedule_dep.py
index 470166db21..99416bbbc8 100644
--- a/tests/ti_deps/deps/test_ready_to_reschedule_dep.py
+++ b/tests/ti_deps/deps/test_ready_to_reschedule_dep.py
@@ -31,7 +31,7 @@ from airflow.utils.timezone import utcnow
class TestNotInReschedulePeriodDep(unittest.TestCase):
def _get_task_instance(self, state):
dag = DAG('test_dag')
-task = Mock(dag=dag)
+task = Mock(dag=dag, reschedule=True)
ti = TaskInstance(task=task, state=state, run_id=None)
return ti
@@ -52,6 +52,11 @@ class TestNotInReschedulePeriodDep(unittest.TestCase):
dep_context = DepContext(ignore_in_reschedule_period=True)
assert ReadyToRescheduleDep().is_met(ti=ti, dep_context=dep_context)
+def test_should_pass_if_not_reschedule_mode(self):
+ti = self._get_task_instance(State.UP_FOR_RESCHEDULE)
+del ti.task.reschedule
+assert ReadyToRescheduleDep().is_met(ti=ti)
+
def test_should_pass_if_not_in_none_state(self):
ti = self._get_task_instance(State.UP_FOR_RETRY)
assert ReadyToRescheduleDep().is_met(ti=ti)
[airflow] 04/04: Fix dag-processor fetch metabase config (#23575)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit d07392fe15d30b61631de30c55b91dd0843bfef3
Author: Andrey Anshin
AuthorDate: Mon May 9 11:50:33 2022 +0300
Fix dag-processor fetch metabase config (#23575)
(cherry picked from commit 9837e6d813744e3c5861c32e87b3aeb496d0f88d)
---
airflow/cli/commands/dag_processor_command.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/airflow/cli/commands/dag_processor_command.py
b/airflow/cli/commands/dag_processor_command.py
index a07035be50..183b77dbe2 100644
--- a/airflow/cli/commands/dag_processor_command.py
+++ b/airflow/cli/commands/dag_processor_command.py
@@ -50,7 +50,7 @@ def dag_processor(args):
if not conf.getboolean("scheduler", "standalone_dag_processor"):
raise SystemExit('The option [scheduler/standalone_dag_processor] must
be True.')
-sql_conn: str = conf.get('core', 'sql_alchemy_conn').lower()
+sql_conn: str = conf.get('database', 'sql_alchemy_conn').lower()
if sql_conn.startswith('sqlite'):
raise SystemExit('Standalone DagProcessor is not supported when using
sqlite.')
[airflow] branch v2-3-test updated (53dd2553b2 -> cf77eb701a)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git omit 53dd2553b2 Add missing "airflow-constraints-reference" parameter (#23844) omit c19f8b62f8 Fix python version command (#23818) omit e3c1bf4934 Update version to 2.3.1 and add release notes add cae61ed71c Fix python version command (#23818) add de2f739f70 Add missing "airflow-constraints-reference" parameter (#23844) add 447e899006 Grid data: do not load all mapped instances (#23813) add cf77eb701a Update version to 2.3.1 and add release notes This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (53dd2553b2) \ N -- N -- N refs/heads/v2-3-test (cf77eb701a) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: airflow/www/utils.py | 19 +++ airflow/www/views.py | 34 -- 2 files changed, 23 insertions(+), 30 deletions(-)
[airflow] branch v2-3-test updated (003971fab9 -> b42172614b)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git from 003971fab9 Add limit for JPype1 (#23847) add b42172614b Update release date on Release note No new revisions were added by this update. Summary of changes: RELEASE_NOTES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[airflow] branch v2-3-stable updated (6b772f68db -> ee100a592e)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-stable in repository https://gitbox.apache.org/repos/asf/airflow.git from 6b772f68db update release note new 443ac72ed1 Fix empty image preparation (#23304) new 1f33df596a Unify style of communication with the users for Breeze. (#23311) new e8fdd95dfc Remove confusion about upgrade-to-newer-dependencies breeze param (#23334) new 45a8483020 resolving conflict (#23052) new 18d7456c6f Cleaner default output when breeze starts (#23341) new 001987cc21 Fix regeneration of breeze screenshots (#23344) new 995c45aef7 Fix mssql in the new Breeze (#23368) new cc1f736529 When exec fails in breeze we do not print stack-trace (#23342) new ec69860529 Add missing --for-production parameter for new breeze docs building (#23376) new 5647a95832 Bump version to 2.3.0 new 8fc0a5aba5 Fix 2.3.0 release date in release notes new 04cc4a510f Clarify 2.3.0 kubernetes min version is about library not cluster (#23398) new d9718c3601 Fix deriving of PyPI branch from airflow version (#23380) new 886c10e42e Improve handling of entry and exit to common Breeze commands (#23395) new fbb6b9cbb9 Add tags inside try block. (#21784) new 683d1dc4fb Bump pre-commit hook versions (#22887) new 44578a80ce Use kubernetes queue in kubernetes hybrid executors (#23048) new 367a6f077a Show warning if '/' is used in a DAG run ID (#23106) new 062308f5ca Override pool for TaskInstance when pool is passed from cli. (#23258) new a0a2ddf60d Remove custom signal handling in Triggerer (#23274) new 8cc26f7df6 Store grid view selection in url params (#23290) new d3be9dd592 Fix duplicated Kubernetes DeprecationWarnings (#23302) new fc8aef0978 Use in Mapped Instance table (#23313) new 2976182e12 Fix update user auth stats (#23314) new c0b91462d2 Ignore some files/directory when releasing source code (#23325) new 7657e65312 Improve react www tests (#23329) new 427a125389 Use run_id for ti.mark_success_url (#23330) new 84021dedbe Don't show grid actions if server would reject with permission denied (#23332) new 5290cfd506 Hide some task instance attributes (#23338) new a5389e8313 fix cli `airflow dags show` for mapped operator (#23339) new 04b815d96a Fix connection test button (#23345) new 636e82e833 Fix broken task instance link in xcom list (#23367) new c06c214581 Add doc notes for keyword-only args for `expand()` and `partial()` (#23373) new 02be4d99af Update missing `version_added` in config.yml (#23387) new 806ea46e0d Fix `version_added` for `[sensors] default_timeout` (#23388) new 98872ef906 [FIX] remove python 3.6 (#23409) new e338621dca Fix literal cross product expansion (#23434) new 372495d4b1 Add backward compatibility for core__sql_alchemy_conn__cmd (#23441) new 321294bbf1 Docs: Python 3.10 is now supported (#23457) new 824d36ff0d Optimize 2.3.0 pre-upgrade check queries (#23458) new 93253c0d57 Add Python 3.10 trove classifier (#23464) new 4bdfeb54ff Remove remaining Python3.6 references (#23474) new a1a1cf1d49 Remove color change for highly nested groups (#23482) new 56a19fd102 Visually distinguish task group summarys (#23488) new fef6f7773d Only count bad refs when `moved` table exists (#23491) new e7d860ab87 Replace DummyOperator references in docs (#23502) new 5af6febf1f Fix _PIP_ADDITIONAL_REQUIREMENTS case for docker-compose (#23517) new 1e3b8014e2 Change approach to finding bad rows to LEFT OUTER JOIN. (#23528) new 7e1611132d Add IPV6 form of the address in cassandra status check (#23537) new 6ba8f44278 Refactor code references from tree to grid (#23254) new d100302158 Mark image as refreshed when pulled on CI (#23410) new de460d3ca2 Improve verbose output of Breeze (#23446) new 0400865b68 Unify approach for user questions asked in Breeze (#23335) new 0968ac7763 Move non-opencontainer labeling of the image to breeze from Dockerfile (#23379) new d4ee616b1f Changed word 'the' instead 'his' (#23493) new 7526742613 Move tests command in new breeze (#23445) new 79eb165801 Seperate provider verification as standalone breeze command (#23454) new c64b0094c3 Fix accidental including of providers in airflow package (#23552) new 3d9c24143c Add logging in to Github Registry for breeze pull (#23551) new c8d393d8f8 Refactor Breeze to group related methods and classes together (#23556) new 77627b2e52 Add slim images to release process (#23391) new 4929d365d3 tHe output of commands of Breeze are only generated when they change (#23570) new 90b9e7365e Clean up in-line f-string concatenation (#23591) new 9809ad75b6 Improve caching for multi-platform images. (#23562) new 69f18fc2fe Use inclusive words
[airflow] annotated tag constraints-2.3.1rc1 updated (bc44527fc3 -> 142d4d0e12)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to annotated tag constraints-2.3.1rc1 in repository https://gitbox.apache.org/repos/asf/airflow.git *** WARNING: tag constraints-2.3.1rc1 was modified! *** from bc44527fc3 (commit) to 142d4d0e12 (tag) tagging bc44527fc318bb7f7b8f2f768930f756af394e92 (commit) replaces constraints-latest by Ephraim Anierobi on Sat May 21 22:40:58 2022 +0100 - Log - Constraints for Apache Airflow 2.3.1rc1 -BEGIN PGP SIGNATURE- iQGzBAABCgAdFiEEAEK9vayTIUi0cM3WNcli2lUp2+wFAmKJXGoACgkQNcli2lUp 2+xqOQwAoeUQ6PMH9gkzcTY0zMFh8ya/uzzuLu7NVFAf2md3NOC+tBbn2mglRM2e i4LbOzuwzKDMourNs+bDQAD6SwTdnHFis4NMN4PByQJxCl6gEwjeGLaiSyi1l1BF pXDlJL5qFGUUg1oPw8OZ3CO9hwWA4GCVRweW7ICUSuoV2ThwB6shG5K9AL0kk5io aVPPseObHk4OVwEbAO4V4DA3nmmt8OvugKtSUEemn6yDjcEMgPmseb54gYQ4sWr1 LHaTzmAjddspgFJY4m9x5QJPYSq30o0DO7sQzQcSy6u8XvlI/bnk/LgCzfp2ycHs ZAp/v0do2iu6ZP6oK9qS2T9kokDTpIw8yZO7WOgUcgRL+bqOvTNGQMISYpcipfEo 4o5zhsn5W2ChUPP+KlA3SE6wJpbNvIie6xUQRPjkIkZQp7DLFPzS1F4lRH73c6O0 zuVXGd8QgVqAy5NO2OCSS0V+hGwaMTnunui+/zJt4l6c1+/v0PGy/bA4pztqPPWs /giHep/3 =xU7L -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
[airflow] annotated tag 2.3.1rc1 updated (ee100a592e -> 2df94a85f4)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to annotated tag 2.3.1rc1 in repository https://gitbox.apache.org/repos/asf/airflow.git *** WARNING: tag 2.3.1rc1 was modified! *** from ee100a592e (commit) to 2df94a85f4 (tag) tagging ee100a592e07761f3b32294f3ad82c4a6c7cf74d (commit) replaces 2.3.0 by Ephraim Anierobi on Sat May 21 22:06:06 2022 +0100 - Log - Apache Airflow 2.3.1rc1 -BEGIN PGP SIGNATURE- iQGzBAABCgAdFiEEAEK9vayTIUi0cM3WNcli2lUp2+wFAmKJVD4ACgkQNcli2lUp 2+zgnwv+I03e0b5xNiELykNTlqmOFQACkp8gdRdmZlBAtX9j2rhGywToacdkaUgP vHmFDN+MSHWdDfJiAohcLZW31GmxRUMvQU30P+52gmrMz0cijUXuC0XjrwtR+pf4 +B4leSjmdJeIrrsedkp7h4btKnFr2K24rR6jZBtqLx3EYV4NMQPIE6T2J9ccCATx gqzPwgm7WpEsRdWLG1NPKECEgrsbaCjQe7GwgH2dnKW9c8jhXFBZ42YX9FTZiyxY 62YzEw4p7J32sZnoaEq/kuo2SDxh7RCblQTNosQktlHKSeHaHVFlzBCm3bE0gzzZ g58IS55ip2HgAfkQ7cm9tQ21ZMcsWh56c+GWvJlRMbDUsFlvkv+81E5Wx3Hcl3Ld GM1o2uECGJJ/PJkwW9hKBlP0SpRVHWiM4rr27f+lSWyIfJ3s2bqnYNm6TBaltzN0 YVo/YoSrkE9S/nOKfvuFK4aTCD0BJG/2IxQ/pxNeALPfuMtyCPkzKlrmuBxR4/B6 cUszo4wo =l9KM -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
svn commit: r54670 - /dev/airflow/2.3.1rc1/
Author: ephraimanierobi Date: Sat May 21 21:47:08 2022 New Revision: 54670 Log: Add artifacts for Airflow 2.3.1rc1 Added: dev/airflow/2.3.1rc1/ dev/airflow/2.3.1rc1/apache-airflow-2.3.1-source.tar.gz (with props) dev/airflow/2.3.1rc1/apache-airflow-2.3.1-source.tar.gz.asc dev/airflow/2.3.1rc1/apache-airflow-2.3.1-source.tar.gz.sha512 dev/airflow/2.3.1rc1/apache-airflow-2.3.1.tar.gz (with props) dev/airflow/2.3.1rc1/apache-airflow-2.3.1.tar.gz.asc dev/airflow/2.3.1rc1/apache-airflow-2.3.1.tar.gz.sha512 dev/airflow/2.3.1rc1/apache_airflow-2.3.1-py3-none-any.whl (with props) dev/airflow/2.3.1rc1/apache_airflow-2.3.1-py3-none-any.whl.asc dev/airflow/2.3.1rc1/apache_airflow-2.3.1-py3-none-any.whl.sha512 Added: dev/airflow/2.3.1rc1/apache-airflow-2.3.1-source.tar.gz == Binary file - no diff available. Propchange: dev/airflow/2.3.1rc1/apache-airflow-2.3.1-source.tar.gz -- svn:mime-type = application/octet-stream Added: dev/airflow/2.3.1rc1/apache-airflow-2.3.1-source.tar.gz.asc == --- dev/airflow/2.3.1rc1/apache-airflow-2.3.1-source.tar.gz.asc (added) +++ dev/airflow/2.3.1rc1/apache-airflow-2.3.1-source.tar.gz.asc Sat May 21 21:47:08 2022 @@ -0,0 +1,14 @@ +-BEGIN PGP SIGNATURE- + +iQHPBAABCgA5FiEEAEK9vayTIUi0cM3WNcli2lUp2+wFAmKJXBgbHGVwaHJhaW1h +bmllcm9iaUBhcGFjaGUub3JnAAoJEDXJYtpVKdvsVK0L+gKxeOAUhse8H4lbZ5QR +Qu9wMLUgTdPVG3ORRhdViMbPnIWjC+kfTCZ9yuXZQPAk3SXghng7Z+yNYr0Xt4y+ +6n8dBsIu5E2Qs4wNhdtCBhZc+PRN6xelLFeS6CkcxLde3+aMI+wskU66NV/bZxeG +IL2TzCaVE35qQ1ONTSJzzqIc0UA3WX29l09VKGwmOhtYkFj6MH4+iCSTJyw5lHQF +xdrqS+adgUcwG+cLy+eRGjW+7AP9jZEBVID1RSzdAtADdvAhQC+qX8twJhuM7mXw +znY8o0M1mhfOpiegF7IOU34ndpECApcLNGm4Gn2//D/gR9+9TZ6ckCRsyEPBghAI +PAx7ERGPV4Vl+hIIpl57iF0bjvXk2HzfwVbS0sm3cEtpGrhMZ864V7PoBO9iLirD +GdKtez0y9kuIXWl5AZ4YU1n70JvJwQr94pDPDV1JZBeATEfvhaG5jjFutbTRSQyb +y60LI9ILppGQzXCinIENIOS7Cx/O2Oscg1+4W5eI/YivtA== +=y13L +-END PGP SIGNATURE- Added: dev/airflow/2.3.1rc1/apache-airflow-2.3.1-source.tar.gz.sha512 == --- dev/airflow/2.3.1rc1/apache-airflow-2.3.1-source.tar.gz.sha512 (added) +++ dev/airflow/2.3.1rc1/apache-airflow-2.3.1-source.tar.gz.sha512 Sat May 21 21:47:08 2022 @@ -0,0 +1 @@ +89a0a0a891c2a6c71a8f75eea9d5882df87b63b7b6aa7d592f2207fd3ddd96b82eabbb62e0e65962c9e140def1f6311c6883b59a0e94c9c04b5d03103f996474 apache-airflow-2.3.1-source.tar.gz Added: dev/airflow/2.3.1rc1/apache-airflow-2.3.1.tar.gz == Binary file - no diff available. Propchange: dev/airflow/2.3.1rc1/apache-airflow-2.3.1.tar.gz -- svn:mime-type = application/octet-stream Added: dev/airflow/2.3.1rc1/apache-airflow-2.3.1.tar.gz.asc == --- dev/airflow/2.3.1rc1/apache-airflow-2.3.1.tar.gz.asc (added) +++ dev/airflow/2.3.1rc1/apache-airflow-2.3.1.tar.gz.asc Sat May 21 21:47:08 2022 @@ -0,0 +1,14 @@ +-BEGIN PGP SIGNATURE- + +iQHPBAABCgA5FiEEAEK9vayTIUi0cM3WNcli2lUp2+wFAmKJXCIbHGVwaHJhaW1h +bmllcm9iaUBhcGFjaGUub3JnAAoJEDXJYtpVKdvs0doMAIWDfXlqTC8ac6T7TIDA +BBtyGxbwp0jHgk6k8d3gB565qTwPA0cNHdG5mFsaIBRoATkzbPmaxdFcalToCG68 +5JesJA0PICpEh5QM8Y/f9TZpOnOaMfEbNEa8gx3CoMd42NeEb8+IaUhmVZErlz/g +9CcAq0b0F1MXbhJLQwmRjQetaQD+Hu8uUlJ70k5DG5tkaH3dqzm9BT56c8mMVJQd +G10lj9s8SHdjKymTOqqOfDIpgyhCO2MxtAUkTMt3kGs+ot0NwsXIW04t9p6r5QWy +HzYf/7YulGINawubtbuc8QfuIpQzZwaoSCdV6EdiW8uj1Yy5Oh54QEeOQ3lQ3GUO +jMaQP6Xlyl+7cIdwJU4vy13U5IHYKk0KacWCpIksUMipH/YMh/jpbalOc0imW/dT +BO6QXfbnsC1/18dAQkUsNKa6B8yNlvXjyjNajMF0aA8EY/2Dv0BFWhR3wr+7+j99 +YA2vLqBGMD+RKfUjSDzEf4xToaubBFtsSwzVd1ZeGBqHgw== +=/ZhK +-END PGP SIGNATURE- Added: dev/airflow/2.3.1rc1/apache-airflow-2.3.1.tar.gz.sha512 == --- dev/airflow/2.3.1rc1/apache-airflow-2.3.1.tar.gz.sha512 (added) +++ dev/airflow/2.3.1rc1/apache-airflow-2.3.1.tar.gz.sha512 Sat May 21 21:47:08 2022 @@ -0,0 +1 @@ +fb6ae753a24fc59648e00dacd487ae6060aa26316b0b2a87a7409e9049af1897f2bf04379ba86349f1f25421aca03093581a52420b76708d5eb37044518a9093 apache-airflow-2.3.1.tar.gz Added: dev/airflow/2.3.1rc1/apache_airflow-2.3.1-py3-none-any.whl == Binary file - no diff available. Propchange: dev/airflow/2.3.1rc1/apache_airflow-2.3.1-py3-none-any.whl -- svn:mime-type = application/octet-stream Added: dev/airflow/2.3.1rc1/apache_airflow-2.3.1-py3-none-any.whl.asc
[airflow] branch v2-3-stable updated (ee100a592e -> d1e2dc5bd1)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-stable in repository https://gitbox.apache.org/repos/asf/airflow.git from ee100a592e Update release date on Release note add 838b97c9a0 Fix release date for 2.3.1 add 4ba96b836b Exclude missing tasks from the gantt view (#23627) add 7ee739bb92 Simplify flash message for _airflow_moved tables (#23635) add 14b1ea9aba Enable clicking on DAG owner in autocomplete dropdown (#23804) add 7f4a833798 Fix UnboundLocalError when sql is empty list in DbApiHook (#23816) add ecbae242c6 Add __wrapped__ property to _TaskDecorator (#23830) add f0122053a7 Fix exception trying to display moved table warnings (#23837) add 6decd20c96 Remove redundant register exit signals in `dag-processor` command (#23886) add a88900c8a7 Prevent UI from crashing if grid task instances are null (#23939) add 2560dd50c4 Fix and speed up grid view (#23947) add 9bc2896651 Highlight task states by hovering on legend row (#23678) add 18a6c88417 Use '--subdir' argument value for standalong dag processor. (#23864) add 5f466edf6a Update dep for databricks #23917 (#23927) add 7b869b6528 Fix cassandra to 3.0.25 (#23522) add 86cfcb14e7 Temporarily pin xmltodict to 0.12.0 to fix main failure (#23577) add 404a3c3cf4 Self upgrade when refreshing images (#23686) add c3a2ec7335 Doc: Add column names for DB Migration Reference (#23853) add 61f5952503 Add better feedback to Breeze users about expected action timing (#23827) add 368962ecf6 Upgrade `pip` to 22.1.1 version (just released) (#23854) add c16fedabe2 update K8S-KIND to 0.14.0 (#23859) add e0c6fdd9a1 Speed up Breeze experience on Mac OS (#23866) add ce1681cabc Few fixes in the providers release doc (#23382) add 99d62a7f70 Avoid printing exception when exiting tests command (#23897) add 7be9bf5e90 Make CI and PROD image builds consistent (#23841) add 16fc9d99d2 Clarify manual merging of PR in release doc (#23928) add 83b0fbdcd7 Fix broken main (#23940) add 74a424490a Disable rebase workflow (#23943) add 4cff4e052f Add cascade to `dag_tag` to `dag` foreignkey (#23444) add 09be0c5c7e Mask sensitive values for not-yet-running TIs (#23807) add 01c998c703 Replaced all days_ago functions with datetime functions (#23237) add 0682ac53cf Ignore the DeprecationWarning in test_days_ago (#23875) add 71d5eba1d8 DagFileProcessorManager: Start a new process group only if current process not a session leader (#23872) add db929386d4 Add is_mapped field to Task response. (#23319) add 75285614e8 Disallow calling expand with no arguments (#23463) add a8dd8f82e8 Add typing to Azure Cosmos Client Hook (#23941) add 28fea157cf Faster grid view (#23951) add b6b7a85616 [23945] Icons in grid view for different dag types (#23970) add e4caa10371 Make provider doc preparation a bit more fun :) (#23629) add 25931dbcff Change chart annotation generator to use RELEASE_NOTES (#23549) add b11bb985f8 Add tool to automaticaly update status of AIP-47 issues. (#23745) add 4a96bd6219 Add Deferrable Databricks operators (#19736) add 37a5696de4 Fix Breeze documentation typo (#23919) add 48efec10b2 Add exception to catch single line private keys (#23043) add 54cad993e0 Introduce `flake8-implicit-str-concat` plugin to static checks (#23873) add 4c8c6b7e8b Fix missing shorthand for docker buildx rm -f (#23984) add 7fe7f6a6d0 Remove fixing cncf.kubernetes provider when generating constraints (#23994) add 312b7f3747 Disable fail-fast on pushing images to docker cache (#24005) add 8d522e1fe2 use explicit --mount with types of mounts rather than --volume flags (#23982) add a4d408c4dd Force colors in yarn test output in CI (#23986) add f72217b510 Fix breeze failures when there is no buildx installed on Mac (#23988) add 5a153e016d Replace generation of docker volumes to be done from python (#23985) add f2dc81e56b Remove pinning for xmltodict (#23992) add 870d0916a7 Revert "Add limit for JPype1 (#23847)" (#23953) add 188afd2ba3 Make --file command in static-checks autocomplete file name (#23896) add d1e2dc5bd1 Update version to 2.3.2 and add release notes No new revisions were added by this update. Summary of changes: .asf.yaml | 2 +- .github/workflows/ci.yml | 11 +- .pre-commit-config.yaml| 7 +- BREEZE.rst | 8 + Dockerfile | 4 +- Dockerfile.ci | 4 +- IMAGES.rst | 2 +- README.md | 14 +- RELEASE
[airflow] annotated tag 2.3.2rc1 updated (d1e2dc5bd1 -> 562bcce500)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to annotated tag 2.3.2rc1 in repository https://gitbox.apache.org/repos/asf/airflow.git *** WARNING: tag 2.3.2rc1 was modified! *** from d1e2dc5bd1 (commit) to 562bcce500 (tag) tagging d1e2dc5bd1a627115870f66523c6bc6b9a17dfd0 (commit) replaces 2.3.1 by Ephraim Anierobi on Mon May 30 12:04:19 2022 +0100 - Log - Apache Airflow 2.3.2rc1 -BEGIN PGP SIGNATURE- iQGzBAABCgAdFiEEAEK9vayTIUi0cM3WNcli2lUp2+wFAmKUpLMACgkQNcli2lUp 2+yvbwv+I8nIyLm4Zqi4/SyCT9jP4ymXbfesGXKPZq/B+9lx7wNPlztBSp/Tm6nD MfX/jynoJ6+VsFU7YGGAE+zgQvNLlXt5e/6LcesUdnq17f+Y2xLY3l0KLdpn1f62 X4XgKzzAVj6qXPs1soRJNyGNkYob0fhXkaX/ZzS8PuDn/8UH/bwzKFmftJy0yVUD HsIqUrMbYtKw7ZN35bhx9AaZ5+HqXztoOGUZ+ijxlzqDK9WkoYn+KeSA2rkfKe/n 9yHxRLbDdaETRd9dWsPwG4CLr204q3u41Gpr+ADzmI+Uy7heSaVx9cG7FNqAGkIk mXbSMj9Xp+ST/BKNyFSvC7l/kRpXuDVihfwuVxrR+uAY+ItR/+GNts0br4ZhQTpT g1YmfieILGGwWFDBwFf8Ic4rEFoAViFp7qNHGXJ33qAFry96kuCWqujpBrJuGfLU BxtbdpJqL0us0z6QJP1gxvjuZtuRn5rcZVLdKcbYxVMfgpq60n8NcdgOrGZ01LDx g59ZkPpx =Um0b -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
[airflow] annotated tag constraints-2.3.2rc1 updated (2a188214ff -> 00555912f8)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to annotated tag constraints-2.3.2rc1 in repository https://gitbox.apache.org/repos/asf/airflow.git *** WARNING: tag constraints-2.3.2rc1 was modified! *** from 2a188214ff (commit) to 00555912f8 (tag) tagging 2a188214ffb65b6226ddc582fea5949ea8244ad7 (commit) replaces constraints-latest by Ephraim Anierobi on Mon May 30 12:22:53 2022 +0100 - Log - Constraints for Apache Airflow 2.3.2rc1 -BEGIN PGP SIGNATURE- iQGzBAABCgAdFiEEAEK9vayTIUi0cM3WNcli2lUp2+wFAmKUqQ0ACgkQNcli2lUp 2+yLRAwAmZ2EzROwRA/twlWYfwExOQxgQ872fnjR0waqATV+A1XO5fuZCHwzAIjZ aP6QXG3HLJVC6kFVT+quxyRybje+K9G91tf39SNjkzbpyv9NfGgNYhP7CEtgCXcC q/Cr45FqlmZZDhKNaRL7a3ZxuMUzgHLgRXHFxgMjuq/kZGhWTfqFbQxuj9ZkRBqx SWhPq3Un9ATAvs7BGiXYaPjWwjFiynYLyRQa8j4I/CLph5VWPmZuGiVeH/7wC8iZ 2wlr+HsDB2C+iirkq3ewD9LZG8VBh6Rt2tmr7xJiE0FHdYhLyPYye7IhgzeCptZo dMBfdaASepPWNmOwc6RD8N/gSydgbb99ftARsyizP/b/efOAX7yCYaDXRq/xjPsr XURpBwubP2kc6Q4Yj+IZMTz1bFjnRT3MYm3P0vQwgY8qy7lXNy19iR0f0F2GXc/q aBCBsRtJ60uJBG/CEZa/4UDZ+0AFB71f+jBxadA3zqGCT8U4kkpWIfwgzgp0BYnB j39EkvW1 =HJAD -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
svn commit: r54789 - /dev/airflow/2.3.2rc1/
Author: ephraimanierobi Date: Mon May 30 11:46:09 2022 New Revision: 54789 Log: Add artifacts for Airflow 2.3.2rc1 Added: dev/airflow/2.3.2rc1/ dev/airflow/2.3.2rc1/apache-airflow-2.3.2-source.tar.gz (with props) dev/airflow/2.3.2rc1/apache-airflow-2.3.2-source.tar.gz.asc dev/airflow/2.3.2rc1/apache-airflow-2.3.2-source.tar.gz.sha512 dev/airflow/2.3.2rc1/apache-airflow-2.3.2.tar.gz (with props) dev/airflow/2.3.2rc1/apache-airflow-2.3.2.tar.gz.asc dev/airflow/2.3.2rc1/apache-airflow-2.3.2.tar.gz.sha512 dev/airflow/2.3.2rc1/apache_airflow-2.3.2-py3-none-any.whl (with props) dev/airflow/2.3.2rc1/apache_airflow-2.3.2-py3-none-any.whl.asc dev/airflow/2.3.2rc1/apache_airflow-2.3.2-py3-none-any.whl.sha512 Added: dev/airflow/2.3.2rc1/apache-airflow-2.3.2-source.tar.gz == Binary file - no diff available. Propchange: dev/airflow/2.3.2rc1/apache-airflow-2.3.2-source.tar.gz -- svn:mime-type = application/octet-stream Added: dev/airflow/2.3.2rc1/apache-airflow-2.3.2-source.tar.gz.asc == --- dev/airflow/2.3.2rc1/apache-airflow-2.3.2-source.tar.gz.asc (added) +++ dev/airflow/2.3.2rc1/apache-airflow-2.3.2-source.tar.gz.asc Mon May 30 11:46:09 2022 @@ -0,0 +1,14 @@ +-BEGIN PGP SIGNATURE- + +iQHPBAABCgA5FiEEAEK9vayTIUi0cM3WNcli2lUp2+wFAmKUq50bHGVwaHJhaW1h +bmllcm9iaUBhcGFjaGUub3JnAAoJEDXJYtpVKdvsE08L/R8w61Kz7NM7QZch4/4u +Neorpa+UM858KkR5os8GrpLMj4JyS5ZtKvFyDfsIYJDGfNdo+S+2YTOemHEiaBXn +q9ZDrKUrMHKGqC6QxnbMdGTxRmK1E7qQJeVy9h77StSDUncSptFH6yuPc/+SKtg/ +hgTnnMUCCpGxnXLyCgSXlDDj8+OlwhZNi8B2Ex+Byulu+rfDdACMs/g20m4VDU/k +1ebZ4HGnZ3O3Kru6hGr4/LWUOCfiiFlDImw1CzAc3Hy1we5iRSNkzJ42HUSeqLW6 +AQSlZT73sGbTLsf8yxmAqm2JvKC6ps/UVdEWTaQrZMYjQPtMt6NX67M+K7xZpyDP +9KD/D+igzloeLYuzx6TcXdOpR6Xsyn2yGUhdr+hNDciH7/u6hHiBI0GbN34BQFa6 +BVi547d0IkDOFTPY+HdEBt4HHKgK1ZX8uUraQD6+8iEEt0hRcDmyajqNwBHgS0NU +Hc02UbOXtJinh0ieaDlqzn4vLdP7ugdqqO6V95kl/KiPyA== +=z0Vo +-END PGP SIGNATURE- Added: dev/airflow/2.3.2rc1/apache-airflow-2.3.2-source.tar.gz.sha512 == --- dev/airflow/2.3.2rc1/apache-airflow-2.3.2-source.tar.gz.sha512 (added) +++ dev/airflow/2.3.2rc1/apache-airflow-2.3.2-source.tar.gz.sha512 Mon May 30 11:46:09 2022 @@ -0,0 +1 @@ +fa2d0d76adff28ec03aae60e70f7dd2e17515895caa13b8b85807e7a7de0d170384b412c2ca7c21f38722271c98058dcf08ec19c02bf5026b83e20129c36e7b2 apache-airflow-2.3.2-source.tar.gz Added: dev/airflow/2.3.2rc1/apache-airflow-2.3.2.tar.gz == Binary file - no diff available. Propchange: dev/airflow/2.3.2rc1/apache-airflow-2.3.2.tar.gz -- svn:mime-type = application/octet-stream Added: dev/airflow/2.3.2rc1/apache-airflow-2.3.2.tar.gz.asc == --- dev/airflow/2.3.2rc1/apache-airflow-2.3.2.tar.gz.asc (added) +++ dev/airflow/2.3.2rc1/apache-airflow-2.3.2.tar.gz.asc Mon May 30 11:46:09 2022 @@ -0,0 +1,14 @@ +-BEGIN PGP SIGNATURE- + +iQHPBAABCgA5FiEEAEK9vayTIUi0cM3WNcli2lUp2+wFAmKUq7MbHGVwaHJhaW1h +bmllcm9iaUBhcGFjaGUub3JnAAoJEDXJYtpVKdvsK1ML/3rfa0UDQN21oGxEnl7o +dMUu+/s7czWzIz5zBPlqeS5AVYTG3xjIkb0WgOfVwO5WSUVDj11hgjBTe+mFMgJK +saqpLqakba/etVN3zyHDrVQVKDSG5mA3ho+1aU6HO7bd/D+Pee/PZhPaBQ7CJ7Jv +qXEWWnXEKZ9Bq38HVsPuAy9hE6f7d0zO82yu8GjZaUUhlM8or4L+yV5SKwtNx7Z+ +x8BxsRvCIYd6eR7SDTCg8EB4/ktowmK1g/Ax3cSMeVC/pdE0hXPmhzLSc1XNhOxk +MziNtuRWgA5fNbJ5nTy+rUHwFfG0Eb9kBGkayvlevuAPUEZyoIIGP8WVNRtiTT7w +wNnCRqEPyl285/n9KeywQrrqn6/utoVUMsFYCJZt20f1EebUB33G7iC4YpH8ud8G +abMgVQ8cvTuXtPZVLSbsPj485rgp9Jz4KjoW9jO42MkNRcRU2m5r31IT2FGSXf7g +jSlbY0JD7hsJaB9kXTeo0+Zea775SZoXdcEpwKGEK7iRkg== +=A6qB +-END PGP SIGNATURE- Added: dev/airflow/2.3.2rc1/apache-airflow-2.3.2.tar.gz.sha512 == --- dev/airflow/2.3.2rc1/apache-airflow-2.3.2.tar.gz.sha512 (added) +++ dev/airflow/2.3.2rc1/apache-airflow-2.3.2.tar.gz.sha512 Mon May 30 11:46:09 2022 @@ -0,0 +1 @@ +672af003338796b9f6a065ef652a374cc39a918a7d2b9701412c790570dcd59414cfa5bcd060ad27e63f3d902e030354aaa361b5776cc0aaddec5ede4b7b3571 apache-airflow-2.3.2.tar.gz Added: dev/airflow/2.3.2rc1/apache_airflow-2.3.2-py3-none-any.whl == Binary file - no diff available. Propchange: dev/airflow/2.3.2rc1/apache_airflow-2.3.2-py3-none-any.whl -- svn:mime-type = application/octet-stream Added: dev/airflow/2.3.2rc1/apache_airflow-2.3.2-py3-none-any.whl.asc
[airflow] branch v2-3-test updated (58b2f3a2e0 -> 537272965f)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git omit 58b2f3a2e0 Update version to 2.3.2 and add release notes add 5f466edf6a Update dep for databricks #23917 (#23927) add 7b869b6528 Fix cassandra to 3.0.25 (#23522) add 86cfcb14e7 Temporarily pin xmltodict to 0.12.0 to fix main failure (#23577) add 404a3c3cf4 Self upgrade when refreshing images (#23686) add c3a2ec7335 Doc: Add column names for DB Migration Reference (#23853) add 61f5952503 Add better feedback to Breeze users about expected action timing (#23827) add 368962ecf6 Upgrade `pip` to 22.1.1 version (just released) (#23854) add c16fedabe2 update K8S-KIND to 0.14.0 (#23859) add e0c6fdd9a1 Speed up Breeze experience on Mac OS (#23866) add ce1681cabc Few fixes in the providers release doc (#23382) add 99d62a7f70 Avoid printing exception when exiting tests command (#23897) add 7be9bf5e90 Make CI and PROD image builds consistent (#23841) add 16fc9d99d2 Clarify manual merging of PR in release doc (#23928) add 83b0fbdcd7 Fix broken main (#23940) add 74a424490a Disable rebase workflow (#23943) add 537272965f Update version to 2.3.2 and add release notes This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (58b2f3a2e0) \ N -- N -- N refs/heads/v2-3-test (537272965f) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: .asf.yaml | 2 +- .github/workflows/ci.yml | 6 +- BREEZE.rst | 8 + Dockerfile | 4 +- Dockerfile.ci | 4 +- IMAGES.rst | 2 +- RELEASE_NOTES.rst | 5 + breeze-complete| 2 +- dev/README_RELEASE_AIRFLOW.md | 7 +- dev/README_RELEASE_PROVIDER_PACKAGES.md| 17 +- dev/TRACKING_BACKTRACKING_ISSUES.md| 2 +- .../airflow_breeze/commands/ci_image_commands.py | 19 +- .../airflow_breeze/commands/developer_commands.py | 2 +- .../airflow_breeze/commands/testing_commands.py| 8 +- dev/breeze/src/airflow_breeze/global_constants.py | 4 +- .../src/airflow_breeze/params/shell_params.py | 28 +- .../airflow_breeze/utils/docker_command_utils.py | 12 + dev/breeze/src/airflow_breeze/utils/path_utils.py | 10 +- dev/breeze/src/airflow_breeze/utils/reinstall.py | 9 +- dev/refresh_images.sh | 2 + docs/apache-airflow/migrations-ref.rst | 486 +++-- docs/docker-stack/build-arg-ref.rst| 2 +- .../docker-compose/backend-mssql-docker-volume.yml | 2 + scripts/ci/docker-compose/backend-mysql.yml| 2 + scripts/ci/docker-compose/backend-postgres.yml | 2 + scripts/ci/docker-compose/backend-sqlite.yml | 2 + scripts/ci/docker-compose/base.yml | 5 - .../ci/docker-compose/integration-cassandra.yml| 3 +- scripts/ci/docker-compose/local.yml| 3 + scripts/ci/libraries/_initialization.sh| 5 +- scripts/ci/pre_commit/pre_commit_flake8.py | 7 +- scripts/ci/pre_commit/pre_commit_mypy.py | 7 +- scripts/docker/common.sh | 2 +- scripts/in_container/_in_container_script_init.sh | 18 +- scripts/in_container/_in_container_utils.sh| 12 + scripts/in_container/run_migration_reference.py| 6 + setup.py | 9 +- 37 files changed, 405 insertions(+), 321 deletions(-)
[airflow] branch v2-3-test updated (537272965f -> 4fc535e81e)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git omit 537272965f Update version to 2.3.2 and add release notes add 4cff4e052f Add cascade to `dag_tag` to `dag` foreignkey (#23444) add 09be0c5c7e Mask sensitive values for not-yet-running TIs (#23807) add 01c998c703 Replaced all days_ago functions with datetime functions (#23237) add 0682ac53cf Ignore the DeprecationWarning in test_days_ago (#23875) add 71d5eba1d8 DagFileProcessorManager: Start a new process group only if current process not a session leader (#23872) add 4fc535e81e Update version to 2.3.2 and add release notes This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (537272965f) \ N -- N -- N refs/heads/v2-3-test (4fc535e81e) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: RELEASE_NOTES.rst | 10 +-- airflow/dag_processing/manager.py | 9 ++- airflow/example_dags/example_subdag_operator.py| 5 +- airflow/migrations/utils.py| 3 +- ...0110_2_3_2_add_cascade_to_dag_tag_foreignkey.py | 84 ++ airflow/models/dag.py | 8 ++- airflow/models/taskinstance.py | 24 +++ airflow/utils/db.py| 2 +- airflow/utils/log/secrets_masker.py| 9 +-- airflow/utils/process_utils.py | 15 docs/apache-airflow/migrations-ref.rst | 4 +- tests/api/common/test_delete_dag.py| 7 +- tests/api/common/test_mark_tasks.py| 21 +++--- .../endpoints/test_extra_link_endpoint.py | 33 - tests/cli/commands/test_task_command.py| 12 ++-- tests/dag_processing/test_processor.py | 18 +++-- tests/dags/test_default_views.py | 6 +- tests/dags/test_example_bash_operator.py | 9 +-- tests/dags/test_mapped_classic.py | 5 +- tests/dags/test_mapped_taskflow.py | 5 +- tests/dags/test_miscellaneous.py | 7 +- tests/dags/test_missing_owner.py | 7 +- tests/dags/test_multiple_dags.py | 7 +- tests/dags/test_with_non_default_owner.py | 7 +- tests/jobs/test_backfill_job.py| 3 +- tests/jobs/test_scheduler_job.py | 3 +- tests/models/test_dagrun.py| 7 +- tests/operators/test_python.py | 3 +- .../google/cloud/operators/test_mlengine.py| 3 +- tests/test_utils/perf/dags/perf_dag_1.py | 14 ++-- tests/test_utils/perf/dags/perf_dag_2.py | 14 ++-- tests/utils/log/test_secrets_masker.py | 3 + tests/utils/test_dates.py | 1 + tests/utils/test_process_utils.py | 24 ++- tests/utils/test_task_group.py | 3 +- tests/www/views/test_views_decorators.py | 4 +- tests/www/views/test_views_extra_links.py | 7 +- tests/www/views/test_views_task_norun.py | 4 +- tests/www/views/test_views_tasks.py| 4 +- 39 files changed, 271 insertions(+), 143 deletions(-) create mode 100644 airflow/migrations/versions/0110_2_3_2_add_cascade_to_dag_tag_foreignkey.py
[airflow] branch main updated: DagFileProcessorManager: Start a new process group only if current process not a session leader (#23872)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 9216489d9a DagFileProcessorManager: Start a new process group only if
current process not a session leader (#23872)
9216489d9a is described below
commit 9216489d9a25f56f7a55d032b0ebfc1bf0bf4a83
Author: Andrey Anshin
AuthorDate: Fri May 27 17:29:11 2022 +0300
DagFileProcessorManager: Start a new process group only if current process
not a session leader (#23872)
---
airflow/dag_processing/manager.py | 9 ++---
airflow/utils/process_utils.py| 15 +++
tests/utils/test_process_utils.py | 19 +++
3 files changed, 40 insertions(+), 3 deletions(-)
diff --git a/airflow/dag_processing/manager.py
b/airflow/dag_processing/manager.py
index 3dc72caa99..a262cf1d42 100644
--- a/airflow/dag_processing/manager.py
+++ b/airflow/dag_processing/manager.py
@@ -49,7 +49,11 @@ from airflow.utils.file import list_py_file_paths,
might_contain_dag
from airflow.utils.log.logging_mixin import LoggingMixin
from airflow.utils.mixins import MultiprocessingStartMethodMixin
from airflow.utils.net import get_hostname
-from airflow.utils.process_utils import kill_child_processes_by_pids,
reap_process_group
+from airflow.utils.process_utils import (
+kill_child_processes_by_pids,
+reap_process_group,
+set_new_process_group,
+)
from airflow.utils.session import NEW_SESSION, provide_session
from airflow.utils.sqlalchemy import prohibit_commit, skip_locked,
with_row_locks
@@ -471,8 +475,7 @@ class DagFileProcessorManager(LoggingMixin):
"""
self.register_exit_signals()
-# Start a new process group
-os.setpgid(0, 0)
+set_new_process_group()
self.log.info("Processing files using up to %s processes at a time ",
self._parallelism)
self.log.info("Process each file at most once every %s seconds",
self._file_process_interval)
diff --git a/airflow/utils/process_utils.py b/airflow/utils/process_utils.py
index ca8fc2433e..1cbb8e8b6c 100644
--- a/airflow/utils/process_utils.py
+++ b/airflow/utils/process_utils.py
@@ -311,3 +311,18 @@ def check_if_pidfile_process_is_running(pid_file: str,
process_name: str):
except psutil.NoSuchProcess:
# If process is dead remove the pidfile
pid_lock_file.break_lock()
+
+
+def set_new_process_group() -> None:
+"""
+Tries to set current process to a new process group
+That makes it easy to kill all sub-process of this at the OS-level,
+rather than having to iterate the child processes.
+If current process spawn by system call ``exec()`` than keep current
process group
+"""
+
+if os.getpid() == os.getsid(0):
+# If PID = SID than process a session leader, and it is not possible
to change process group
+return
+
+os.setpgid(0, 0)
diff --git a/tests/utils/test_process_utils.py
b/tests/utils/test_process_utils.py
index ab76a05172..3154fa8805 100644
--- a/tests/utils/test_process_utils.py
+++ b/tests/utils/test_process_utils.py
@@ -39,6 +39,7 @@ from airflow.utils.process_utils import (
check_if_pidfile_process_is_running,
execute_in_subprocess,
execute_in_subprocess_with_kwargs,
+set_new_process_group,
)
@@ -220,3 +221,21 @@ class
TestCheckIfPidfileProcessIsRunning(unittest.TestCase):
f.flush()
with pytest.raises(AirflowException, match="is already running
under PID"):
check_if_pidfile_process_is_running(f.name,
process_name="test")
+
+
+class TestSetNewProcessGroup(unittest.TestCase):
+@mock.patch("os.setpgid")
+def test_not_session_leader(self, mock_set_pid):
+pid = os.getpid()
+with mock.patch('os.getsid', autospec=True) as mock_get_sid:
+mock_get_sid.return_value = pid + 1
+set_new_process_group()
+assert mock_set_pid.call_count == 1
+
+@mock.patch("os.setpgid")
+def test_session_leader(self, mock_set_pid):
+pid = os.getpid()
+with mock.patch('os.getsid', autospec=True) as mock_get_sid:
+mock_get_sid.return_value = pid
+set_new_process_group()
+assert mock_set_pid.call_count == 0
[airflow] branch v2-3-test updated (4fc535e81e -> 345255bed2)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git from 4fc535e81e Update version to 2.3.2 and add release notes add 345255bed2 Apply suggestions from code review No new revisions were added by this update. Summary of changes: RELEASE_NOTES.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
[airflow] branch v2-3-test updated (804a246652 -> bf1d12975c)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git omit 804a246652 Update version to 2.3.2 and add release notes add bf1d12975c Update version to 2.3.2 and add release notes This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (804a246652) \ N -- N -- N refs/heads/v2-3-test (bf1d12975c) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: RELEASE_NOTES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[airflow] branch v2-3-test updated: Update version to 2.3.2 and add release notes
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/v2-3-test by this push: new 58b2f3a2e0 Update version to 2.3.2 and add release notes 58b2f3a2e0 is described below commit 58b2f3a2e069290f83d7554b3e1de4ad63035525 Author: Ephraim Anierobi AuthorDate: Fri May 27 13:40:44 2022 +0100 Update version to 2.3.2 and add release notes --- README.md | 14 +++--- RELEASE_NOTES.rst | 18 ++ airflow/utils/db.py| 1 + .../apache-airflow/installation/supported-versions.rst | 2 +- docs/docker-stack/README.md| 10 +- .../customizing/pypi-extras-and-deps.sh| 2 +- .../customizing/pypi-selected-version.sh | 2 +- .../extending/add-apt-packages/Dockerfile | 2 +- .../extending/add-build-essential-extend/Dockerfile| 2 +- .../docker-examples/extending/add-providers/Dockerfile | 2 +- .../extending/add-pypi-packages/Dockerfile | 2 +- .../extending/custom-providers/Dockerfile | 2 +- .../extending/embedding-dags/Dockerfile| 2 +- .../extending/writable-directory/Dockerfile| 2 +- docs/docker-stack/entrypoint.rst | 18 +- scripts/ci/pre_commit/pre_commit_supported_versions.py | 2 +- setup.py | 2 +- 17 files changed, 52 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 3acb883dc4..3f38a69808 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ Airflow is not a streaming solution, but it is often used to process real-time d Apache Airflow is tested with: -| | Main version (dev) | Stable version (2.3.1) | +| | Main version (dev) | Stable version (2.3.2) | |-|--|--| | Python | 3.7, 3.8, 3.9, 3.10 | 3.7, 3.8, 3.9, 3.10 | | Platform| AMD64/ARM64(\*) | AMD64/ARM64(\*) | @@ -160,15 +160,15 @@ them to the appropriate format and workflow that your tool requires. ```bash -pip install 'apache-airflow==2.3.1' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.3.1/constraints-3.7.txt; +pip install 'apache-airflow==2.3.2' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.3.2/constraints-3.7.txt; ``` 2. Installing with extras (i.e., postgres, google) ```bash -pip install 'apache-airflow[postgres,google]==2.3.1' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.3.1/constraints-3.7.txt; +pip install 'apache-airflow[postgres,google]==2.3.2' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.3.2/constraints-3.7.txt; ``` For information on installing provider packages, check @@ -273,7 +273,7 @@ Apache Airflow version life cycle: | Version | Current Patch/Minor | State | First Release | Limited Support | EOL/Terminated | |---|---|---|-|---|--| -| 2 | 2.3.1 | Supported | Dec 17, 2020| TBD | TBD | +| 2 | 2.3.2 | Supported | Dec 17, 2020| TBD | TBD | | 1.10 | 1.10.15 | EOL | Aug 27, 2018| Dec 17, 2020 | June 17, 2021| | 1.9 | 1.9.0 | EOL | Jan 03, 2018| Aug 27, 2018 | Aug 27, 2018 | | 1.8 | 1.8.2 | EOL | Mar 19, 2017| Jan 03, 2018 | Jan 03, 2018 | @@ -303,7 +303,7 @@ They are based on the official release schedule of Python and Kubernetes, nicely 2. The "oldest" supported version of Python/Kubernetes is the default one until we decide to switch to later version. "Default" is only meaningful in terms of "smoke tests" in CI PRs, which are run using this default version and the default reference image available. Currently `apache/airflow:latest` - and `apache/airflow:2.3.1` images are Python 3.7 images. This means that default reference image will + and `apache/airflow:2.3.2` images are Python 3.7 images. This means that default reference image will become the default at the time when we start preparing for dropping 3.7 support which is few months before the end of life for Python 3.7. diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index ee76dec6bb..6bd5a0bcfa 100644 --- a/RELEASE
[airflow] branch v2-3-test updated (345255bed2 -> 804a246652)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git omit 345255bed2 Apply suggestions from code review omit 4fc535e81e Update version to 2.3.2 and add release notes add db929386d4 Add is_mapped field to Task response. (#23319) add 75285614e8 Disallow calling expand with no arguments (#23463) add 804a246652 Update version to 2.3.2 and add release notes This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (345255bed2) \ N -- N -- N refs/heads/v2-3-test (804a246652) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: RELEASE_NOTES.rst | 4 + airflow/api_connexion/openapi/v1.yaml | 3 + airflow/api_connexion/schemas/task_schema.py | 1 + airflow/decorators/base.py | 3 + airflow/models/mappedoperator.py | 5 + airflow/serialization/serialized_objects.py| 3 +- .../api_connexion/endpoints/test_task_endpoint.py | 116 - tests/api_connexion/schemas/test_task_schema.py| 2 + tests/models/test_taskinstance.py | 8 +- 9 files changed, 141 insertions(+), 4 deletions(-)
[airflow-site] branch announce-airflow-2.3.1 updated (912e07878 -> 2f83fa028)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch announce-airflow-2.3.1 in repository https://gitbox.apache.org/repos/asf/airflow-site.git omit 912e07878 Announce Apache Airflow 2.3.1 add 2f83fa028 Announce Apache Airflow 2.3.1 This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (912e07878) \ N -- N -- N refs/heads/announce-airflow-2.3.1 (2f83fa028) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: landing-pages/site/content/en/announcements/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[airflow-site] branch announce-airflow-2.3.1 updated (99ff5bd1f -> 912e07878)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch announce-airflow-2.3.1 in repository https://gitbox.apache.org/repos/asf/airflow-site.git omit 99ff5bd1f Announce Apache Airflow 2.3.1 add 912e07878 Announce Apache Airflow 2.3.1 This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (99ff5bd1f) \ N -- N -- N refs/heads/announce-airflow-2.3.1 (912e07878) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: landing-pages/site/content/en/announcements/_index.md | 9 - 1 file changed, 4 insertions(+), 5 deletions(-)
[airflow] branch main updated: Chart: Update default airflow version to `2.3.1` (#23913)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 3aec3aa81c Chart: Update default airflow version to `2.3.1` (#23913)
3aec3aa81c is described below
commit 3aec3aa81c84bed626e022462e414aebe0f8277e
Author: Ephraim Anierobi
AuthorDate: Wed May 25 20:05:42 2022 +0100
Chart: Update default airflow version to `2.3.1` (#23913)
---
chart/Chart.yaml | 20 ++--
chart/values.schema.json | 4 ++--
chart/values.yaml| 4 ++--
3 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/chart/Chart.yaml b/chart/Chart.yaml
index ee394ab339..62f3538bcb 100644
--- a/chart/Chart.yaml
+++ b/chart/Chart.yaml
@@ -20,7 +20,7 @@
apiVersion: v2
name: airflow
version: 1.7.0-dev
-appVersion: 2.3.0
+appVersion: 2.3.1
description: The official Helm chart to deploy Apache Airflow, a platform to
programmatically author, schedule, and monitor workflows
home: https://airflow.apache.org/
@@ -47,23 +47,23 @@ annotations:
url: https://airflow.apache.org/docs/helm-chart/1.6.0/
artifacthub.io/screenshots: |
- title: DAGs View
- url:
https://airflow.apache.org/docs/apache-airflow/2.3.0/_images/dags.png
+ url:
https://airflow.apache.org/docs/apache-airflow/2.3.1/_images/dags.png
- title: Tree View
- url:
https://airflow.apache.org/docs/apache-airflow/2.3.0/_images/grid.png
+ url:
https://airflow.apache.org/docs/apache-airflow/2.3.1/_images/grid.png
- title: Graph View
- url:
https://airflow.apache.org/docs/apache-airflow/2.3.0/_images/graph.png
+ url:
https://airflow.apache.org/docs/apache-airflow/2.3.1/_images/graph.png
- title: Calendar View
- url:
https://airflow.apache.org/docs/apache-airflow/2.3.0/_images/calendar.png
+ url:
https://airflow.apache.org/docs/apache-airflow/2.3.1/_images/calendar.png
- title: Variable View
- url:
https://airflow.apache.org/docs/apache-airflow/2.3.0/_images/variable_hidden.png
+ url:
https://airflow.apache.org/docs/apache-airflow/2.3.1/_images/variable_hidden.png
- title: Gantt Chart
- url:
https://airflow.apache.org/docs/apache-airflow/2.3.0/_images/gantt.png
+ url:
https://airflow.apache.org/docs/apache-airflow/2.3.1/_images/gantt.png
- title: Task Duration
- url:
https://airflow.apache.org/docs/apache-airflow/2.3.0/_images/duration.png
+ url:
https://airflow.apache.org/docs/apache-airflow/2.3.1/_images/duration.png
- title: Code View
- url:
https://airflow.apache.org/docs/apache-airflow/2.3.0/_images/code.png
+ url:
https://airflow.apache.org/docs/apache-airflow/2.3.1/_images/code.png
- title: Task Instance Context Menu
- url:
https://airflow.apache.org/docs/apache-airflow/2.3.0/_images/context.png
+ url:
https://airflow.apache.org/docs/apache-airflow/2.3.1/_images/context.png
artifacthub.io/changes: |
- description: Support ``annotations`` on ``volumeClaimTemplates``
kind: added
diff --git a/chart/values.schema.json b/chart/values.schema.json
index 0900616d0d..7abd24c6af 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -67,13 +67,13 @@
"defaultAirflowTag": {
"description": "Default airflow tag to deploy.",
"type": "string",
-"default": "2.3.0",
+"default": "2.3.1",
"x-docsSection": "Common"
},
"airflowVersion": {
"description": "Airflow version (Used to make some decisions based
on Airflow Version being deployed).",
"type": "string",
-"default": "2.3.0",
+"default": "2.3.1",
"x-docsSection": "Common"
},
"securityContext": {
diff --git a/chart/values.yaml b/chart/values.yaml
index 578a31fef5..25767e0940 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -46,10 +46,10 @@ airflowHome: /opt/airflow
defaultAirflowRepository: apache/airflow
# Default airflow tag to deploy
-defaultAirflowTag: "2.3.0"
+defaultAirflowTag: "2.3.1"
# Airflow version (Used to make some decisions based on Airflow Version being
deployed)
-airflowVersion: "2.3.0"
+airflowVersion: "2.3.1"
# Images
images:
[airflow] branch main updated: Clarify manual merging of PR in release doc (#23928)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new bfe51ea64a Clarify manual merging of PR in release doc (#23928)
bfe51ea64a is described below
commit bfe51ea64a6957587c33baf32916b205dea91a45
Author: Ephraim Anierobi
AuthorDate: Thu May 26 15:52:10 2022 +0100
Clarify manual merging of PR in release doc (#23928)
It was not clear to me what this really means
---
dev/README_RELEASE_AIRFLOW.md | 7 ++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md
index ebe94e6abb..6c050ebd4a 100644
--- a/dev/README_RELEASE_AIRFLOW.md
+++ b/dev/README_RELEASE_AIRFLOW.md
@@ -224,7 +224,12 @@ The Release Candidate artifacts we vote upon should be the
exact ones we vote ag
- Update the `REVISION_HEADS_MAP` at airflow/utils/db.py to include the
revision head of the release even if there are no migrations.
- Commit the version change.
-- PR from the 'test' branch to the 'stable' branch, and manually merge it once
approved.
+- PR from the 'test' branch to the 'stable' branch, and manually merge it once
approved. Here's how to manually merge the PR:
+
+```shell script
+git merge --ff-only v${VERSION_BRANCH}-test
+```
+
- Check out the 'stable' branch
```shell script
[airflow] branch main updated: Removing magic status code numbers from api_connecxion (#24050)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f2e6452efd Removing magic status code numbers from api_connecxion
(#24050)
f2e6452efd is described below
commit f2e6452efdf74828de9f73a8bf4f42f6dd10eb58
Author: Bernardo Couto <35502483+bernardoco...@users.noreply.github.com>
AuthorDate: Thu Jun 2 14:52:19 2022 -0300
Removing magic status code numbers from api_connecxion (#24050)
---
airflow/api_connexion/endpoints/config_endpoint.py | 4 +++-
airflow/api_connexion/endpoints/connection_endpoint.py | 3 ++-
airflow/api_connexion/endpoints/dag_endpoint.py | 3 ++-
airflow/api_connexion/endpoints/dag_run_endpoint.py | 3 ++-
airflow/api_connexion/endpoints/dag_source_endpoint.py | 4 +++-
airflow/api_connexion/endpoints/pool_endpoint.py| 4 +++-
.../api_connexion/endpoints/role_and_permission_endpoint.py | 3 ++-
airflow/api_connexion/endpoints/user_endpoint.py| 3 ++-
airflow/api_connexion/endpoints/variable_endpoint.py| 3 ++-
airflow/api_connexion/exceptions.py | 13 +++--
10 files changed, 28 insertions(+), 15 deletions(-)
diff --git a/airflow/api_connexion/endpoints/config_endpoint.py
b/airflow/api_connexion/endpoints/config_endpoint.py
index 9514621447..bdd2b3a959 100644
--- a/airflow/api_connexion/endpoints/config_endpoint.py
+++ b/airflow/api_connexion/endpoints/config_endpoint.py
@@ -15,6 +15,8 @@
# specific language governing permissions and limitations
# under the License.
+from http import HTTPStatus
+
from flask import Response, request
from airflow.api_connexion import security
@@ -72,7 +74,7 @@ def get_config() -> Response:
}
return_type = request.accept_mimetypes.best_match(serializer.keys())
if return_type not in serializer:
-return Response(status=406)
+return Response(status=HTTPStatus.NOT_ACCEPTABLE)
elif conf.getboolean("webserver", "expose_config"):
conf_dict = conf.as_dict(display_source=False, display_sensitive=True)
config = _conf_dict_to_config(conf_dict)
diff --git a/airflow/api_connexion/endpoints/connection_endpoint.py
b/airflow/api_connexion/endpoints/connection_endpoint.py
index f9be9c227e..b196b3236b 100644
--- a/airflow/api_connexion/endpoints/connection_endpoint.py
+++ b/airflow/api_connexion/endpoints/connection_endpoint.py
@@ -16,6 +16,7 @@
# under the License.
import os
+from http import HTTPStatus
from connexion import NoContent
from flask import request
@@ -51,7 +52,7 @@ def delete_connection(*, connection_id: str, session: Session
= NEW_SESSION) ->
detail=f"The Connection with connection_id: `{connection_id}` was
not found",
)
session.delete(connection)
-return NoContent, 204
+return NoContent, HTTPStatus.NO_CONTENT
@security.requires_access([(permissions.ACTION_CAN_READ,
permissions.RESOURCE_CONNECTION)])
diff --git a/airflow/api_connexion/endpoints/dag_endpoint.py
b/airflow/api_connexion/endpoints/dag_endpoint.py
index e94707b127..0505f864ee 100644
--- a/airflow/api_connexion/endpoints/dag_endpoint.py
+++ b/airflow/api_connexion/endpoints/dag_endpoint.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from http import HTTPStatus
from typing import Collection, Optional
from connexion import NoContent
@@ -176,4 +177,4 @@ def delete_dag(dag_id: str, session: Session = NEW_SESSION)
-> APIResponse:
except AirflowException:
raise AlreadyExists(detail=f"Task instances of dag with id: '{dag_id}'
are still running")
-return NoContent, 204
+return NoContent, HTTPStatus.NO_CONTENT
diff --git a/airflow/api_connexion/endpoints/dag_run_endpoint.py
b/airflow/api_connexion/endpoints/dag_run_endpoint.py
index 9fde3db885..e510126534 100644
--- a/airflow/api_connexion/endpoints/dag_run_endpoint.py
+++ b/airflow/api_connexion/endpoints/dag_run_endpoint.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from http import HTTPStatus
from typing import List, Optional, Tuple
import pendulum
@@ -62,7 +63,7 @@ def delete_dag_run(*, dag_id: str, dag_run_id: str, session:
Session = NEW_SESSI
"""Delete a DAG Run"""
if session.query(DagRun).filter(DagRun.dag_id == dag_id, DagRun.run_id ==
dag_run_id).delete() == 0:
raise NotFound(detail=f"DAGRun with DAG ID: '{dag_id}' and DagRun ID:
'{dag_run_id}' not found")
-return NoContent, 204
+return NoContent, HTTPStatus.NO_CONTENT
@security.requires_access(
diff --git a/airflow/api_connexion/e
[airflow] branch main updated (f2e6452efd -> 75fdbf01dd)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from f2e6452efd Removing magic status code numbers from api_connecxion (#24050) add 75fdbf01dd Do not support MSSQL less than v2017 in code (#24095) No new revisions were added by this update. Summary of changes: airflow/migrations/db_types.py | 25 -- airflow/migrations/db_types.pyi| 2 -- .../0088_2_2_0_improve_mssql_compatibility.py | 24 ++--- 3 files changed, 11 insertions(+), 40 deletions(-)
[airflow] branch v2-3-test updated (972bbaff8a -> 04e700d0c7)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git omit 972bbaff8a Update version to 2.3.2 and add release notes add 04e700d0c7 Update version to 2.3.2 and add release notes This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (972bbaff8a) \ N -- N -- N refs/heads/v2-3-test (04e700d0c7) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes:
[airflow] 07/12: Increase size of ARM build instance (#24036)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit c45eb48bd5780b1fde40255b8916313aea59643f
Author: Jarek Potiuk
AuthorDate: Tue May 31 10:15:03 2022 +0200
Increase size of ARM build instance (#24036)
Our ARM cache builds started to hang recently at yarn prod step.
The most likely reason are limited resources we had for the ARM
instance to run the docker build - it was rather small instance
with 2GB RAM and it is likely not nearly enought to cope with
recent changes related to Grid View where we likely need much
more memory during the yarn build step.
This change increases the instance memory to 8 GB (c6g.xlarge).
Also this instance type gives 70% cost saving and has very low
probability of being evicted (it's not in high demand in Ohio
Region of AWS.
Also the AMI used is refreshed with latest software (docker)
(cherry picked from commit b55e4c3e2ac5bf3c7ea9f467447fa1c661da936d)
---
.github/workflows/ci.yml | 6 +++---
scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh | 4 ++--
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4b82120db2..c5438eebb2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1653,7 +1653,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
permissions:
packages: write
timeout-minutes: 120
-name: "Push images as cache to GitHub Registry"
+name: "Push Image Cache"
runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
needs:
- build-info
@@ -1687,7 +1687,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Start ARM instance"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
if: matrix.platform == 'linux/arm64'
- - name: "Build & Push CI image ${{ matrix.python-version }}:latest"
+ - name: "Push CI cache ${{ matrix.python-version }} ${{ matrix.platform
}}"
run: >
breeze build-image
--prepare-buildx-cache
@@ -1717,7 +1717,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
VERSION_SUFFIX_FOR_PYPI: "dev0"
- name: "Move dist packages to docker-context files"
run: mv -v ./dist/*.whl ./docker-context-files
- - name: Build & Push PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION
}}:latest
+ - name: "Push PROD cache ${{ matrix.python-version }} ${{
matrix.platform }}"
run: >
breeze build-prod-image
--airflow-is-in-context
diff --git a/scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
b/scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
index c65ecd7453..a3fbf6b5ba 100755
--- a/scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
+++ b/scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
@@ -22,8 +22,8 @@ SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"
# This is an AMI that is based on Basic Amazon Linux AMI with installed and
configured docker service
WORKING_DIR="/tmp/armdocker"
INSTANCE_INFO="${WORKING_DIR}/instance_info.json"
-ARM_AMI="ami-002fa24639ab2520a"
-INSTANCE_TYPE="c6gd.medium"
+ARM_AMI="ami-06b8158ea372d3259"
+INSTANCE_TYPE="c6g.xlarge"
MARKET_OPTIONS="MarketType=spot,SpotOptions={MaxPrice=0.1,SpotInstanceType=one-time}"
REGION="us-east-2"
EC2_USER="ec2-user"
[airflow] 01/12: Add better diagnostics capabilities for pre-commits run via CI image (#23980)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 012b88d96c45ad7ceade25a5d4273aa776802486
Author: Jarek Potiuk
AuthorDate: Sun May 29 10:33:40 2022 +0200
Add better diagnostics capabilities for pre-commits run via CI image
(#23980)
The pre-commits that require CI image run docker command under
the hood that is highly optimized for performance (only mounts
files that are necessary to be mounted) - in order to improve
performance on Mac OS and make sure that artifacts are not left
in the source code of Airflow.
However that makes the command slightly more difficult to debug
because they generate dynamically the docker command used,
including the volumens that should be mounted when the docker
command is run.
This PR adds better diagnostics to the pre-commit scripts
allowing VERBOSE="true" and DRY_RUN="true" variables that can
help with diagnosing problems such as running the scripts on
WSL2.
It also fixes a few documentation bugs that have been missed
after changing names of the image-related static checks and
thanks to separating the common code to utility function
it allows to set SKIP_IMAGE_PRE_COMMITS variable to true
which will skip running all pre-commit checks that require
breeze image to be available locally.
(cherry picked from commit 5af83ce736b92f651f75d5d00c2043be59ab91f5)
---
.github/workflows/ci.yml | 11 +-
STATIC_CODE_CHECKS.rst | 52 ++---
.../airflow_breeze/commands/developer_commands.py | 2 +-
dev/breeze/src/airflow_breeze/utils/path_utils.py | 8 +-
dev/breeze/src/airflow_breeze/utils/run_utils.py | 30 ++
images/breeze/output-commands-hash.txt | 2 +-
images/breeze/output-static-checks.svg | 120 ++---
.../pre_commit_check_pre_commit_hooks.py | 2 +-
scripts/ci/pre_commit/pre_commit_flake8.py | 28 +++--
.../pre_commit/pre_commit_migration_reference.py | 34 +++---
scripts/ci/pre_commit/pre_commit_mypy.py | 28 +++--
scripts/ci/pre_commit/pre_commit_ui_lint.py| 36 +++
scripts/ci/pre_commit/pre_commit_www_lint.py | 35 +++---
13 files changed, 215 insertions(+), 173 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5488b765be..e8de7a7f9b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -637,9 +637,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Static checks"
run: breeze static-checks --all-files --show-diff-on-failure --color
always
env:
- VERBOSE: false
+ VERBOSE: "false"
SKIP: "identity"
- COLUMNS: 250
+ COLUMNS: "250"
- name: "Fix ownership"
run: breeze fix-ownership
if: always()
@@ -654,8 +654,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
needs: [build-info]
env:
RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- SKIP:
"build,run-mypy,run-flake8,lint-javascript,update-migration-references,identity"
- MOUNT_SELECTED_LOCAL_SOURCES: "true"
if: needs.build-info.outputs.basic-checks-only == 'true'
steps:
- name: Cleanup repo
@@ -695,7 +693,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
breeze static-checks --all-files --show-diff-on-failure --color
always
--commit-ref "${{ github.sha }}"
env:
- VERBOSE: false
+ VERBOSE: "false"
+ SKIP_IMAGE_PRE_COMMITS: "true"
+ SKIP: "identity"
+ COLUMNS: "250"
- name: "Fix ownership"
run: breeze fix-ownership
if: always()
diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst
index 71f132e7d5..e2b57eae6c 100644
--- a/STATIC_CODE_CHECKS.rst
+++ b/STATIC_CODE_CHECKS.rst
@@ -111,8 +111,8 @@ For details on advanced usage of the install method, use:
Available pre-commit checks
...
-This table lists pre-commit hooks used by Airflow. The ``Breeze`` column
indicates which hooks
-require Breeze Docker images to be installed locally.
+This table lists pre-commit hooks used by Airflow. The ``Image`` column
indicates which hooks
+require Breeze Docker image to be build locally.
.. note:: Disabling particular checks
@@ -123,6 +123,10 @@ require Breeze Docker images to be installed locally.
``export SKIP=run-flake8,run-mypy``. You can also add this to your
``.bashrc`` or ``.zshrc`` if you
do not want to set it manually every time you enter the terminal.
+ In case you do not have breeze image configured lo
[airflow] 04/12: Preparing buildx cache is allowed without --push-image flag (#24028)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git commit 8ddd7ac474332d41c2775dbd5534598b6bce67e2 Author: Jarek Potiuk AuthorDate: Mon May 30 17:09:35 2022 +0200 Preparing buildx cache is allowed without --push-image flag (#24028) The previous version of buildx cache preparation implied --push-image flag, but now this is completely separated (we do not push image, we just prepare cache), so when mutli-platform buildx preparation is run we should also allow the cache to run without --push-image flag. (cherry picked from commit 8f3a9b8542346c35712cba373baaafb518503562) --- dev/breeze/src/airflow_breeze/commands/ci_image_commands.py | 9 +++-- .../src/airflow_breeze/commands/production_image_commands.py | 9 +++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index c27e5cc25d..b9e0288540 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -444,9 +444,14 @@ def build_ci_image(verbose: bool, dry_run: bool, ci_image_params: BuildCiParams) :param dry_run: do not execute "write" commands - just print what would happen :param ci_image_params: CI image parameters """ -if not ci_image_params.push_image and ci_image_params.is_multi_platform(): +if ( +ci_image_params.is_multi_platform() +and not ci_image_params.push_image +and not ci_image_params.prepare_buildx_cache +): get_console().print( -"\n[red]You cannot use multi-platform build without using --push-image flag![/]\n" +"\n[red]You cannot use multi-platform build without using --push-image flag or " +"preparing buildx cache![/]\n" ) return 1, "Error: building multi-platform image without --push-image." fix_group_permissions(verbose=verbose) diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py index e10d46cda4..544095f0b2 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -480,9 +480,14 @@ def build_production_image( :param dry_run: do not execute "write" commands - just print what would happen :param prod_image_params: PROD image parameters """ -if not prod_image_params.push_image and prod_image_params.is_multi_platform(): +if ( +prod_image_params.is_multi_platform() +and not prod_image_params.push_image +and not prod_image_params.prepare_buildx_cache +): get_console().print( -"\n[red]You cannot use multi-platform build without using --push-image flag![/]\n" +"\n[red]You cannot use multi-platform build without using --push-image flag" +" or preparing buildx cache![/]\n" ) return 1, "Error: building multi-platform image without --push-image." fix_group_permissions(verbose=verbose)
[airflow] 10/12: Re-add --force-build flag (#24061)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 1555e67d37edbba38230eff02ac9c6eab6d9a604
Author: Jarek Potiuk
AuthorDate: Wed Jun 1 01:55:31 2022 +0200
Re-add --force-build flag (#24061)
After #24052 we also need to add --force-build flag as for
Python 3.7 rebuilding CI cache would have been silently ignored as
no image building would be needed
(cherry picked from commit 5005ff2735b0bb80dc913f2070d3a507156a9394)
---
.github/workflows/ci.yml| 1 +
dev/breeze/src/airflow_breeze/commands/ci_image_commands.py | 1 +
dev/refresh_images.sh | 1 +
images/breeze/output-commands-hash.txt | 1 -
4 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 781cdbb08e..05781d33d7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1712,6 +1712,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: >
breeze build-image
--prepare-buildx-cache
+ --force-build
--platform ${{ matrix.platform }}
env:
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
index 42054b1244..889a9d7bef 100644
--- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
@@ -217,6 +217,7 @@ CI_IMAGE_TOOLS_PARAMETERS = {
@option_additional_runtime_apt_command
@option_dev_apt_command
@option_dev_apt_deps
+@option_force_build
@option_runtime_apt_command
@option_runtime_apt_deps
@option_force_build
diff --git a/dev/refresh_images.sh b/dev/refresh_images.sh
index 333b3b87eb..ed5591585a 100755
--- a/dev/refresh_images.sh
+++ b/dev/refresh_images.sh
@@ -28,6 +28,7 @@ breeze self-upgrade --force
breeze build-image \
--build-multiple-images \
--prepare-buildx-cache \
+ --force-build \
--platform linux/amd64,linux/arm64 \
--verbose
diff --git a/images/breeze/output-commands-hash.txt
b/images/breeze/output-commands-hash.txt
deleted file mode 100644
index cf38decd7f..00
--- a/images/breeze/output-commands-hash.txt
+++ /dev/null
@@ -1 +0,0 @@
-ce7896bb79ac7862ff8a4d6c7e326671
[airflow] 11/12: Fix grid view for mapped tasks (#24059)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit c9bba620986b7db6347c7d5d0264826709579f30
Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com>
AuthorDate: Tue May 31 18:12:10 2022 -0600
Fix grid view for mapped tasks (#24059)
(cherry picked from commit 6e96f04eb515149f185448b8dfb84813c5879fc0)
---
airflow/www/utils.py | 8 +---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/airflow/www/utils.py b/airflow/www/utils.py
index 48512f03fa..8c05f37885 100644
--- a/airflow/www/utils.py
+++ b/airflow/www/utils.py
@@ -113,10 +113,11 @@ def get_mapped_summary(parent_instance, task_instances):
)
try_count = (
-parent_instance.prev_attempted_tries
-if parent_instance.prev_attempted_tries != 0
-else parent_instance.try_number
+parent_instance._try_number
+if parent_instance._try_number != 0 or parent_instance.state in
State.running
+else parent_instance._try_number + 1
)
+
return {
'task_id': parent_instance.task_id,
'run_id': parent_instance.run_id,
@@ -130,6 +131,7 @@ def get_mapped_summary(parent_instance, task_instances):
def get_task_summaries(task, dag_runs: List[DagRun], session: Session) ->
List[Dict[str, Any]]:
tis = session.query(
+TaskInstance.dag_id,
TaskInstance.task_id,
TaskInstance.run_id,
TaskInstance.map_index,
[airflow] 08/12: Upgrade to pip 22.1.2 (#24043)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 34dc2ca40605af8d1fc969bab9792ce592149d49
Author: Jarek Potiuk
AuthorDate: Tue May 31 16:49:47 2022 +0200
Upgrade to pip 22.1.2 (#24043)
Pip has been upgraded to version 22.1.2 12 minutes ago. Time to
catch up.
(cherry picked from commit 2247b6d43c81b345ab465c7cd2e97a3ab20b117a)
---
Dockerfile | 4 ++--
Dockerfile.ci | 4 ++--
IMAGES.rst | 2 +-
dev/TRACKING_BACKTRACKING_ISSUES.md | 2 +-
docs/docker-stack/build-arg-ref.rst | 2 +-
scripts/ci/libraries/_initialization.sh | 2 +-
scripts/docker/common.sh| 2 +-
7 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 58afdb2ac4..39049dcb4d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -48,7 +48,7 @@ ARG AIRFLOW_VERSION="2.2.4"
ARG PYTHON_BASE_IMAGE="python:3.7-slim-bullseye"
-ARG AIRFLOW_PIP_VERSION=22.1.1
+ARG AIRFLOW_PIP_VERSION=22.1.2
ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow;
ARG
AIRFLOW_IMAGE_README_URL="https://raw.githubusercontent.com/apache/airflow/main/docs/docker-stack/README.md;
@@ -367,7 +367,7 @@ function common::get_airflow_version_specification() {
function common::override_pip_version_if_needed() {
if [[ -n ${AIRFLOW_VERSION} ]]; then
if [[ ${AIRFLOW_VERSION} =~ ^2\.0.* || ${AIRFLOW_VERSION} =~ ^1\.* ]];
then
-export AIRFLOW_PIP_VERSION="22.1.1"
+export AIRFLOW_PIP_VERSION="22.1.2"
fi
fi
}
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 457eaf9aa4..41aaf579ec 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -327,7 +327,7 @@ function common::get_airflow_version_specification() {
function common::override_pip_version_if_needed() {
if [[ -n ${AIRFLOW_VERSION} ]]; then
if [[ ${AIRFLOW_VERSION} =~ ^2\.0.* || ${AIRFLOW_VERSION} =~ ^1\.* ]];
then
-export AIRFLOW_PIP_VERSION="22.1.1"
+export AIRFLOW_PIP_VERSION="22.1.2"
fi
fi
}
@@ -1195,7 +1195,7 @@ ARG AIRFLOW_CI_BUILD_EPOCH="3"
ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true"
# By default in the image, we are installing all providers when installing
from sources
ARG INSTALL_PROVIDERS_FROM_SOURCES="true"
-ARG AIRFLOW_PIP_VERSION=22.1.1
+ARG AIRFLOW_PIP_VERSION=22.1.2
# Setup PIP
# By default PIP install run without cache to make image smaller
ARG PIP_NO_CACHE_DIR="true"
diff --git a/IMAGES.rst b/IMAGES.rst
index 0a16bf18b7..58ef0cca54 100644
--- a/IMAGES.rst
+++ b/IMAGES.rst
@@ -475,7 +475,7 @@ The following build arguments (``--build-arg`` in docker
build command) can be u
| ``ADDITIONAL_RUNTIME_APT_ENV`` |
| Additional env variables defined |
| |
| when installing runtime deps |
+--+--+--+
-| ``AIRFLOW_PIP_VERSION`` | ``22.1.1``
| PIP version used.|
+| ``AIRFLOW_PIP_VERSION`` | ``22.1.2``
| PIP version used.|
+--+--+--+
| ``PIP_PROGRESS_BAR`` | ``on``
| Progress bar for PIP installation|
+--+--+--+
diff --git a/dev/TRACKING_BACKTRACKING_ISSUES.md
b/dev/TRACKING_BACKTRACKING_ISSUES.md
index 57de81e042..27afc390ab 100644
--- a/dev/TRACKING_BACKTRACKING_ISSUES.md
+++ b/dev/TRACKING_BACKTRACKING_ISSUES.md
@@ -42,7 +42,7 @@ image build jobs in CI.
An example of such issue is described
[here](https://github.com/pypa/pip/issues/10924).
Unfortunately the problem is that in such cases, it is not possible to figure
out what caused the
-problem from `pip` output (state as of `pip` 22.1.1).
+problem from `pip` output (state as of `pip` 22.1.2).
There are a number of issues in `pip` that describe the issue, and some
backtracking reasons have been already
tracked down and fixed by `pip` maintainers, but this is a difficult problem
to solve and it is likely it
diff --git a/docs/docker-stack/build-arg-ref.rst
b/docs/docker-stack/build-arg-ref.rst
index 74078c84dd..f8e30c02ce 100644
--- a/docs/docker-stack/build-arg-ref.rst
+++ b/docs/docker-stack/build-a
[airflow] 02/12: Ensure @contextmanager decorates generator func (#23103)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git commit d2b766f8cc779fa1b22a6200c52782dad453e02b Author: Tzu-ping Chung AuthorDate: Mon May 30 03:24:08 2022 -0400 Ensure @contextmanager decorates generator func (#23103) (cherry picked from commit e58985598f202395098e15b686aec33645a906ff) --- airflow/cli/commands/task_command.py | 4 ++-- airflow/models/taskinstance.py| 3 +-- airflow/providers/google/cloud/hooks/gcs.py | 19 --- .../google/cloud/utils/credentials_provider.py| 9 ++--- airflow/providers/google/common/hooks/base_google.py | 10 +- airflow/providers/microsoft/psrp/hooks/psrp.py| 4 ++-- airflow/utils/db.py | 11 --- airflow/utils/process_utils.py| 4 ++-- airflow/utils/session.py | 4 ++-- dev/breeze/src/airflow_breeze/utils/run_utils.py | 4 ++-- dev/provider_packages/prepare_provider_packages.py| 4 ++-- 11 files changed, 48 insertions(+), 28 deletions(-) diff --git a/airflow/cli/commands/task_command.py b/airflow/cli/commands/task_command.py index ea20ebb646..2b743b91fe 100644 --- a/airflow/cli/commands/task_command.py +++ b/airflow/cli/commands/task_command.py @@ -23,7 +23,7 @@ import logging import os import textwrap from contextlib import contextmanager, redirect_stderr, redirect_stdout -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, Generator, List, Optional, Tuple, Union from pendulum.parsing.exceptions import ParserError from sqlalchemy.orm.exc import NoResultFound @@ -269,7 +269,7 @@ def _extract_external_executor_id(args) -> Optional[str]: @contextmanager -def _capture_task_logs(ti): +def _capture_task_logs(ti: TaskInstance) -> Generator[None, None, None]: """Manage logging context for a task run - Replace the root logger configuration with the airflow.task configuration diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index 2885d56b54..5cd582ce3e 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -40,7 +40,6 @@ from typing import ( Dict, Generator, Iterable, -Iterator, List, NamedTuple, Optional, @@ -142,7 +141,7 @@ if TYPE_CHECKING: @contextlib.contextmanager -def set_current_context(context: Context) -> Iterator[Context]: +def set_current_context(context: Context) -> Generator[Context, None, None]: """ Sets the current execution context to the provided context object. This method should be called once per Task execution, before calling operator.execute. diff --git a/airflow/providers/google/cloud/hooks/gcs.py b/airflow/providers/google/cloud/hooks/gcs.py index 29ad6ac438..93717e00e9 100644 --- a/airflow/providers/google/cloud/hooks/gcs.py +++ b/airflow/providers/google/cloud/hooks/gcs.py @@ -29,7 +29,20 @@ from functools import partial from io import BytesIO from os import path from tempfile import NamedTemporaryFile -from typing import Callable, List, Optional, Sequence, Set, Tuple, TypeVar, Union, cast, overload +from typing import ( +IO, +Callable, +Generator, +List, +Optional, +Sequence, +Set, +Tuple, +TypeVar, +Union, +cast, +overload, +) from urllib.parse import urlparse from google.api_core.exceptions import NotFound @@ -385,7 +398,7 @@ class GCSHook(GoogleBaseHook): object_name: Optional[str] = None, object_url: Optional[str] = None, dir: Optional[str] = None, -): +) -> Generator[IO[bytes], None, None]: """ Downloads the file to a temporary directory and returns a file handle @@ -413,7 +426,7 @@ class GCSHook(GoogleBaseHook): bucket_name: str = PROVIDE_BUCKET, object_name: Optional[str] = None, object_url: Optional[str] = None, -): +) -> Generator[IO[bytes], None, None]: """ Creates temporary file, returns a file handle and uploads the files content on close. diff --git a/airflow/providers/google/cloud/utils/credentials_provider.py b/airflow/providers/google/cloud/utils/credentials_provider.py index 0a8143ceae..1cf33ea70b 100644 --- a/airflow/providers/google/cloud/utils/credentials_provider.py +++ b/airflow/providers/google/cloud/utils/credentials_provider.py @@ -74,7 +74,10 @@ def build_gcp_conn( @contextmanager -def provide_gcp_credentials(key_file_path: Optional[str] = None, key_file_dict: Optional[Dict] = None): +def provide_gcp_credentials( +key_file_path: Optional[str] = None, +key_file_dict: Optional[Dict] = None, +) -> Generator[None, None, None]: ""
[airflow] branch v2-3-test updated (e4a31e6e2f -> 972bbaff8a)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git discard e4a31e6e2f Re-add --force-build flag (#24061) discard 9eaa746e0e Shaves-off about 3 minutes from usage of ARM instances on CI (#24052) discard 81c099e0d1 Upgrade to pip 22.1.2 (#24043) discard 36b4aaa212 Increase size of ARM build instance (#24036) discard 0e78dec818 Add foldable groups in CI outputs in commands that need it (#24035) discard 10c8318d3d Adds foldable CI group for command output (#24026) discard 854dad637d Preparing buildx cache is allowed without --push-image flag (#24028) discard 44871b967c Only run separate per-platform build when preparing build cache (#24023) discard 1db9c1713d Ensure @contextmanager decorates generator func (#23103) discard b74b58b108 Add better diagnostics capabilities for pre-commits run via CI image (#23980) new 012b88d96c Add better diagnostics capabilities for pre-commits run via CI image (#23980) new d2b766f8cc Ensure @contextmanager decorates generator func (#23103) new 871915c51a Only run separate per-platform build when preparing build cache (#24023) new 8ddd7ac474 Preparing buildx cache is allowed without --push-image flag (#24028) new 691247e4b2 Adds foldable CI group for command output (#24026) new 726a0efdeb Add foldable groups in CI outputs in commands that need it (#24035) new c45eb48bd5 Increase size of ARM build instance (#24036) new 34dc2ca406 Upgrade to pip 22.1.2 (#24043) new 6b3c1d0b16 Shaves-off about 3 minutes from usage of ARM instances on CI (#24052) new 1555e67d37 Re-add --force-build flag (#24061) new c9bba62098 Fix grid view for mapped tasks (#24059) new 972bbaff8a Update version to 2.3.2 and add release notes This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (e4a31e6e2f) \ N -- N -- N refs/heads/v2-3-test (972bbaff8a) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 12 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: RELEASE_NOTES.rst| 3 ++- airflow/www/utils.py | 8 +--- 2 files changed, 7 insertions(+), 4 deletions(-)
[airflow] 09/12: Shaves-off about 3 minutes from usage of ARM instances on CI (#24052)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 6b3c1d0b16070779a2c5f21a2f67a7a8adbed7a3
Author: Jarek Potiuk
AuthorDate: Tue May 31 17:37:50 2022 +0200
Shaves-off about 3 minutes from usage of ARM instances on CI (#24052)
Preparing airflow packages and provider packages does not
need to be done on ARM and actually the ARM instance is idle
while they are prepared during cache building.
This change moves preparation of the packages to before
the ARM instance is started which saves about 3 minutes of ARM
instance time.
(cherry picked from commit 8804b1a94132eab5148e59c8589d1f28ea89c4ab)
---
.github/workflows/ci.yml | 24
1 file changed, 12 insertions(+), 12 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c5438eebb2..781cdbb08e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1684,16 +1684,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: breeze free-space
- - name: "Start ARM instance"
-run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
-if: matrix.platform == 'linux/arm64'
- - name: "Push CI cache ${{ matrix.python-version }} ${{ matrix.platform
}}"
-run: >
- breeze build-image
- --prepare-buildx-cache
- --platform ${{ matrix.platform }}
-env:
- PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- name: >
Pull CI image for PROD build
${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{
env.IMAGE_TAG_FOR_THE_BUILD }}"
@@ -1704,17 +1694,27 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Cleanup dist and context file"
run: rm -fv ./dist/* ./docker-context-files/*
- - name: "Prepare providers packages"
+ - name: "Prepare providers packages for PROD build"
run: >
breeze prepare-provider-packages
--package-list-file ./scripts/ci/installed_providers.txt
--package-format wheel
env:
VERSION_SUFFIX_FOR_PYPI: "dev0"
- - name: "Prepare airflow package"
+ - name: "Prepare airflow package for PROD build"
run: breeze prepare-airflow-package --package-format wheel
env:
VERSION_SUFFIX_FOR_PYPI: "dev0"
+ - name: "Start ARM instance"
+run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
+if: matrix.platform == 'linux/arm64'
+ - name: "Push CI cache ${{ matrix.python-version }} ${{ matrix.platform
}}"
+run: >
+ breeze build-image
+ --prepare-buildx-cache
+ --platform ${{ matrix.platform }}
+env:
+ PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
- name: "Move dist packages to docker-context files"
run: mv -v ./dist/*.whl ./docker-context-files
- name: "Push PROD cache ${{ matrix.python-version }} ${{
matrix.platform }}"
[airflow] 12/12: Update version to 2.3.2 and add release notes
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git commit 972bbaff8a13b7a28943951880d32538c500a327 Author: Ephraim Anierobi AuthorDate: Fri May 27 13:40:44 2022 +0100 Update version to 2.3.2 and add release notes --- RELEASE_NOTES.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index b4c42027d3..c87eed432e 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -21,7 +21,7 @@ .. towncrier release notes start -Airflow 2.3.2 (2021-06-02) +Airflow 2.3.2 (2021-06-04) -- No significant changes @@ -29,6 +29,7 @@ No significant changes Bug Fixes ^ +- Fix grid view for mapped tasks (#24059) - Icons in grid view for different DAG run types (#23970) - Faster grid view (#23951) - Disallow calling expand with no arguments (#23463)
[airflow] 05/12: Adds foldable CI group for command output (#24026)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 691247e4b2b4d2d25d4f4d61d780da99f8e889c4
Author: Jarek Potiuk
AuthorDate: Tue May 31 04:14:59 2022 +0200
Adds foldable CI group for command output (#24026)
(cherry picked from commit d135ffc8e1c862385ef69047528646173abd0f00)
---
.../airflow_breeze/commands/ci_image_commands.py | 17 +---
.../commands/production_image_commands.py | 5 +++-
.../commands/release_management_commands.py| 17 +---
.../airflow_breeze/utils/docker_command_utils.py | 18 ++---
dev/breeze/src/airflow_breeze/utils/image.py | 7 -
dev/breeze/src/airflow_breeze/utils/run_utils.py | 31 --
6 files changed, 68 insertions(+), 27 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
index b9e0288540..42054b1244 100644
--- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
@@ -234,7 +234,9 @@ def build_image(
"""Build CI image. Include building multiple images for all python
versions (sequentially)."""
def run_build(ci_image_params: BuildCiParams) -> None:
-return_code, info = build_ci_image(verbose=verbose, dry_run=dry_run,
ci_image_params=ci_image_params)
+return_code, info = build_ci_image(
+verbose=verbose, dry_run=dry_run, ci_image_params=ci_image_params,
parallel=False
+)
if return_code != 0:
get_console().print(f"[error]Error when building image! {info}")
sys.exit(return_code)
@@ -426,7 +428,9 @@ def should_we_run_the_build(build_ci_params: BuildCiParams,
verbose: bool) -> bo
sys.exit(1)
-def build_ci_image(verbose: bool, dry_run: bool, ci_image_params:
BuildCiParams) -> Tuple[int, str]:
+def build_ci_image(
+verbose: bool, dry_run: bool, ci_image_params: BuildCiParams, parallel:
bool
+) -> Tuple[int, str]:
"""
Builds CI image:
@@ -443,6 +447,7 @@ def build_ci_image(verbose: bool, dry_run: bool,
ci_image_params: BuildCiParams)
:param verbose: print commands when running
:param dry_run: do not execute "write" commands - just print what would
happen
:param ci_image_params: CI image parameters
+:param parallel: whether the pull is run as part of parallel execution
"""
if (
ci_image_params.is_multi_platform()
@@ -466,7 +471,9 @@ def build_ci_image(verbose: bool, dry_run: bool,
ci_image_params: BuildCiParams)
if ci_image_params.prepare_buildx_cache or ci_image_params.push_image:
login_to_github_docker_registry(image_params=ci_image_params,
dry_run=dry_run, verbose=verbose)
if ci_image_params.prepare_buildx_cache:
-build_command_result = build_cache(image_params=ci_image_params,
dry_run=dry_run, verbose=verbose)
+build_command_result = build_cache(
+image_params=ci_image_params, dry_run=dry_run, verbose=verbose,
parallel=parallel
+)
else:
if ci_image_params.empty_image:
env = os.environ.copy()
@@ -480,6 +487,7 @@ def build_ci_image(verbose: bool, dry_run: bool,
ci_image_params: BuildCiParams)
cwd=AIRFLOW_SOURCES_ROOT,
text=True,
env=env,
+enabled_output_group=not parallel,
)
else:
get_console().print(f"\n[info]Building CI Image for Python
{ci_image_params.python}\n")
@@ -493,6 +501,7 @@ def build_ci_image(verbose: bool, dry_run: bool,
ci_image_params: BuildCiParams)
cwd=AIRFLOW_SOURCES_ROOT,
text=True,
check=False,
+enabled_output_group=not parallel,
)
if build_command_result.returncode == 0:
if ci_image_params.tag_as_latest:
@@ -543,4 +552,4 @@ def rebuild_ci_image_if_needed(
'Forcing build.[/]'
)
ci_image_params.force_build = True
-build_ci_image(verbose, dry_run=dry_run, ci_image_params=ci_image_params)
+build_ci_image(verbose, dry_run=dry_run, ci_image_params=ci_image_params,
parallel=False)
diff --git
a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py
b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py
index 544095f0b2..5418eac696 100644
--- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py
@@ -362,6 +362,7 @@ def pull_prod_image(
wait_for_image=wait_for_image,
tag_as_latest=tag_as_latest,
poll_time=10.0,
+
[airflow] 06/12: Add foldable groups in CI outputs in commands that need it (#24035)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 726a0efdebdb90ecd1409f1b5ab9cef9301db0c4
Author: Jarek Potiuk
AuthorDate: Tue May 31 09:41:18 2022 +0200
Add foldable groups in CI outputs in commands that need it (#24035)
This is follow-up after #24026 which added capability of selectively
deciding for each breeze command, whether the output of the command
should be "foldable" group. All CI output has been reviewed, and
the commands which "need" it were identified.
This also fixes a problem introduced there - that the command itself
was not "foldable" group itself.
(cherry picked from commit 2551acb692f2e5cae91808a907a75db404eb79be)
---
.../commands/configuration_and_maintenance_commands.py | 12 ++--
.../src/airflow_breeze/commands/production_image_commands.py | 1 +
.../airflow_breeze/commands/release_management_commands.py | 2 ++
dev/breeze/src/airflow_breeze/utils/ci_group.py | 2 +-
dev/breeze/src/airflow_breeze/utils/image.py | 1 -
dev/breeze/src/airflow_breeze/utils/run_utils.py | 2 +-
6 files changed, 15 insertions(+), 5 deletions(-)
diff --git
a/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py
b/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py
index 2d7eb88f21..818ba49ee8 100644
---
a/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py
+++
b/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py
@@ -177,7 +177,13 @@ def cleanup(verbose: bool, dry_run: bool,
github_repository: str, all: bool, ans
given_answer = user_confirm("Are you sure with the removal?")
if given_answer == Answer.YES:
system_prune_command_to_execute = ['docker', 'system', 'prune']
-run_command(system_prune_command_to_execute, verbose=verbose,
dry_run=dry_run, check=False)
+run_command(
+system_prune_command_to_execute,
+verbose=verbose,
+dry_run=dry_run,
+check=False,
+enabled_output_group=True,
+)
elif given_answer == Answer.QUIT:
sys.exit(0)
get_console().print(f"Removing build cache dir ${BUILD_CACHE_DIR}")
@@ -455,7 +461,9 @@ def fix_ownership(github_repository: str, verbose: bool,
dry_run: bool):
shell_params.airflow_image_name_with_tag,
"/opt/airflow/scripts/in_container/run_fix_ownership.sh",
]
-run_command(cmd, verbose=verbose, dry_run=dry_run, text=True, env=env,
check=False)
+run_command(
+cmd, verbose=verbose, dry_run=dry_run, text=True, env=env,
check=False, enabled_output_group=True
+)
# Always succeed
sys.exit(0)
diff --git
a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py
b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py
index 5418eac696..8ac7aacab0 100644
--- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py
@@ -533,6 +533,7 @@ def build_production_image(
cwd=AIRFLOW_SOURCES_ROOT,
check=False,
text=True,
+enabled_output_group=True,
)
if build_command_result.returncode == 0:
if prod_image_params.tag_as_latest:
diff --git
a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index ba40e373a8..ce80e6a193 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -267,6 +267,7 @@ def prepare_airflow_packages(
verbose=verbose,
dry_run=dry_run,
debug=debug,
+enabled_output_group=True,
)
sys.exit(result_command.returncode)
@@ -550,6 +551,7 @@ def verify_provider_packages(
verbose=verbose,
dry_run=dry_run,
debug=debug,
+enabled_output_group=True,
)
sys.exit(result_command.returncode)
diff --git a/dev/breeze/src/airflow_breeze/utils/ci_group.py
b/dev/breeze/src/airflow_breeze/utils/ci_group.py
index e0cb7b41ee..e65751a322 100644
--- a/dev/breeze/src/airflow_breeze/utils/ci_group.py
+++ b/dev/breeze/src/airflow_breeze/utils/ci_group.py
@@ -37,6 +37,6 @@ def ci_group(title: str, enabled: bool = True):
get_console().print(f"[info]{title}[/]")
yield
return
-get_console().print(f"::group::: [info]{title}[/]")
+get_console().print(f"::group::: [info]{title}[/]")
yield
get_console().print("::endgroup::&quo
[airflow] annotated tag constraints-2.3.2rc2 updated (30a23b8db5 -> 4c4708f6d5)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to annotated tag constraints-2.3.2rc2 in repository https://gitbox.apache.org/repos/asf/airflow.git *** WARNING: tag constraints-2.3.2rc2 was modified! *** from 30a23b8db5 (commit) to 4c4708f6d5 (tag) tagging 30a23b8db52b6390720b272578e995470de6d115 (commit) replaces constraints-2.3.2rc1 by Ephraim Anierobi on Wed Jun 1 14:12:40 2022 +0100 - Log - Constraints for Apache Airflow 2.3.2rc2 -BEGIN PGP SIGNATURE- iQGzBAABCgAdFiEEAEK9vayTIUi0cM3WNcli2lUp2+wFAmKXZcgACgkQNcli2lUp 2+xIIQwAjwBe0ZUi0SHxdgtQBtLKzpjRU/7oWA3ywWEX9UAVC/CYX2AhiU4z+KmP sCEVbqK0CI1PhNftRT/6n00Qf6simBDMKt5neXBV0Y37Yhk0LKMZ/ToiGSO58sdl 5v3zrQ5C84UYmC9MAvZQ11kOEmQ8r8xPx908US6m30dvtgcpWkZTBn2KvqptFQWd 5ym+hJfHLEJTwf16AY2IeTV4FndbKSyfGLQbtHq/J+ZLi6zwgzJ1LSWolr7HPLrB dDR2hJqQpNkUVtxVU8N1tJxtO04AciyA0Bam95h1ze77zan+7SojRGGsm2+pXgwg 5LSNHlARLeLHDh3RNgymnUq9JitWNwnq0Xw1ioA2c0eVV7onNi5bQolwJRPbcVAB NIySjy2UXAQdlaL0kLITJ9+dAfpUAfMvxRXboiZCPoaAQQ/SvgUiIq4c2IPcT4ee qTLtS+FY3DayjNRFDnmGcQhwkp7fVR+csv3SRk2UlLs4VpskdQow/k4eLAgXZ/xG BtWTb0v+ =rjFG -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
[airflow] branch v2-3-test updated (04e700d0c7 -> 0bf5f495d4)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git discard 04e700d0c7 Update version to 2.3.2 and add release notes add 0e67aeeef7 Run the `check_migration` loop at least once (#24068) add 0bf5f495d4 Update version to 2.3.2 and add release notes This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (04e700d0c7) \ N -- N -- N refs/heads/v2-3-test (0bf5f495d4) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: RELEASE_NOTES.rst | 1 + airflow/utils/db.py| 1 + tests/utils/test_db.py | 1 + 3 files changed, 3 insertions(+)
svn commit: r54815 - /dev/airflow/2.3.2rc2/
Author: ephraimanierobi Date: Wed Jun 1 13:15:09 2022 New Revision: 54815 Log: Add artifacts for Airflow 2.3.2rc2 Added: dev/airflow/2.3.2rc2/ dev/airflow/2.3.2rc2/apache-airflow-2.3.2-source.tar.gz (with props) dev/airflow/2.3.2rc2/apache-airflow-2.3.2-source.tar.gz.asc dev/airflow/2.3.2rc2/apache-airflow-2.3.2-source.tar.gz.sha512 dev/airflow/2.3.2rc2/apache-airflow-2.3.2.tar.gz (with props) dev/airflow/2.3.2rc2/apache-airflow-2.3.2.tar.gz.asc dev/airflow/2.3.2rc2/apache-airflow-2.3.2.tar.gz.sha512 dev/airflow/2.3.2rc2/apache_airflow-2.3.2-py3-none-any.whl (with props) dev/airflow/2.3.2rc2/apache_airflow-2.3.2-py3-none-any.whl.asc dev/airflow/2.3.2rc2/apache_airflow-2.3.2-py3-none-any.whl.sha512 Added: dev/airflow/2.3.2rc2/apache-airflow-2.3.2-source.tar.gz == Binary file - no diff available. Propchange: dev/airflow/2.3.2rc2/apache-airflow-2.3.2-source.tar.gz -- svn:mime-type = application/octet-stream Added: dev/airflow/2.3.2rc2/apache-airflow-2.3.2-source.tar.gz.asc == --- dev/airflow/2.3.2rc2/apache-airflow-2.3.2-source.tar.gz.asc (added) +++ dev/airflow/2.3.2rc2/apache-airflow-2.3.2-source.tar.gz.asc Wed Jun 1 13:15:09 2022 @@ -0,0 +1,14 @@ +-BEGIN PGP SIGNATURE- + +iQHPBAABCgA5FiEEAEK9vayTIUi0cM3WNcli2lUp2+wFAmKXZawbHGVwaHJhaW1h +bmllcm9iaUBhcGFjaGUub3JnAAoJEDXJYtpVKdvs5AkL/3nt1jXwa5xj28POx36u +rJhTzTizgafunUs17k+p+0hV/9M4ITetrytHr9PK6xtpkAnCLWeme54eGGHzM8jn +dRJCa1jmR4B/fUULq6MCuFAH/3VrPPYB2pJnUJZ4oDPdTf7IkcvD0iTMJvz+ZNba ++35yx5Go7R6Dknl7ZKRpvFwvN/v+aUhuhZXcrqkYPq5puIe/GD/+yfqz+yTG34B8 +Duz72MEgEM+T+o02cr/TunbvI7k5pa1uiE7TfmcVIuZP6xPiHYaXQVyLbuxg9NAn +rFiovdUSVKY/PWI+8DkZu8n/mjlcGLkaNQ76Tljtmv+bvL39RX3eQh/yps0U+Zbd +FrTG2+UOxjRcC22kbSi6/zGoNJmnQ4XRp1H65NJ9X0H4jFi+Q23KHzA/Ll0GL3A3 +sOznp4lXR+wplss2xpW9AiyLNAl7X4cEaAx/nc1g1Z6ZLlU/7niG5PIzIz4JnAUb +w07dYbCjMun797JcxMaFBg0egAUhH1hhbDFB9DKFVxSYIQ== +=NjeQ +-END PGP SIGNATURE- Added: dev/airflow/2.3.2rc2/apache-airflow-2.3.2-source.tar.gz.sha512 == --- dev/airflow/2.3.2rc2/apache-airflow-2.3.2-source.tar.gz.sha512 (added) +++ dev/airflow/2.3.2rc2/apache-airflow-2.3.2-source.tar.gz.sha512 Wed Jun 1 13:15:09 2022 @@ -0,0 +1 @@ +b41a93a08ef40414ee2b4fafe820cff4ffa412d01d694f2d4abd548c6a1ef80b782a4960060be92a45ae6e06a016dcb6966352120e0817add70372e3195556e8 apache-airflow-2.3.2-source.tar.gz Added: dev/airflow/2.3.2rc2/apache-airflow-2.3.2.tar.gz == Binary file - no diff available. Propchange: dev/airflow/2.3.2rc2/apache-airflow-2.3.2.tar.gz -- svn:mime-type = application/octet-stream Added: dev/airflow/2.3.2rc2/apache-airflow-2.3.2.tar.gz.asc == --- dev/airflow/2.3.2rc2/apache-airflow-2.3.2.tar.gz.asc (added) +++ dev/airflow/2.3.2rc2/apache-airflow-2.3.2.tar.gz.asc Wed Jun 1 13:15:09 2022 @@ -0,0 +1,14 @@ +-BEGIN PGP SIGNATURE- + +iQHPBAABCgA5FiEEAEK9vayTIUi0cM3WNcli2lUp2+wFAmKXZbMbHGVwaHJhaW1h +bmllcm9iaUBhcGFjaGUub3JnAAoJEDXJYtpVKdvsNxML/2+YyD5f3QhfffpRaG3z +kyKOhryqMMyb+bNyWpBKzExIgjdyQZKoK2KVUkuw9/3EiMAZ74zIuhDfxknkzwtj +KumGwQVchHxeJHwaGYCRcRsiadJFVA6RNOtNWfFHz4R7woC+AgIg8n3FEdfk6d5/ +MMisUZ1ugWgINXwyAEAcCr6W50Ui7qFHL74E5hSCidHIhio3lzvlvsZ0c00htAvs +D8Dts+iJr7hKQmfk7ER4930OM5YYPjrziDae7Ba9Rrh+IhRDC1ROhCJ8HV7vObZP +yeML1lGnBLtH45UU5BzQXb+YQzHbDE4SkjIGY3lOKWybuCeHcqZSbWERnGhaixx0 +SZ8zpCBfUD0MOkL9v585oDA6Z2NfqvZ4TxHJU7ibCRaCTtiUYgG6SPzo8c/NO3ig +khTYNcoDsXRuS66uH8YOeNuzqTnzXInCzMS4yjBMSRFAWLjWr4w1czuoimV2Ycki +8rr090sUDGft8ZgCHz7y6Otyk4ql8aqxedNADefWZA8+fQ== +=EUby +-END PGP SIGNATURE- Added: dev/airflow/2.3.2rc2/apache-airflow-2.3.2.tar.gz.sha512 == --- dev/airflow/2.3.2rc2/apache-airflow-2.3.2.tar.gz.sha512 (added) +++ dev/airflow/2.3.2rc2/apache-airflow-2.3.2.tar.gz.sha512 Wed Jun 1 13:15:09 2022 @@ -0,0 +1 @@ +3f9a7c664c7ab02f19fd6968addc03ab9abbf0cd67af37d80230231c217daee47506bdbfe6f60455035d7aa526b7b299ce2ac1187a6d91f901b6d08320401a42 apache-airflow-2.3.2.tar.gz Added: dev/airflow/2.3.2rc2/apache_airflow-2.3.2-py3-none-any.whl == Binary file - no diff available. Propchange: dev/airflow/2.3.2rc2/apache_airflow-2.3.2-py3-none-any.whl -- svn:mime-type = application/octet-stream Added: dev/airflow/2.3.2rc2/apache_airflow-2.3.2-py3-none-any.whl.asc
[airflow] branch v2-3-stable updated (d1e2dc5bd1 -> 0bf5f495d4)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a change to branch v2-3-stable
in repository https://gitbox.apache.org/repos/asf/airflow.git
from d1e2dc5bd1 Update version to 2.3.2 and add release notes
add 012b88d96c Add better diagnostics capabilities for pre-commits run via
CI image (#23980)
add d2b766f8cc Ensure @contextmanager decorates generator func (#23103)
add 871915c51a Only run separate per-platform build when preparing build
cache (#24023)
add 8ddd7ac474 Preparing buildx cache is allowed without --push-image flag
(#24028)
add 691247e4b2 Adds foldable CI group for command output (#24026)
add 726a0efdeb Add foldable groups in CI outputs in commands that need it
(#24035)
add c45eb48bd5 Increase size of ARM build instance (#24036)
add 34dc2ca406 Upgrade to pip 22.1.2 (#24043)
add 6b3c1d0b16 Shaves-off about 3 minutes from usage of ARM instances on
CI (#24052)
add 1555e67d37 Re-add --force-build flag (#24061)
add c9bba62098 Fix grid view for mapped tasks (#24059)
add 0e67aeeef7 Run the `check_migration` loop at least once (#24068)
add 0bf5f495d4 Update version to 2.3.2 and add release notes
No new revisions were added by this update.
Summary of changes:
.github/workflows/ci.yml | 44 +++---
Dockerfile | 4 +-
Dockerfile.ci | 4 +-
IMAGES.rst | 2 +-
RELEASE_NOTES.rst | 4 +-
STATIC_CODE_CHECKS.rst | 52 ---
airflow/cli/commands/task_command.py | 4 +-
airflow/models/taskinstance.py | 3 +-
airflow/providers/google/cloud/hooks/gcs.py| 19 ++-
.../google/cloud/utils/credentials_provider.py | 9 +-
.../providers/google/common/hooks/base_google.py | 10 +-
airflow/providers/microsoft/psrp/hooks/psrp.py | 4 +-
airflow/utils/db.py| 12 +-
airflow/utils/process_utils.py | 4 +-
airflow/utils/session.py | 4 +-
airflow/www/utils.py | 8 +-
dev/TRACKING_BACKTRACKING_ISSUES.md| 2 +-
.../airflow_breeze/commands/ci_image_commands.py | 112 +++---
.../configuration_and_maintenance_commands.py | 12 +-
.../airflow_breeze/commands/developer_commands.py | 2 +-
.../commands/production_image_commands.py | 75 +
.../commands/release_management_commands.py| 19 ++-
.../src/airflow_breeze/params/build_ci_params.py | 9 +-
.../src/airflow_breeze/params/build_prod_params.py | 4 +-
...mmon_build_params.py => common_build_params.py} | 9 +-
dev/breeze/src/airflow_breeze/utils/ci_group.py| 2 +-
.../src/airflow_breeze/utils/common_options.py | 3 +-
.../airflow_breeze/utils/docker_command_utils.py | 83 +++---
dev/breeze/src/airflow_breeze/utils/image.py | 14 +-
dev/breeze/src/airflow_breeze/utils/path_utils.py | 8 +-
dev/breeze/src/airflow_breeze/utils/registry.py| 4 +-
dev/breeze/src/airflow_breeze/utils/run_utils.py | 122 ++-
dev/provider_packages/prepare_provider_packages.py | 4 +-
dev/refresh_images.sh | 1 +
docs/docker-stack/build-arg-ref.rst| 2 +-
images/breeze/output-build-image.svg | 136 -
images/breeze/output-build-prod-image.svg | 170 ++---
images/breeze/output-commands-hash.txt | 1 -
images/breeze/output-static-checks.svg | 120 +++
.../ci_start_arm_instance_and_connect_to_docker.sh | 4 +-
scripts/ci/libraries/_initialization.sh| 2 +-
.../pre_commit_check_pre_commit_hooks.py | 2 +-
scripts/ci/pre_commit/pre_commit_flake8.py | 28 ++--
.../pre_commit/pre_commit_migration_reference.py | 34 +++--
scripts/ci/pre_commit/pre_commit_mypy.py | 28 ++--
scripts/ci/pre_commit/pre_commit_ui_lint.py| 36 ++---
scripts/ci/pre_commit/pre_commit_www_lint.py | 35 ++---
scripts/docker/common.sh | 2 +-
tests/utils/test_db.py | 1 +
49 files changed, 680 insertions(+), 593 deletions(-)
rename dev/breeze/src/airflow_breeze/params/{_common_build_params.py =>
common_build_params.py} (96%)
delete mode 100644 images/breeze/output-commands-hash.txt
[airflow] branch main updated (841ed27101 -> 84d7b5ba39)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 841ed27101 Spanner assets & system tests migration (AIP-47) (#23957) add 84d7b5ba39 Run the `check_migration` loop at least once (#24068) No new revisions were added by this update. Summary of changes: airflow/utils/db.py| 1 + tests/utils/test_db.py | 1 + 2 files changed, 2 insertions(+)
[airflow] annotated tag 2.3.2rc2 updated (0bf5f495d4 -> f631d0bee0)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to annotated tag 2.3.2rc2 in repository https://gitbox.apache.org/repos/asf/airflow.git *** WARNING: tag 2.3.2rc2 was modified! *** from 0bf5f495d4 (commit) to f631d0bee0 (tag) tagging 0bf5f495d4131109fba449697adee68a62516851 (commit) replaces 2.3.2rc1 by Ephraim Anierobi on Wed Jun 1 13:52:51 2022 +0100 - Log - Apache Airflow 2.3.2rc2 -BEGIN PGP SIGNATURE- iQGzBAABCgAdFiEEAEK9vayTIUi0cM3WNcli2lUp2+wFAmKXYSMACgkQNcli2lUp 2+ybegv9EhrIU5H4ng34gKpT6E0ti3pjuvmxudvPdOuOIa9mXjq3YPPF181uoCqr 5uQEOobEc18ZJG13jePsoMOiLdkw4Wv/vBKhHtjfreEcRE5M7ceetkE+oY25JShL SP5sRpimEgsdN2Dmi9Ea1RtR/KSQzvFF7eo0lwTWx2a6Rs+c1l/eKcYgV3lM2DbL 2j715p9tTn8NMRa6ZHqYvqmELlcxvaIIsOcop73GC4Y0lMKHekibzYxWx5BDJdRG DUP3RZ9qmFCq23HPZcgqZmjKk5ENNVjAe8DaZYZDC4yWEc5U3x+HeoO7I5H2j/Zn 0FDXDNrGRSZANFFagq1Bgofv6SzTj3inyGcxqhCV542kMaYM5Sc2/C91aBH7nWDN /4+JJyYcnONXoXwVOY0JlghzdDEa+UVlxzcE+Y+zoHxkMJqf2IZaIxHqHlVKofcb ltwCdeoTD6FLN48KG2TEXh28ts0BWe6dhDG5PBmC5zKWZyW+E0eUvY+7N+CCVQPO wFn9eAf3 =t3x0 -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
[airflow] annotated tag constraints-2.3.1 updated (d18aee0214 -> 62f4949a2f)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to annotated tag constraints-2.3.1 in repository https://gitbox.apache.org/repos/asf/airflow.git *** WARNING: tag constraints-2.3.1 was modified! *** from d18aee0214 (commit) to 62f4949a2f (tag) tagging d18aee02142e0ab285101c45c60de1c81ef7e7ad (commit) replaces constraints-2.3.0rc2 by Ephraim Anierobi on Wed May 25 08:59:39 2022 +0100 - Log - Constraints for Apache Airflow 2.3.1 -BEGIN PGP SIGNATURE- iQGzBAABCgAdFiEEAEK9vayTIUi0cM3WNcli2lUp2+wFAmKN4esACgkQNcli2lUp 2+zdNQv+JUECKgvKBtNT7FnWXLgseteT4LhUVlNYqGs33LjfBML8OaYRUo/ixJzX 2quMJk862LAvZD9fE2A1wEiGKddBs0rQ4HzkR2KzZNxmjBAm8FitfDjdPHGEa+1Z /sY+cHrqQwabYOAjUsTSqYyJh1s7m2E5HNxzZuuf1uEc2ChaB7wRb4nCW07H3ilU I0HBEZVd5vV7FkiQUyypkmdmcw6eP3DbsXGZqSZue099Am/hg6grQ/EO1Ie159+f HaAe64AYyjtfRMlQBDPxfM4u1sFb4huYJ5FOQ0SQFlC3Ra9WqVYbBxrkNg2mwFSO iZT9ONXq9TzGXe++R5XCXYfKh7SR5N9UR4X1TVDXg086MrbQrqDZ6SDp0w2ktPOt NfYDwR6eirzM77sSvJVPU5yRsl5GRo1LzA/3cYib5c1wGPL12O7ZvqcQDQP8UKKV GDOm8C9CefCM8iq6ZrsHy1Jp1jaB/e43pewFvcQNiyg5pdD5qAApxDwOe7yrHmHR k0IRPAKU =ITiT -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
[airflow] annotated tag constraints-2.3.1 updated (bc44527fc3 -> c56e97523c)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to annotated tag constraints-2.3.1 in repository https://gitbox.apache.org/repos/asf/airflow.git *** WARNING: tag constraints-2.3.1 was modified! *** from bc44527fc3 (commit) to c56e97523c (tag) tagging bc44527fc318bb7f7b8f2f768930f756af394e92 (commit) replaces constraints-latest by Ephraim Anierobi on Wed May 25 08:44:47 2022 +0100 - Log - Constraints for Apache Airflow 2.3.1 -BEGIN PGP SIGNATURE- iQGzBAABCgAdFiEEAEK9vayTIUi0cM3WNcli2lUp2+wFAmKN3m8ACgkQNcli2lUp 2+yYxwwAmqz3tnKlizqtGDoCzJmaozWfEzG+lO33DCutIoSb01NOh/7cXmELGRmp npqjbEKt1XeaBGhARIN/7o8Zuziu6t04rGAnmv09kLitM8r19758WUMg1Q+WxlUS rwZZMNY93iDnbd3L5kHR7cA8209E6TFZXWgRqnW6FQ5Ux6HmSOkXmYCA4DIOGER0 PVfMo38y2GnZDAuW8WDIwVHOb91OrL3PDqqM8u4WDXOIQPQ/V5bydgWV8De1rhfT q1ZhOGnc8ON2D1L8oimnj3RC6rSpIxAKzwgJH62zrfMHYG0bSeXSQr2pV/FQHoVV D5AHVjueLwZdygmO9Mn6e5kBRxHgMU47dg9qwC44MeStbTlT3V2w3xTTtnoXWGKr 5/0Ad14NpYW5512d8dhJdN3COzzPndlDpJcAfRhmT0sfNSfxwC6ZNT2tybzOMoQY wE/1JX3JEz5RWZ0PstL+2MTCS4put37qLTfKPFkZyDervuQp4RVJVh8dlXOofQ8y 95nvY6Tg =h1fZ -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
[airflow] annotated tag constraints-latest updated (7ce5e06aac -> 663209186a)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to annotated tag constraints-latest in repository https://gitbox.apache.org/repos/asf/airflow.git *** WARNING: tag constraints-latest was modified! *** from 7ce5e06aac (tag) to 663209186a (tag) tagging bc44527fc318bb7f7b8f2f768930f756af394e92 (commit) replaces constraints-2.3.0rc2 by Ephraim Anierobi on Wed May 25 08:50:15 2022 +0100 - Log - Latest constraints set to Apache Airflow 2.3.1 -BEGIN PGP SIGNATURE- iQGzBAABCgAdFiEEAEK9vayTIUi0cM3WNcli2lUp2+wFAmKN37cACgkQNcli2lUp 2+yriQv/Vs1I1jbdBaDwUpmDRxNiY4A//Ofa2Rs2pi27krcVKa3h5R7lcqCYPCf1 F1OZXqTajYLY1PXG7lOAVxQUEmz+Zjxn50uBVhNFxEqsRS/ImDf75bl8NqVwSv+e giPZJa4tsYxJgVr6PHq2uDN6KkbuHHK9EEpr84BiHLFFo9ypWVKWpnPBhLeL6JBX NakUozqiK0SEG+ReklzUREkb0b4MNyjxSnlhfgSdaFIcTLabp2orPc/qEtfHi8GL ipJpL83f47+pP3YC3Bg1oP7VoSXFkzUlPDDAFdw7Xv3w/iWVriZwCxvMWwvOYYIS v5ySXg3hStelyl7AZcC3+WmiF2KRJf9tpkEFWDlMgPLqtglnA8HI6yklI6WaOnrL huEDH3VKNv3DKi7y0IePzS7HbWFMCwHDdycW71U/XJF+FUGKw0Jq8dbwqNTpcROo Le5xTHzZawGk5LZ4t8NPcs58ceUphNwH1ILriV/94roz/lUholkVLMchJDvlNT0d QhR2u9Ve =3q3I -END PGP SIGNATURE- --- from ad9a2e1426 Update to latest dependencies add d7c3d86706 Updating constraints. Build id:2251750739 add 54b91a40a5 Updating constraints. Build id:2257028467 add 4ac932460e Updating constraints. Build id:2286626838 add de3e98c608 Updating constraints. Build id:2314717034 add 308dd79315 Updating constraints. Build id:2315062527 add 3c4710ccdb Updating constraints. Build id:2318885926 add d838b728ec Updating constraints. Build id:2331980228 add 16073010d8 Updating constraints. Build id:2334771599 add edf8c465b9 Updating constraints. Build id:2337095685 add ea21439793 Updating constraints. Build id:2353329087 add 78cf602e04 Updating constraints. Build id:2353814637 add f352244f2f Updating constraints. Build id:2354962786 add 886d50c7bf Updating constraints. Build id:2356585604 add d337fa563b Updating constraints. Build id:2358164512 add 427e1d10bc Updating constraints. Build id:2358791896 add d14f1ba5fa Updating constraints. Build id:2359208920 add e285f40237 Updating constraints. Build id:2359723199 add 592612173d Updating constraints. Build id:2360278386 add 11dc7fb83d Updating constraints. Build id:2362653575 add 295c0c103b Updating constraints. Build id:2363598500 add bc44527fc3 Updating constraints. Build id:2363984792 No new revisions were added by this update. Summary of changes: constraints-3.10.txt | 197 ++- constraints-3.7.txt | 235 + constraints-3.8.txt | 241 +- constraints-3.9.txt | 241 +- constraints-no-providers-3.10.txt | 50 +++ constraints-no-providers-3.7.txt | 58 constraints-no-providers-3.8.txt | 64 - constraints-no-providers-3.9.txt | 64 - constraints-source-providers-3.10.txt | 158 +++--- constraints-source-providers-3.7.txt | 198 ++-- constraints-source-providers-3.8.txt | 204 ++-- constraints-source-providers-3.9.txt | 204 ++-- 12 files changed, 955 insertions(+), 959 deletions(-)
[airflow] annotated tag constraints-latest updated (d18aee0214 -> 32c6084f7c)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to annotated tag constraints-latest in repository https://gitbox.apache.org/repos/asf/airflow.git *** WARNING: tag constraints-latest was modified! *** from d18aee0214 (commit) to 32c6084f7c (tag) tagging d18aee02142e0ab285101c45c60de1c81ef7e7ad (commit) replaces constraints-2.3.0rc2 by Ephraim Anierobi on Wed May 25 09:00:55 2022 +0100 - Log - Latest constraints set to Apache Airflow 2.3.1 -BEGIN PGP SIGNATURE- iQGzBAABCgAdFiEEAEK9vayTIUi0cM3WNcli2lUp2+wFAmKN4jcACgkQNcli2lUp 2+wJrgwAouU1qlOtX+UYGqdYL1vPmwLyQmJcbqWHuwYnHLgzTIvZOVuZZ7Wdd6Di TddH2T02inBJ36QqNMJTzO0Wqt8c6DbJVFzM4Xojn6yqlPA/D/mP+WmIwnxSFexm 7gPmFMf3uefVZySf51A0REsjnURRIFqVR1t1aP6lfOq9QrBPejzigIuYa24A9j0d gxmY7vL/V/A4ElouarEMshX8/Upo8mHdJsqczHhem+xELx2DaX8/GL/rkMbEn57u P4506H0UpHCzpE3Nz0/Wm+T3HU6EE3y4YuWLpIc6NfLn47dqc29/4YL519+w1hjZ hO54fw6VCDV0N6W/1xsZMxAglL/363Qm5M5u9gYBxgTy7AYRRONvIceRxn2pWPZ/ 2i57No/Uqm07ovlsV9LDR0YhQhywCerjebW7JBf17CO5fNU4fc/yEupq8QoyAXEO nzT0ZtfAHoCassCfaRZv0+mSx3r6V0BmWRFu0ngshPcXqb73LIKHEixlIRGuMTWY NSZ1EFMt =j2gW -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
svn commit: r54720 - /release/airflow/2.3.1/
Author: ephraimanierobi Date: Wed May 25 07:07:14 2022 New Revision: 54720 Log: Release Airflow 2.3.1 from 2.3.1rc1 Added: release/airflow/2.3.1/ release/airflow/2.3.1/apache-airflow-2.3.1-source.tar.gz - copied unchanged from r54719, dev/airflow/2.3.1rc1/apache-airflow-2.3.1-source.tar.gz release/airflow/2.3.1/apache-airflow-2.3.1-source.tar.gz.asc - copied unchanged from r54719, dev/airflow/2.3.1rc1/apache-airflow-2.3.1-source.tar.gz.asc release/airflow/2.3.1/apache-airflow-2.3.1-source.tar.gz.sha512 - copied unchanged from r54719, dev/airflow/2.3.1rc1/apache-airflow-2.3.1-source.tar.gz.sha512 release/airflow/2.3.1/apache-airflow-2.3.1.tar.gz - copied unchanged from r54719, dev/airflow/2.3.1rc1/apache-airflow-2.3.1.tar.gz release/airflow/2.3.1/apache-airflow-2.3.1.tar.gz.asc - copied unchanged from r54719, dev/airflow/2.3.1rc1/apache-airflow-2.3.1.tar.gz.asc release/airflow/2.3.1/apache-airflow-2.3.1.tar.gz.sha512 - copied unchanged from r54719, dev/airflow/2.3.1rc1/apache-airflow-2.3.1.tar.gz.sha512 release/airflow/2.3.1/apache_airflow-2.3.1-py3-none-any.whl - copied unchanged from r54719, dev/airflow/2.3.1rc1/apache_airflow-2.3.1-py3-none-any.whl release/airflow/2.3.1/apache_airflow-2.3.1-py3-none-any.whl.asc - copied unchanged from r54719, dev/airflow/2.3.1rc1/apache_airflow-2.3.1-py3-none-any.whl.asc release/airflow/2.3.1/apache_airflow-2.3.1-py3-none-any.whl.sha512 - copied unchanged from r54719, dev/airflow/2.3.1rc1/apache_airflow-2.3.1-py3-none-any.whl.sha512
[airflow] annotated tag 2.3.1 updated (ee100a592e -> bf8397f574)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to annotated tag 2.3.1 in repository https://gitbox.apache.org/repos/asf/airflow.git *** WARNING: tag 2.3.1 was modified! *** from ee100a592e (commit) to bf8397f574 (tag) tagging ee100a592e07761f3b32294f3ad82c4a6c7cf74d (commit) replaces 2.3.0 by Ephraim Anierobi on Wed May 25 08:46:22 2022 +0100 - Log - Apache Airflow 2.3.1 -BEGIN PGP SIGNATURE- iQGzBAABCgAdFiEEAEK9vayTIUi0cM3WNcli2lUp2+wFAmKN3s4ACgkQNcli2lUp 2+zTYAv/YEfDpxLBnwr1GuizzdEJWqiRu+Z29MavoEQ3i7Iyj94sO6watjrGcns7 Jgxm7UnSbcm+wgyH3OmlHYpVcNNf76mLqaW1SPaXE/EISQ1BAwoneix/nS1bPGn+ xqGw9zHpvHs/6/YyrXp4TSYb4SrG/bK91ZDHq1XNntxBJk8J3YEPapPy0ime9eNh A9u472VA0aE3JSKnyL2szydRs69wzmz1DPK21tNUKKHIp5q3EGTGtYQKOAcuRrkh 8cIo3/b+i6KwF0t4FRmSR0X5yFCpWl4hvgTLA8K82URLl3jwesDkuY/wweiZSsN+ qFoJwyJG62ZnOsb8SPj5qJQ+eewPWztZr6uwktGPBZsPqa2g0bQmjJGgChG6gz13 qeN4wWEGKLJ8c2HQCxKd/1gMwMjP8Lf0dPmSsOsVQbCOHg0Cr+c1zJIXAaDrZ8B0 oO51nYSAicwDrfCN5sh4sWQWAL5Xr4l9fw81AiOvA+1TH0l/sdLBVuYkPxYVXbrk 736m19Hb =VlXW -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
svn commit: r54721 - /release/airflow/2.3.0/
Author: ephraimanierobi Date: Wed May 25 07:08:42 2022 New Revision: 54721 Log: Remove old release: 2.3.0 Removed: release/airflow/2.3.0/
[airflow-site] branch 2.3.1-docs created (now b4ab9a30f)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch 2.3.1-docs in repository https://gitbox.apache.org/repos/asf/airflow-site.git at b4ab9a30f Add documentation for Apache Airflow 2.3.1 This branch includes the following new commits: new b4ab9a30f Add documentation for Apache Airflow 2.3.1 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[airflow-site] branch announce-airflow-2.3.1 created (now 99ff5bd1f)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch announce-airflow-2.3.1 in repository https://gitbox.apache.org/repos/asf/airflow-site.git at 99ff5bd1f Announce Apache Airflow 2.3.1 This branch includes the following new commits: new 99ff5bd1f Announce Apache Airflow 2.3.1 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[airflow-site] 01/01: Announce Apache Airflow 2.3.1
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch announce-airflow-2.3.1 in repository https://gitbox.apache.org/repos/asf/airflow-site.git commit 99ff5bd1f142ce63ac339ae4438e98cfe7f5586a Author: Ephraim Anierobi AuthorDate: Wed May 25 11:07:18 2022 +0100 Announce Apache Airflow 2.3.1 --- landing-pages/site/content/en/announcements/_index.md | 10 ++ 1 file changed, 10 insertions(+) diff --git a/landing-pages/site/content/en/announcements/_index.md b/landing-pages/site/content/en/announcements/_index.md index c63ce656b..e56d4433d 100644 --- a/landing-pages/site/content/en/announcements/_index.md +++ b/landing-pages/site/content/en/announcements/_index.md @@ -13,6 +13,16 @@ menu: **Note:** Follow [@ApacheAirflow](https://twitter.com/ApacheAirflow) on Twitter for the latest news and announcements! +# May 25, 2022 + +We’ve just released Apache **Airflow 2.3.1** + + PyPI: https://pypi.org/project/apache-airflow/2.3.1/ + Docs: https://airflow.apache.org/docs/apache-airflow/2.3.1/ + Release Notes: https://airflow.apache.org/docs/apache-airflow/2.3.1/release_notes.html + Docker Image: “docker pull apache/airflow:2.3.1" + Constraints: https://github.com/apache/airflow/tree/constraints-2.3.1 + # May 20, 2022 We've just released Apache **Airflow Helm chart 1.6.0**.
[airflow] branch main updated: Add TaskInstance State 'REMOVED' to finished states and success states (#23797)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 73446f28e9 Add TaskInstance State 'REMOVED' to finished states and
success states (#23797)
73446f28e9 is described below
commit 73446f28e9eb1e4c6f2f32c700147b61ab3da600
Author: Ephraim Anierobi
AuthorDate: Sat May 28 09:06:07 2022 +0100
Add TaskInstance State 'REMOVED' to finished states and success states
(#23797)
Now that we support dynamic task mapping, we should have the 'REMOVED'
state of task instances as a finished state because
for dynamic tasks with a removed task instance, the dagrun would be stuck in
running state if 'REMOVED' state is not in finished states.
---
airflow/models/dagrun.py| 2 +-
airflow/utils/state.py | 1 +
tests/models/test_dagrun.py | 23 +++
3 files changed, 25 insertions(+), 1 deletion(-)
diff --git a/airflow/models/dagrun.py b/airflow/models/dagrun.py
index 58974c39be..eeec4d5b09 100644
--- a/airflow/models/dagrun.py
+++ b/airflow/models/dagrun.py
@@ -540,7 +540,7 @@ class DagRun(Base, LoggingMixin):
)
leaf_task_ids = {t.task_id for t in dag.leaves}
-leaf_tis = [ti for ti in tis if ti.task_id in leaf_task_ids]
+leaf_tis = [ti for ti in tis if ti.task_id in leaf_task_ids if
ti.state != TaskInstanceState.REMOVED]
# if all roots finished and at least one failed, the run failed
if not unfinished_tis and any(leaf_ti.state in State.failed_states for
leaf_ti in leaf_tis):
diff --git a/airflow/utils/state.py b/airflow/utils/state.py
index 8415dd1666..a79169f861 100644
--- a/airflow/utils/state.py
+++ b/airflow/utils/state.py
@@ -154,6 +154,7 @@ class State:
TaskInstanceState.FAILED,
TaskInstanceState.SKIPPED,
TaskInstanceState.UPSTREAM_FAILED,
+TaskInstanceState.REMOVED,
]
)
"""
diff --git a/tests/models/test_dagrun.py b/tests/models/test_dagrun.py
index f73f5d1c45..14f4b7f34b 100644
--- a/tests/models/test_dagrun.py
+++ b/tests/models/test_dagrun.py
@@ -196,6 +196,29 @@ class TestDagRun:
dag_run.update_state()
assert DagRunState.SUCCESS == dag_run.state
+def
test_dagrun_not_stuck_in_running_when_all_tasks_instances_are_removed(self,
session):
+"""
+Tests that a DAG run succeeds when all tasks are removed
+"""
+dag = DAG(dag_id='test_dagrun_success_when_all_skipped',
start_date=timezone.datetime(2017, 1, 1))
+dag_task1 = ShortCircuitOperator(
+task_id='test_short_circuit_false', dag=dag,
python_callable=lambda: False
+)
+dag_task2 = EmptyOperator(task_id='test_state_skipped1', dag=dag)
+dag_task3 = EmptyOperator(task_id='test_state_skipped2', dag=dag)
+dag_task1.set_downstream(dag_task2)
+dag_task2.set_downstream(dag_task3)
+
+initial_task_states = {
+'test_short_circuit_false': TaskInstanceState.REMOVED,
+'test_state_skipped1': TaskInstanceState.REMOVED,
+'test_state_skipped2': TaskInstanceState.REMOVED,
+}
+
+dag_run = self.create_dag_run(dag=dag,
task_states=initial_task_states, session=session)
+dag_run.update_state()
+assert DagRunState.SUCCESS == dag_run.state
+
def test_dagrun_success_conditions(self, session):
dag = DAG('test_dagrun_success_conditions', start_date=DEFAULT_DATE,
default_args={'owner': 'owner1'})
[airflow] branch v2-3-test updated (c9eeddf993 -> f1e19f2ba1)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a change to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
discard c9eeddf993 Update version to 2.3.2 and add release notes
add e4caa10371 Make provider doc preparation a bit more fun :) (#23629)
add 25931dbcff Change chart annotation generator to use RELEASE_NOTES
(#23549)
add b11bb985f8 Add tool to automaticaly update status of AIP-47 issues.
(#23745)
add 4a96bd6219 Add Deferrable Databricks operators (#19736)
add 37a5696de4 Fix Breeze documentation typo (#23919)
add 48efec10b2 Add exception to catch single line private keys (#23043)
add 54cad993e0 Introduce `flake8-implicit-str-concat` plugin to static
checks (#23873)
add f1e19f2ba1 Update version to 2.3.2 and add release notes
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (c9eeddf993)
\
N -- N -- N refs/heads/v2-3-test (f1e19f2ba1)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
airflow/configuration.py | 2 +-
airflow/providers/amazon/aws/hooks/eks.py | 2 +-
airflow/providers/amazon/aws/hooks/ses.py | 2 +-
airflow/providers/amazon/aws/hooks/sns.py | 2 +-
airflow/providers/amazon/aws/hooks/sqs.py | 2 +-
airflow/providers/databricks/hooks/databricks.py | 29 ++
.../providers/databricks/hooks/databricks_base.py | 237 +-
.../providers/databricks/operators/databricks.py | 106 +--
.../databricks/operators/databricks_repos.py | 2 +-
.../databricks/triggers}/__init__.py | 0
.../providers/databricks/triggers/databricks.py| 77 +
.../databricks/utils}/__init__.py | 0
airflow/providers/databricks/utils/databricks.py | 69
.../google/cloud/hooks/kubernetes_engine.py| 6 +-
.../google/cloud/utils/credentials_provider.py | 2 +-
airflow/providers/microsoft/psrp/operators/psrp.py | 2 +-
airflow/providers/ssh/hooks/ssh.py | 3 +
airflow/utils/email.py | 2 +-
airflow/utils/file.py | 2 +-
airflow/www/fab_security/manager.py| 8 +-
airflow/www/views.py | 2 +-
dev/breeze/README.md | 4 +-
.../airflow_breeze/utils/docker_command_utils.py | 2 +-
dev/breeze/src/airflow_breeze/utils/image.py | 4 +-
dev/breeze/src/airflow_breeze/utils/run_utils.py | 2 +-
dev/chart/build_changelog_annotations.py | 13 +-
dev/provider_packages/prepare_provider_packages.py | 129 +---
{clients => dev/system_tests}/README.md| 27 +-
dev/system_tests/update_issue_status.py| 185 +++
.../operators/run_now.rst | 7 +
.../operators/submit_run.rst | 7 +
.../run_prepare_provider_documentation.sh | 15 +-
setup.py | 3 +
tests/always/test_connection.py| 2 +-
.../endpoints/test_task_instance_endpoint.py | 4 +-
tests/core/test_providers_manager.py | 2 +-
tests/providers/amazon/aws/hooks/test_glacier.py | 12 +-
.../providers/databricks/hooks/test_databricks.py | 352 -
.../databricks/operators/test_databricks.py| 243 +++---
.../providers/databricks/triggers}/__init__.py | 0
.../databricks/triggers/test_databricks.py | 153 +
.../providers/databricks/utils}/__init__.py| 0
tests/providers/databricks/utils/databricks.py | 62
.../google/cloud/hooks/test_datacatalog.py | 4 +-
.../google/cloud/operators/test_datacatalog.py | 4 +-
.../google/cloud/operators/test_mlengine.py| 4 +-
.../google/cloud/transfers/test_mysql_to_gcs.py| 6 +-
.../cloud/utils/test_credentials_provider.py | 8 +-
.../google/common/hooks/test_base_google.py| 2 +-
tests/providers/ssh/hooks/test_ssh.py | 18 ++
.../google/bigquery/example_bigquery_sensors.py| 2 +-
tests/utils/test_file.py | 7 +-
52 files changed, 1622 insertions(+), 218 deletions(-)
copy airflow/{api/auth => providers/databricks/
[airflow] branch v2-3-test updated (657095299d -> fdb9cc5bbe)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git discard 657095299d Apply suggestions from code review discard bf1d12975c Update version to 2.3.2 and add release notes add a8dd8f82e8 Add typing to Azure Cosmos Client Hook (#23941) add fdb9cc5bbe Update version to 2.3.2 and add release notes This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (657095299d) \ N -- N -- N refs/heads/v2-3-test (fdb9cc5bbe) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: airflow/providers/microsoft/azure/hooks/cosmos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[airflow] branch v2-3-test updated (ee6da982a9 -> 5ce67c3e99)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git omit ee6da982a9 Remove fixing cncf.kubernetes provider when generating constraints (#23994) omit ea8a570a5e Fix missing shorthand for docker buildx rm -f (#23984) omit f1e19f2ba1 Update version to 2.3.2 and add release notes add 4c8c6b7e8b Fix missing shorthand for docker buildx rm -f (#23984) add 7fe7f6a6d0 Remove fixing cncf.kubernetes provider when generating constraints (#23994) add 5ce67c3e99 Update version to 2.3.2 and add release notes This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (ee6da982a9) \ N -- N -- N refs/heads/v2-3-test (5ce67c3e99) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: RELEASE_NOTES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[airflow] branch v2-3-test updated (5a9699a960 -> f6b22a341e)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git discard 5a9699a960 Add automated retries on retryable condition for building images in CI (#24006) discard 0b627dbcc9 Make --file command in static-checks autocomplete file name (#23896) discard 9a80336bf1 Revert "Add limit for JPype1 (#23847)" (#23953) discard f19511eab1 Remove pinning for xmltodict (#23992) discard cee43615c5 Replace generation of docker volumes to be done from python (#23985) discard bc0cc8d72f Fix breeze failures when there is no buildx installed on Mac (#23988) discard 7b88036890 Force colors in yarn test output in CI (#23986) discard 76597c1517 use explicit --mount with types of mounts rather than --volume flags (#23982) discard 65a26dea85 Disable fail-fast on pushing images to docker cache (#24005) discard 5ce67c3e99 Update version to 2.3.2 and add release notes add 312b7f3747 Disable fail-fast on pushing images to docker cache (#24005) add 8d522e1fe2 use explicit --mount with types of mounts rather than --volume flags (#23982) add a4d408c4dd Force colors in yarn test output in CI (#23986) add f72217b510 Fix breeze failures when there is no buildx installed on Mac (#23988) add 5a153e016d Replace generation of docker volumes to be done from python (#23985) add f2dc81e56b Remove pinning for xmltodict (#23992) add 870d0916a7 Revert "Add limit for JPype1 (#23847)" (#23953) add 188afd2ba3 Make --file command in static-checks autocomplete file name (#23896) add 6963acb975 Add automated retries on retryable condition for building images in CI (#24006) add f6b22a341e Update version to 2.3.2 and add release notes This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (5a9699a960) \ N -- N -- N refs/heads/v2-3-test (f6b22a341e) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: RELEASE_NOTES.rst | 8 +++- 1 file changed, 7 insertions(+), 1 deletion(-)
[airflow] branch v2-3-test updated (f6b22a341e -> d1e2dc5bd1)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git omit f6b22a341e Update version to 2.3.2 and add release notes omit 6963acb975 Add automated retries on retryable condition for building images in CI (#24006) add d1e2dc5bd1 Update version to 2.3.2 and add release notes This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (f6b22a341e) \ N -- N -- N refs/heads/v2-3-test (d1e2dc5bd1) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: .github/workflows/ci.yml | 2 - .../airflow_breeze/commands/ci_image_commands.py | 40 + .../commands/production_image_commands.py | 53 ++- .../airflow_breeze/params/_common_build_params.py | 1 - .../src/airflow_breeze/utils/common_options.py | 6 - dev/breeze/src/airflow_breeze/utils/run_utils.py | 38 - images/breeze/output-build-image.svg | 141 - images/breeze/output-build-prod-image.svg | 175 ++--- images/breeze/output-commands-hash.txt | 2 +- 9 files changed, 175 insertions(+), 283 deletions(-)
[airflow] 08/10: Shorten max pre-commit hook name length (#23677)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 7d4090d5fca3d9061f68468d19de4562aef64dd7
Author: Daniel Standish <15932138+dstand...@users.noreply.github.com>
AuthorDate: Thu May 12 11:46:56 2022 -0700
Shorten max pre-commit hook name length (#23677)
When names are too long, pre-commit output looks very ugly and takes up 2x
lines. Here I reduce max length just a little bit further so that pre-commit
output renders properly on a macbook pro 16" with terminal window splitting
screen horizontally.
(cherry picked from commit c74e67a4ffb94ef5152b34fa96e88f1cbfe0c75a)
---
.pre-commit-config.yaml| 6 +-
STATIC_CODE_CHECKS.rst | 374 ++---
.../pre_commit_check_pre_commit_hooks.py | 6 +-
3 files changed, 194 insertions(+), 192 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 3fbf914d67..61043bc116 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -512,7 +512,7 @@ repos:
pass_filenames: true
- id: check-start-date-not-used-in-defaults
language: pygrep
-name: "'start_date' should not be defined in default_args in
example_dags"
+name: "'start_date' not to be defined in default_args in example_dags"
entry:
"default_args\\s*=\\s*{\\s*(\"|')start_date(\"|')|(\"|')start_date(\"|'):"
files: \.*example_dags.*\.py$
exclude: ^airflow/_vendor/
@@ -603,10 +603,10 @@ repos:
- 'jsonpath-ng==1.5.3'
- 'rich>=12.4.1'
- id: check-pre-commit-information-consistent
-name: Update information about pre-commit hooks and verify ids and
names
+name: Update information re pre-commit hooks and verify ids and names
entry: ./scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py
args:
- - --max-length=70
+ - --max-length=64
language: python
files:
^\.pre-commit-config\.yaml$|^scripts/ci/pre_commit/pre_commit_check_pre_commit_hook_names\.py$
additional_dependencies: ['pyyaml', 'jinja2', 'black==22.3.0',
'tabulate', 'rich>=12.4.1']
diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst
index 38ce6e4e88..791b86b71f 100644
--- a/STATIC_CODE_CHECKS.rst
+++ b/STATIC_CODE_CHECKS.rst
@@ -125,193 +125,193 @@ require Breeze Docker images to be installed locally.
.. BEGIN AUTO-GENERATED STATIC CHECK LIST
-+++-+
-| ID | Description
| Image |
-+++=+
-| black | Run Black (the
uncompromising Python code formatter) | |
-+++-+
-| blacken-docs | Run black on python
code blocks in documentation files | |
-+++-+
-| check-airflow-2-1-compatibility| Check that
providers are 2.1 compatible. | |
-+++-+
-| check-airflow-config-yaml-consistent | Checks for
consistency between config.yml and default_config.cfg | |
-+++-+
-| check-airflow-providers-have-extras| Checks providers
available when declared by extras in setup.py | |
-+++-+
-| check-apache-license-rat | Check if licenses
are OK for Apache| |
-+++-+
-| check-base-operator-usage | * Check
BaseOperator[Link] core imports| |
-|
[airflow] 06/10: Add version to migration prefix (#23564)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git commit bae9a1b2dd873a1a60926f12255bec62e9a3c64b Author: Daniel Standish <15932138+dstand...@users.noreply.github.com> AuthorDate: Mon May 16 14:13:53 2022 -0700 Add version to migration prefix (#23564) We don't really need the alembic revision id in the filename. having version instead is much more useful. having both of them takes up too much space. (cherry picked from commit 741f8029115cc4698f9451b62546c3b1ff3d3a9c) --- CONTRIBUTING.rst | 14 ++- ...rent_schema.py => 0001_1_5_0_current_schema.py} | 0 ...rypted.py => 0002_1_5_0_create_is_encrypted.py} | 0 ...tibility.py => 0003_1_5_0_for_compatibility.py} | 0 ... 0004_1_5_0_more_logging_into_task_isntance.py} | 0 ..._id_indices.py => 0005_1_5_2_job_id_indices.py} | 0 ...to_log.py => 0006_1_6_0_adding_extra_to_log.py} | 0 ...b76e_add_dagrun.py => 0007_1_6_0_add_dagrun.py} | 0 ...ask_duration.py => 0008_1_6_0_task_duration.py} | 0 ...agrun_config.py => 0009_1_6_0_dagrun_config.py} | 0 ...y => 0010_1_6_2_add_password_column_to_user.py} | 0 ...start_end.py => 0011_1_6_2_dagrun_start_end.py} | 0 ..._0_add_notification_sent_column_to_sla_miss.py} | 0 ...1_7_0_add_a_column_to_track_the_encryption_.py} | 0 ..._1_7_0_add_is_encrypted_column_to_variable_.py} | 0 ...er_table.py => 0015_1_7_1_rename_user_table.py} | 0 ...e_index.py => 0016_1_7_1_add_ti_state_index.py} | 0 ... => 0017_1_7_1_add_task_fails_journal_table.py} | 0 ..._table.py => 0018_1_7_1_add_dag_stats_table.py} | 0 ..._7_1_add_fractional_seconds_to_mysql_tables.py} | 0 ...ices.py => 0020_1_7_1_xcom_dag_task_indices.py} | 0 ...=> 0021_1_7_1_add_pid_field_to_taskinstance.py} | 0 ...7_1_add_dag_id_state_index_on_dag_run_table.py} | 0 ...1_8_2_add_max_tries_column_to_task_instance.py} | 0 ...1_8_2_make_xcom_value_column_a_large_binary.py} | 0 ..._index.py => 0025_1_8_2_add_ti_job_id_index.py} | 0 ... => 0026_1_8_2_increase_text_size_for_mysql.py} | 0 ...s.py => 0027_1_10_0_add_time_zone_awareness.py} | 0 ..._10_0_add_kubernetes_resource_checkpointing.py} | 0 ...1_10_0_add_executor_config_to_task_instance.py} | 0 ..._1_10_0_add_kubernetes_scheduler_uniqueness.py} | 0 ...6_merge_heads.py => 0031_1_10_0_merge_heads.py} | 0 ...> 0032_1_10_0_fix_mysql_not_null_constraint.py} | 0 ...ey.py => 0033_1_10_0_fix_sqlite_foreign_key.py} | 0 ...x_taskfail.py => 0034_1_10_0_index_taskfail.py} | 0 ...x_log_dag.py => 0035_1_10_2_add_idx_log_dag.py} | 0 ...py => 0036_1_10_2_add_index_to_taskinstance.py} | 0 ...py => 0037_1_10_2_add_task_reschedule_table.py} | 0 ...ag_index.py => 0038_1_10_2_add_sm_dag_index.py} | 0 ...field.py => 0039_1_10_2_add_superuser_field.py} | 0 ..._to_dag.py => 0040_1_10_3_add_fields_to_dag.py} | 0 ...=> 0041_1_10_3_add_schedule_interval_to_dag.py} | 0 ...1_10_3_task_reschedule_fk_on_cascade_delete.py} | 0 ..._1_10_4_make_taskinstance_pool_not_nullable.py} | 0 py => 0044_1_10_7_add_serialized_dag_table.py} | 0 ...ag.py => 0045_1_10_7_add_root_dag_id_to_dag.py} | 0 ..._5_change_datetime_to_datetime2_6_on_mssql_.py} | 0 ... 0047_1_10_4_increase_queue_name_size_limit.py} | 0 ...ble.py => 0048_1_10_3_remove_dag_stat_table.py} | 0 ...4_merge_heads.py => 0049_1_10_7_merge_heads.py} | 0 ...0_7_increase_length_for_connection_password.py} | 0 ...s_table.py => 0051_1_10_8_add_dagtags_table.py} | 0 ...10_10_add_pool_slots_field_to_task_instance.py} | 0 ..._10_add_rendered_task_instance_fields_table.py} | 0 ...table.py => 0054_1_10_10_add_dag_code_table.py} | 0 ...11_add_precision_to_execution_date_in_mysql.py} | 0 ...0_12_add_dag_hash_column_to_serialized_dag_.py} | 0 ...ab_tables.py => 0057_1_10_13_add_fab_tables.py} | 0 ..._10_13_increase_length_of_fab_ab_view_menu_.py} | 0 ..._chart.py => 0059_2_0_0_drop_user_and_chart.py} | 0 ...py => 0060_2_0_0_remove_id_column_from_xcom.py} | 0 ... => 0061_2_0_0_increase_length_of_pool_name.py} | 0 ...n_type.py => 0062_2_0_0_add_dagrun_run_type.py} | 0 ...=> 0063_2_0_0_set_conn_type_as_non_nullable.py} | 0 ...0064_2_0_0_add_unique_constraint_to_conn_id.py} | 0 ...> 0065_2_0_0_update_schema_for_smart_sensor.py} | 0 ...py => 0066_2_0_0_add_queued_by_job_id_to_ti.py} | 0 ...> 0067_2_0_0_add_external_executor_id_to_ti.py} | 0 ...=> 0068_2_0_0_drop_kuberesourceversion_and_.py} | 0 ..._0_0_add_scheduling_decision_to_dagrun_and_.py} | 0 ..._fix_mssql_exec_date_rendered_task_instance.py} | 0 ...py => 0071_2_0_0_add_job_id_to_dagrun_table.py} | 0 ...72_2_0_0_add_k8s_yaml_to_rendered_templates.py} | 0 ...ons.py => 0073_2
[airflow] 05/10: Fix typos in README.md and airflow_doc_issue_report.yml (#23294)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git commit b77e616aa4d4cd8ca1239dac6954fb9b473b86ff Author: TreyYi AuthorDate: Thu Apr 28 06:30:12 2022 +0900 Fix typos in README.md and airflow_doc_issue_report.yml (#23294) (cherry picked from commit c26796e31a9543cd8b45b50264128ac17455002c) --- .github/ISSUE_TEMPLATE/airflow_doc_issue_report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/airflow_doc_issue_report.yml b/.github/ISSUE_TEMPLATE/airflow_doc_issue_report.yml index c32fd9c917..0977e98221 100644 --- a/.github/ISSUE_TEMPLATE/airflow_doc_issue_report.yml +++ b/.github/ISSUE_TEMPLATE/airflow_doc_issue_report.yml @@ -14,7 +14,7 @@ body: Note, you do not need to create an issue if you have a change ready to submit! -You can open a [pull eequest](https://github.com/apache/airflow/pulls) immediately instead. +You can open a [pull request](https://github.com/apache/airflow/pulls) immediately instead. " # yamllint enable rule:line-length - type: textarea
[airflow] 10/10: Fix "breeze-legacy" after building images was removed (#23404)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit d8d87162ea664c660372885d390638e17e1b9818
Author: Jarek Potiuk
AuthorDate: Mon May 2 16:26:05 2022 +0200
Fix "breeze-legacy" after building images was removed (#23404)
The `breeze-legacy` stopped working after building images were
removed as few parameters were still checked for allowed values
but they were missing,
This PR fixes it by removing the parameters.
(cherry picked from commit 8622808aa79531bcaa5099d26fbaf54b4afe931a)
---
breeze-legacy | 2 --
1 file changed, 2 deletions(-)
diff --git a/breeze-legacy b/breeze-legacy
index d5ca8b8b4f..805a3c9810 100755
--- a/breeze-legacy
+++ b/breeze-legacy
@@ -,7 +,6 @@ function breeze::check_and_save_all_params() {
fi
parameters::check_and_save_allowed_param "BACKEND" "backend" "--backend"
-parameters::check_and_save_allowed_param "PLATFORM" "platform" "--platform"
parameters::check_and_save_allowed_param "KUBERNETES_MODE" "Kubernetes
mode" "--kubernetes-mode"
parameters::check_and_save_allowed_param "KUBERNETES_VERSION" "Kubernetes
version" "--kubernetes-version"
parameters::check_and_save_allowed_param "KIND_VERSION" "KinD version"
"--kind-version"
@@ -2233,7 +2232,6 @@ function breeze::check_and_save_all_params() {
parameters::check_and_save_allowed_param "MSSQL_VERSION" "MSSql version"
"--mssql-version"
parameters::check_allowed_param TEST_TYPE "Type of tests" "--test-type"
-parameters::check_allowed_param PACKAGE_FORMAT "Format of packages to
build" "--package-format"
}
###
[airflow] 04/10: Fix `check_files.py` to work on new minor releases (#23287)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit f9287821cb4969b0f164a0e371ae8bf2ae49ae0a
Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com>
AuthorDate: Tue May 3 12:33:05 2022 -0600
Fix `check_files.py` to work on new minor releases (#23287)
(cherry picked from commit f61119c0f3caa5963f00fa158f9d27f8f8e2cda8)
---
dev/check_files.py | 14 +++---
1 file changed, 3 insertions(+), 11 deletions(-)
diff --git a/dev/check_files.py b/dev/check_files.py
index b0855da8fd..f6267aa162 100644
--- a/dev/check_files.py
+++ b/dev/check_files.py
@@ -31,7 +31,7 @@ FROM apache/airflow:latest
"""
AIRFLOW_DOCKER = """\
-FROM apache/airflow:{}
+FROM python:3.7
# Upgrade
RUN pip install "apache-airflow=={}"
@@ -46,11 +46,6 @@ RUN pip install "apache-airflow-upgrade-check=={}"
"""
-DOCKER_CMD = """
-docker build --tag local/airflow .
-docker local/airflow info
-"""
-
AIRFLOW = "AIRFLOW"
PROVIDERS = "PROVIDERS"
UPGRADE_CHECK = "UPGRADE_CHECK"
@@ -78,7 +73,7 @@ def create_docker(txt: str):
print(
"""\
docker build -f Dockerfile.pmc --tag local/airflow .
-docker run local/airflow info
+docker run --rm local/airflow airflow info
"""
)
@@ -209,10 +204,7 @@ def main(check_type: str, path: str, version: str):
if check_type.upper() == AIRFLOW:
files = os.listdir(os.path.join(path, version))
missing_files = check_release(files, version)
-
-base_version = version.split("rc")[0]
-prev_version = base_version[:-1] + str(int(base_version[-1]) - 1)
-create_docker(AIRFLOW_DOCKER.format(prev_version, version))
+create_docker(AIRFLOW_DOCKER.format(version))
if missing_files:
warn_of_missing_files(missing_files)
return
[airflow] 07/10: Make Breeze help generation indepdent from having breeze installed (#23612)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 6235ac248a56dc47081b34db3c2b6e97ac6bb8cd
Author: Jarek Potiuk
AuthorDate: Tue May 10 11:49:39 2022 +0200
Make Breeze help generation indepdent from having breeze installed (#23612)
Generation of Breeze help requires breeze to be installed. However
if you have locally installed breeze with different dependencies
and did not run self-upgrade, the results of generation of the
images might be different (for example when different rich
version is used). This change works in the way that:
* you do not have to have breeze installed at all to make it work
* it always upgrades to latest breeze when it is not installed
* but this only happens when you actually modified some breeze code
(cherry picked from commit e36868ba1b8a26f838851d1d09cca9db64cbcdc8)
---
scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py | 9 +++--
1 file changed, 3 insertions(+), 6 deletions(-)
diff --git a/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py
b/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py
index 16494672c4..c01c6bfbf6 100755
--- a/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py
+++ b/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py
@@ -25,7 +25,8 @@ from rich.console import Console
AIRFLOW_SOURCES_DIR = Path(__file__).parents[3].resolve()
BREEZE_IMAGES_DIR = AIRFLOW_SOURCES_DIR / "images" / "breeze"
-BREEZE_SOURCES_DIR = AIRFLOW_SOURCES_DIR / "dev" / "breeze" / "src"
+BREEZE_INSTALL_DIR = AIRFLOW_SOURCES_DIR / "dev" / "breeze"
+BREEZE_SOURCES_DIR = BREEZE_INSTALL_DIR / "src"
SCREENSHOT_WIDTH = "120"
@@ -66,11 +67,7 @@ def print_help_for_all_commands():
if old_hash == new_hash:
console.print(f"[bright_blue]Skip generation of SVG images as command
hash is unchanged {old_hash}")
return
-if run(["breeze", "--help"], check=False).returncode != 0:
-console.print("[red]ERROR! You need to install breeze with pipx to run
this pre-commit[/]")
-console.print("\n[bright_blue]Run this command:[/]\n")
-console.print("pip install -e ./dev/breeze --force\n")
-sys.exit(1)
+run([sys.executable, "-m", "pip", "install", "--upgrade", "-e",
BREEZE_INSTALL_DIR])
env = os.environ.copy()
env['AIRFLOW_SOURCES_ROOT'] = str(AIRFLOW_SOURCES_DIR)
env['RECORD_BREEZE_WIDTH'] = SCREENSHOT_WIDTH
[airflow] branch v2-3-test updated (891102c28a -> 2f35d6e132)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git omit 891102c28a fixup! Update version to 2.3.1 and add release notes omit 8db8b9e5db Update version to 2.3.1 and add release notes add 2f35d6e132 Update version to 2.3.1 and add release notes This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (891102c28a) \ N -- N -- N refs/heads/v2-3-test (2f35d6e132) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: RELEASE_NOTES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[airflow] 02/10: Remove tagging of `constraint-x-y` branch from release process (#23399)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 65f6a66f6238b9e3c4ef2e45f990bb6f2ce7bc6e
Author: Ephraim Anierobi
AuthorDate: Mon May 2 10:56:05 2022 +0100
Remove tagging of `constraint-x-y` branch from release process (#23399)
(cherry picked from commit 9faffc6c610adb86e421f24c24d1f8749551b286)
---
dev/README_RELEASE_AIRFLOW.md | 1 -
1 file changed, 1 deletion(-)
diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md
index 27774426aa..ebe94e6abb 100644
--- a/dev/README_RELEASE_AIRFLOW.md
+++ b/dev/README_RELEASE_AIRFLOW.md
@@ -424,7 +424,6 @@ protected_branches:
git checkout constraints-main
git checkout -b constraints-${BRANCH_PREFIX}
git push --set-upstream origin constraints-${BRANCH_PREFIX}
- git push origin tag constraints-${BRANCH_PREFIX}
```
[airflow] branch v2-3-test updated (ffb3ee0617 -> d8d87162ea)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git from ffb3ee0617 Fix python version used for cache preparaation (#23785) new 8ed7a80ead Update Airflow Release Doc (#23322) new 65f6a66f62 Remove tagging of `constraint-x-y` branch from release process (#23399) new 7de32629e2 Add missing steps to release process (#23384) new f9287821cb Fix `check_files.py` to work on new minor releases (#23287) new b77e616aa4 Fix typos in README.md and airflow_doc_issue_report.yml (#23294) new bae9a1b2dd Add version to migration prefix (#23564) new 6235ac248a Make Breeze help generation indepdent from having breeze installed (#23612) new 7d4090d5fc Shorten max pre-commit hook name length (#23677) new 91438f4dc0 Modify db clean to also catch the ProgrammingError exception (#23699) new d8d87162ea Fix "breeze-legacy" after building images was removed (#23404) The 10 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../ISSUE_TEMPLATE/airflow_doc_issue_report.yml| 2 +- .pre-commit-config.yaml| 6 +- CONTRIBUTING.rst | 14 +- STATIC_CODE_CHECKS.rst | 374 ++--- ...rent_schema.py => 0001_1_5_0_current_schema.py} | 0 ...rypted.py => 0002_1_5_0_create_is_encrypted.py} | 0 ...tibility.py => 0003_1_5_0_for_compatibility.py} | 0 ... 0004_1_5_0_more_logging_into_task_isntance.py} | 0 ..._id_indices.py => 0005_1_5_2_job_id_indices.py} | 0 ...to_log.py => 0006_1_6_0_adding_extra_to_log.py} | 0 ...b76e_add_dagrun.py => 0007_1_6_0_add_dagrun.py} | 0 ...ask_duration.py => 0008_1_6_0_task_duration.py} | 0 ...agrun_config.py => 0009_1_6_0_dagrun_config.py} | 0 ...y => 0010_1_6_2_add_password_column_to_user.py} | 0 ...start_end.py => 0011_1_6_2_dagrun_start_end.py} | 0 ..._0_add_notification_sent_column_to_sla_miss.py} | 0 ...1_7_0_add_a_column_to_track_the_encryption_.py} | 0 ..._1_7_0_add_is_encrypted_column_to_variable_.py} | 0 ...er_table.py => 0015_1_7_1_rename_user_table.py} | 0 ...e_index.py => 0016_1_7_1_add_ti_state_index.py} | 0 ... => 0017_1_7_1_add_task_fails_journal_table.py} | 0 ..._table.py => 0018_1_7_1_add_dag_stats_table.py} | 0 ..._7_1_add_fractional_seconds_to_mysql_tables.py} | 0 ...ices.py => 0020_1_7_1_xcom_dag_task_indices.py} | 0 ...=> 0021_1_7_1_add_pid_field_to_taskinstance.py} | 0 ...7_1_add_dag_id_state_index_on_dag_run_table.py} | 0 ...1_8_2_add_max_tries_column_to_task_instance.py} | 0 ...1_8_2_make_xcom_value_column_a_large_binary.py} | 0 ..._index.py => 0025_1_8_2_add_ti_job_id_index.py} | 0 ... => 0026_1_8_2_increase_text_size_for_mysql.py} | 0 ...s.py => 0027_1_10_0_add_time_zone_awareness.py} | 0 ..._10_0_add_kubernetes_resource_checkpointing.py} | 0 ...1_10_0_add_executor_config_to_task_instance.py} | 0 ..._1_10_0_add_kubernetes_scheduler_uniqueness.py} | 0 ...6_merge_heads.py => 0031_1_10_0_merge_heads.py} | 0 ...> 0032_1_10_0_fix_mysql_not_null_constraint.py} | 0 ...ey.py => 0033_1_10_0_fix_sqlite_foreign_key.py} | 0 ...x_taskfail.py => 0034_1_10_0_index_taskfail.py} | 0 ...x_log_dag.py => 0035_1_10_2_add_idx_log_dag.py} | 0 ...py => 0036_1_10_2_add_index_to_taskinstance.py} | 0 ...py => 0037_1_10_2_add_task_reschedule_table.py} | 0 ...ag_index.py => 0038_1_10_2_add_sm_dag_index.py} | 0 ...field.py => 0039_1_10_2_add_superuser_field.py} | 0 ..._to_dag.py => 0040_1_10_3_add_fields_to_dag.py} | 0 ...=> 0041_1_10_3_add_schedule_interval_to_dag.py} | 0 ...1_10_3_task_reschedule_fk_on_cascade_delete.py} | 0 ..._1_10_4_make_taskinstance_pool_not_nullable.py} | 0 py => 0044_1_10_7_add_serialized_dag_table.py} | 0 ...ag.py => 0045_1_10_7_add_root_dag_id_to_dag.py} | 0 ..._5_change_datetime_to_datetime2_6_on_mssql_.py} | 0 ... 0047_1_10_4_increase_queue_name_size_limit.py} | 0 ...ble.py => 0048_1_10_3_remove_dag_stat_table.py} | 0 ...4_merge_heads.py => 0049_1_10_7_merge_heads.py} | 0 ...0_7_increase_length_for_connection_password.py} | 0 ...s_table.py => 0051_1_10_8_add_dagtags_table.py} | 0 ...10_10_add_pool_slots_field_to_task_instance.py} | 0 ..._10_add_rendered_task_instance_fields_table.py} | 0 ...table.py => 0054_1_10_10_add_dag_code_table.py} | 0 ...11_add_precision_to_execution_date_in_mysql.py} | 0 ...0_12_add_dag_hash_column_to_serialized_dag_.py} | 0 ...ab_tables.py => 0057_1_10_13_add_fa
[airflow] 09/10: Modify db clean to also catch the ProgrammingError exception (#23699)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 91438f4dc0244d6fa29c8fe8372ad5fcc04ae17e
Author: Jian Yuan Lee
AuthorDate: Thu May 19 17:47:56 2022 +0100
Modify db clean to also catch the ProgrammingError exception (#23699)
(cherry picked from commit a80b2fcaea984813995d4a2610987a1c9068fdb5)
---
airflow/utils/db_cleanup.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/airflow/utils/db_cleanup.py b/airflow/utils/db_cleanup.py
index 93e86c7825..b02d08503f 100644
--- a/airflow/utils/db_cleanup.py
+++ b/airflow/utils/db_cleanup.py
@@ -27,7 +27,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional,
Union
from pendulum import DateTime
from sqlalchemy import and_, false, func
-from sqlalchemy.exc import OperationalError
+from sqlalchemy.exc import OperationalError, ProgrammingError
from airflow.cli.simple_table import AirflowConsole
from airflow.jobs.base_job import BaseJob
@@ -260,7 +260,7 @@ class _warn_if_missing(AbstractContextManager):
return self
def __exit__(self, exctype, excinst, exctb):
-caught_error = exctype is not None and issubclass(exctype,
OperationalError)
+caught_error = exctype is not None and issubclass(exctype,
(OperationalError, ProgrammingError))
if caught_error:
logger.warning("Table %r not found. Skipping.", self.table)
return caught_error
[airflow] 01/10: Update Airflow Release Doc (#23322)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 8ed7a80eadc16280da740f69943bafa962d09381
Author: Ephraim Anierobi
AuthorDate: Thu Apr 28 19:51:20 2022 +0100
Update Airflow Release Doc (#23322)
Co-authored-by: Kaxil Naik
Co-authored-by: Jed Cunningham
<66968678+jedcunning...@users.noreply.github.com>
(cherry picked from commit 2dfccfa82f602ab71db417bd9b45e86988ce29e4)
---
dev/README_RELEASE_AIRFLOW.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md
index ebe94e6abb..27774426aa 100644
--- a/dev/README_RELEASE_AIRFLOW.md
+++ b/dev/README_RELEASE_AIRFLOW.md
@@ -424,6 +424,7 @@ protected_branches:
git checkout constraints-main
git checkout -b constraints-${BRANCH_PREFIX}
git push --set-upstream origin constraints-${BRANCH_PREFIX}
+ git push origin tag constraints-${BRANCH_PREFIX}
```
[airflow] 03/10: Add missing steps to release process (#23384)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git commit 7de32629e26b53b73f91aec048dd777dfb697d16 Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com> AuthorDate: Sun May 1 04:09:53 2022 -0600 Add missing steps to release process (#23384) (cherry picked from commit d670e571e499ad8e97dedfed3d81ed59469bce7c) --- dev/README_RELEASE_HELM_CHART.md| 5 + dev/README_RELEASE_PROVIDER_PACKAGES.md | 10 ++ 2 files changed, 15 insertions(+) diff --git a/dev/README_RELEASE_HELM_CHART.md b/dev/README_RELEASE_HELM_CHART.md index b632e10e76..8662f80a96 100644 --- a/dev/README_RELEASE_HELM_CHART.md +++ b/dev/README_RELEASE_HELM_CHART.md @@ -42,6 +42,7 @@ - [Update Announcements page](#update-announcements-page) - [Create release on GitHub](#create-release-on-github) - [Close the milestone](#close-the-milestone) + - [Close the testing status issue](#close-the-testing-status-issue) - [Update issue template with the new release](#update-issue-template-with-the-new-release) - [Announce the release on the community slack](#announce-the-release-on-the-community-slack) - [Tweet about the release](#tweet-about-the-release) @@ -702,6 +703,10 @@ Close the milestone on GitHub. Create the next one if it hasn't been already (it Update the new milestone in the [*Currently we are working on* issue](https://github.com/apache/airflow/issues/10176) make sure to update the last updated timestamp as well. +## Close the testing status issue + +Don't forget to thank the folks who tested and close the issue tracking the testing status. + ## Update issue template with the new release Updating issue templates in `.github/ISSUE_TEMPLATE/airflow_helmchart_bug_report.yml` with the new version diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md index 29bebef09d..4847dca6bd 100644 --- a/dev/README_RELEASE_PROVIDER_PACKAGES.md +++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md @@ -44,6 +44,8 @@ - [Publish documentation prepared before](#publish-documentation-prepared-before) - [Add tags in git](#add-tags-in-git-1) - [Notify developers of release](#notify-developers-of-release) + - [Add release data to Apache Committee Report Helper](#add-release-data-to-apache-committee-report-helper) + - [Close the testing status issue](#close-the-testing-status-issue) @@ -870,3 +872,11 @@ Cheers, EOF ``` + +## Add release data to Apache Committee Report Helper + +Add the release data (version and date) at: https://reporter.apache.org/addrelease.html?airflow + +## Close the testing status issue + +Don't forget to thank the folks who tested and close the issue tracking the testing status.
[airflow] branch v2-3-test updated: Update version to 2.3.1 and add release notes
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/v2-3-test by this push: new 8db8b9e5db Update version to 2.3.1 and add release notes 8db8b9e5db is described below commit 8db8b9e5db3f97ce04e31ba717961af3943a0f11 Author: Ephraim Anierobi AuthorDate: Thu May 19 22:27:16 2022 +0100 Update version to 2.3.1 and add release notes --- README.md | 14 ++-- RELEASE_NOTES.rst | 76 ++ airflow/utils/db.py| 1 + .../installation/supported-versions.rst| 2 +- docs/docker-stack/README.md| 10 +-- .../customizing/pypi-extras-and-deps.sh| 2 +- .../customizing/pypi-selected-version.sh | 2 +- .../extending/add-apt-packages/Dockerfile | 2 +- .../add-build-essential-extend/Dockerfile | 2 +- .../extending/add-providers/Dockerfile | 2 +- .../extending/add-pypi-packages/Dockerfile | 2 +- .../extending/custom-providers/Dockerfile | 2 +- .../extending/embedding-dags/Dockerfile| 2 +- .../extending/writable-directory/Dockerfile| 2 +- docs/docker-stack/entrypoint.rst | 18 ++--- .../ci/pre_commit/pre_commit_supported_versions.py | 2 +- setup.py | 2 +- 17 files changed, 110 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index b7282212cb..3acb883dc4 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ Airflow is not a streaming solution, but it is often used to process real-time d Apache Airflow is tested with: -| | Main version (dev) | Stable version (2.3.0) | +| | Main version (dev) | Stable version (2.3.1) | |-|--|--| | Python | 3.7, 3.8, 3.9, 3.10 | 3.7, 3.8, 3.9, 3.10 | | Platform| AMD64/ARM64(\*) | AMD64/ARM64(\*) | @@ -160,15 +160,15 @@ them to the appropriate format and workflow that your tool requires. ```bash -pip install 'apache-airflow==2.3.0' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.3.0/constraints-3.7.txt; +pip install 'apache-airflow==2.3.1' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.3.1/constraints-3.7.txt; ``` 2. Installing with extras (i.e., postgres, google) ```bash -pip install 'apache-airflow[postgres,google]==2.3.0' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.3.0/constraints-3.7.txt; +pip install 'apache-airflow[postgres,google]==2.3.1' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.3.1/constraints-3.7.txt; ``` For information on installing provider packages, check @@ -273,7 +273,7 @@ Apache Airflow version life cycle: | Version | Current Patch/Minor | State | First Release | Limited Support | EOL/Terminated | |---|---|---|-|---|--| -| 2 | 2.3.0 | Supported | Dec 17, 2020| TBD | TBD | +| 2 | 2.3.1 | Supported | Dec 17, 2020| TBD | TBD | | 1.10 | 1.10.15 | EOL | Aug 27, 2018| Dec 17, 2020 | June 17, 2021| | 1.9 | 1.9.0 | EOL | Jan 03, 2018| Aug 27, 2018 | Aug 27, 2018 | | 1.8 | 1.8.2 | EOL | Mar 19, 2017| Jan 03, 2018 | Jan 03, 2018 | @@ -303,7 +303,7 @@ They are based on the official release schedule of Python and Kubernetes, nicely 2. The "oldest" supported version of Python/Kubernetes is the default one until we decide to switch to later version. "Default" is only meaningful in terms of "smoke tests" in CI PRs, which are run using this default version and the default reference image available. Currently `apache/airflow:latest` - and `apache/airflow:2.3.0` images are Python 3.7 images. This means that default reference image will + and `apache/airflow:2.3.1` images are Python 3.7 images. This means that default reference image will become the default at the time when we start preparing for dropping 3.7 support which is few months before the end of life for Python 3.7. diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 3a51174411..60944e7443 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -21,6 +21,82 @@ .. towncrie
[airflow] branch v2-3-test updated (8db8b9e5db -> 891102c28a)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git from 8db8b9e5db Update version to 2.3.1 and add release notes add 891102c28a fixup! Update version to 2.3.1 and add release notes No new revisions were added by this update. Summary of changes: RELEASE_NOTES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[airflow] branch main updated: Have consistent types between the ORM and the migration files (#24044)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 25537acfa2 Have consistent types between the ORM and the migration
files (#24044)
25537acfa2 is described below
commit 25537acfa28eebc82a90274840e0e6fb5c91e271
Author: Ephraim Anierobi
AuthorDate: Fri Jun 24 16:51:01 2022 +0100
Have consistent types between the ORM and the migration files (#24044)
We currently don't compare column types between ORM and the migration
files. Some columns in the migration files have different types from the same
columns in the ORM.
Here, I made effort to match the types in migration files with the
types in ORM, using the migration files as the source of truth in most
cases.
I couldn't convert the MySQL VARCHAR collation in db(utf8_bin) to use the
one in ORM(utf8mb3_bin). It seems it's not possible to convert a collation of
an already existing column in MySQL.
---
airflow/migrations/env.py | 9 +-
.../versions/0112_2_4_0_add_dagwarning_model.py| 2 +-
.../0113_2_4_0_compare_types_between_orm_and_db.py | 253 +
airflow/models/connection.py | 2 +-
airflow/models/dag.py | 2 +-
airflow/models/dagcode.py | 3 +-
airflow/models/dagpickle.py| 4 +-
airflow/models/dagrun.py | 6 +-
airflow/models/dagwarning.py | 4 +-
airflow/models/log.py | 4 +-
airflow/models/renderedtifields.py | 4 +-
airflow/models/serialized_dag.py | 2 +-
airflow/models/taskfail.py | 4 +-
airflow/models/taskinstance.py | 23 +-
airflow/models/variable.py | 3 +-
airflow/models/xcom.py | 4 +-
airflow/utils/db.py| 55 +
airflow/utils/sqlalchemy.py| 20 +-
airflow/www/fab_security/sqla/models.py| 60 +
docs/apache-airflow/migrations-ref.rst | 4 +-
tests/utils/test_db.py | 15 +-
21 files changed, 389 insertions(+), 94 deletions(-)
diff --git a/airflow/migrations/env.py b/airflow/migrations/env.py
index 58a8f7f4a0..039d7cf23f 100644
--- a/airflow/migrations/env.py
+++ b/airflow/migrations/env.py
@@ -21,6 +21,7 @@ from logging.config import fileConfig
from alembic import context
from airflow import models, settings
+from airflow.utils.db import compare_server_default, compare_type
def include_object(_, name, type_, *args):
@@ -51,8 +52,6 @@ target_metadata = models.base.Base.metadata
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
-COMPARE_TYPE = False
-
def run_migrations_offline():
"""Run migrations in 'offline' mode.
@@ -70,7 +69,8 @@ def run_migrations_offline():
url=settings.SQL_ALCHEMY_CONN,
target_metadata=target_metadata,
literal_binds=True,
-compare_type=COMPARE_TYPE,
+compare_type=compare_type,
+compare_server_default=compare_server_default,
render_as_batch=True,
)
@@ -92,7 +92,8 @@ def run_migrations_online():
connection=connection,
transaction_per_migration=True,
target_metadata=target_metadata,
-compare_type=COMPARE_TYPE,
+compare_type=compare_type,
+compare_server_default=compare_server_default,
include_object=include_object,
render_as_batch=True,
)
diff --git a/airflow/migrations/versions/0112_2_4_0_add_dagwarning_model.py
b/airflow/migrations/versions/0112_2_4_0_add_dagwarning_model.py
index cb7d089871..3d03210e53 100644
--- a/airflow/migrations/versions/0112_2_4_0_add_dagwarning_model.py
+++ b/airflow/migrations/versions/0112_2_4_0_add_dagwarning_model.py
@@ -44,7 +44,7 @@ def upgrade():
'dag_warning',
sa.Column('dag_id', StringID(), primary_key=True),
sa.Column('warning_type', sa.String(length=50), primary_key=True),
-sa.Column('message', sa.String(1000), nullable=False),
+sa.Column('message', sa.Text(), nullable=False),
sa.Column('timestamp', TIMESTAMP, nullable=False),
sa.ForeignKeyConstraint(
('dag_id',),
diff --git
a/airflow/migrations/versions/0113_2_4_0_compare_types_between_orm_and_db.py
b/airflow/migrations/versions/0113_2_4_0_compare_types_between_orm_and_db.py
new file mode 100644
index 00..68f02dd81e
--- /dev/null
+++ b/airflow/migrations/versions/0113_2_4_0_compare_types_between_orm_and_db.py
@@ -0,0 +1,253 @@
+#
+# Licensed to the Apache Software F
[airflow] branch v2-3-test updated (78f215a192 -> e86dbd7c1c)
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a change to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git discard 78f215a192 Update the release note discard a8762fca89 improve grid date tick spacing (#24849) discard 7a4869c737 Fix cycle bug with attaching label to task group (#24847) discard 691ceab953 Fix timestamp defaults for sensorinstance (#24638) discard 51c58c0c68 Move fallible ti.task.dag assignment back inside try/except block (#24533) (#24592) discard 465834bfbe Fix typo (#24568) discard 2b9f6d0512 Add missing types to FSHook (#24470) discard 7a11046ca1 Mask secrets in stdout for 'airflow tasks test' (#24362) discard b9fb473eb3 DebugExecutor use ti.run() instead of ti._run_raw_task (#24357) discard d3366fcdde Fix bugs in URI constructor for MySQL connection (#24320) discard 49fc732db3 scheduleinterval nullable true added in openapi (#24253) discard 065cd19de4 Unify return_code interface for task runner (#24093) discard 4284d03e98 Handle occasional deadlocks in trigger with retries (#24071) discard 46d26c76af Drop Python 3.6 compatibility objects/modules (#24048) discard 4081a65b77 Remove special serde logic for mapped op_kwargs (#23860) discard 6ca7f3a317 Fix doc description of [core] parallelism config setting (#23768) discard 7b548b0e69 ExternalTaskSensor respects soft_fail if the external task enters a failed_state (#23647) discard 40f16612f9 Fix StatD timing metric units (#21106) discard 679bd6afaf Add cache_ok flag to sqlalchemy TypeDecorators. (#24499) discard 069a84c4ad Remove upper-binding for SQLAlchemy (#24819) discard ec02981daa Adding Docker context check for breeze (#24751) discard cd6987f7cb Add "generated" folder to volumes mounted when "MOUNT_SELECTED" used (#24818) discard 951eb0029f Prepare ARM images much faster and cheaper (#24813) discard 0ad211d9d2 Unified "dash-name" convention for outputs in ci workflows. add b5202fa4ef Unified "dash-name" convention for outputs in ci workflows. add bffde56e05 Unified "dash-name" convention for outputs in ci workflows. add 7c2f45c029 Prepare ARM images much faster and cheaper (#24813) add 0d31c8b913 Add "generated" folder to volumes mounted when "MOUNT_SELECTED" used (#24818) add 29a5ceff48 Adding Docker context check for breeze (#24751) add a33bf03424 Remove upper-binding for SQLAlchemy (#24819) add fb33bbfd6d Add cache_ok flag to sqlalchemy TypeDecorators. (#24499) add aa1f8d9531 Fix StatD timing metric units (#21106) add f89c8f0693 ExternalTaskSensor respects soft_fail if the external task enters a failed_state (#23647) add 57dc2c5b78 Fix doc description of [core] parallelism config setting (#23768) add a2968fff47 Remove special serde logic for mapped op_kwargs (#23860) add 9c5a07dbc9 Drop Python 3.6 compatibility objects/modules (#24048) add c76b324d5e Handle occasional deadlocks in trigger with retries (#24071) add e950d5ca24 Unify return_code interface for task runner (#24093) add e9457d3fad scheduleinterval nullable true added in openapi (#24253) add 9c6380db0e Fix bugs in URI constructor for MySQL connection (#24320) add 13e4a752a5 DebugExecutor use ti.run() instead of ti._run_raw_task (#24357) add 3ed053dca6 Mask secrets in stdout for 'airflow tasks test' (#24362) add 5dca4fb64b Add missing types to FSHook (#24470) add 7c6bca27df Fix typo (#24568) add 53121c5ffc Move fallible ti.task.dag assignment back inside try/except block (#24533) (#24592) add e32c1bffbf Fix timestamp defaults for sensorinstance (#24638) add efc3c67006 Fix cycle bug with attaching label to task group (#24847) add eea449e136 improve grid date tick spacing (#24849) add e86dbd7c1c Update the release note This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (78f215a192) \ N -- N -- N refs/heads/v2-3-test (e86dbd7c1c) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes:
[airflow] 01/03: Fix cycle bug with attaching label to task group (#24847)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 7a4869c737e46acb1fe794acef588b7251d2c4a9
Author: Ash Berlin-Taylor
AuthorDate: Tue Jul 5 16:40:00 2022 +0100
Fix cycle bug with attaching label to task group (#24847)
The problem was specific to EdgeModifiers as they try to be
"transparent" to upstream/downstream
The fix is to set track the upstream/downstream for the task group
before making any changes to the EdgeModifiers' relations -- otherwise
the roots of the TG were added as dependencies to themeslves!
(cherry picked from commit efc05a5f0b3d261293c2efaf6771e4af9a2f324c)
---
airflow/utils/task_group.py| 12 ++--
tests/utils/test_task_group.py | 25 +
2 files changed, 31 insertions(+), 6 deletions(-)
diff --git a/airflow/utils/task_group.py b/airflow/utils/task_group.py
index ed8d380ff0..64c11f79db 100644
--- a/airflow/utils/task_group.py
+++ b/airflow/utils/task_group.py
@@ -283,6 +283,12 @@ class TaskGroup(DAGNode):
Call set_upstream/set_downstream for all root/leaf tasks within this
TaskGroup.
Update
upstream_group_ids/downstream_group_ids/upstream_task_ids/downstream_task_ids.
"""
+if not isinstance(task_or_task_list, Sequence):
+task_or_task_list = [task_or_task_list]
+
+for task_like in task_or_task_list:
+self.update_relative(task_like, upstream)
+
if upstream:
for task in self.get_roots():
task.set_upstream(task_or_task_list)
@@ -290,12 +296,6 @@ class TaskGroup(DAGNode):
for task in self.get_leaves():
task.set_downstream(task_or_task_list)
-if not isinstance(task_or_task_list, Sequence):
-task_or_task_list = [task_or_task_list]
-
-for task_like in task_or_task_list:
-self.update_relative(task_like, upstream)
-
def __enter__(self) -> "TaskGroup":
TaskGroupContext.push_context_managed_task_group(self)
return self
diff --git a/tests/utils/test_task_group.py b/tests/utils/test_task_group.py
index 9aacc96b82..864b2fb68a 100644
--- a/tests/utils/test_task_group.py
+++ b/tests/utils/test_task_group.py
@@ -1222,3 +1222,28 @@ def test_add_to_another_group():
tg.add(task)
assert str(ctx.value) == "cannot add 'section_2.task' to 'section_1'
(already in group 'section_2')"
+
+
+def test_task_group_edge_modifier_chain():
+from airflow.models.baseoperator import chain
+from airflow.utils.edgemodifier import Label
+
+with DAG(dag_id="test", start_date=pendulum.DateTime(2022, 5, 20)) as dag:
+start = EmptyOperator(task_id="sleep_3_seconds")
+
+with TaskGroup(group_id="group1") as tg:
+t1 = EmptyOperator(task_id="dummy1")
+t2 = EmptyOperator(task_id="dummy2")
+
+t3 = EmptyOperator(task_id="echo_done")
+
+# The case we are testing for is when a Label is inside a list -- meaning
that we do tg.set_upstream
+# instead of label.set_downstream
+chain(start, [Label("branch three")], tg, t3)
+
+assert start.downstream_task_ids == {t1.node_id, t2.node_id}
+assert t3.upstream_task_ids == {t1.node_id, t2.node_id}
+assert tg.upstream_task_ids == set()
+assert tg.downstream_task_ids == {t3.node_id}
+# Check that we can perform a topological_sort
+dag.topological_sort()
[airflow] 03/03: Update the release note
This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-3-test in repository https://gitbox.apache.org/repos/asf/airflow.git commit 78f215a192914ed3ef1a7105dddf727be8494dd1 Author: Ephraim Anierobi AuthorDate: Tue Jul 5 15:44:34 2022 +0100 Update the release note --- RELEASE_NOTES.rst | 19 ++- docs/docker-stack/index.rst| 4 ++-- docs/spelling_wordlist.txt | 1 + newsfragments/23647.bugfix.rst | 1 - 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index ed9ea874af..de769ef453 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -64,8 +64,22 @@ Here is the list of breaking changes in dependencies that comes together with FA Bug Fixes ^ +- Fix cycle bug with attaching label to task group (#24847) +- Fix timestamp defaults for ``sensorinstance`` (#24638) +- Move fallible ``ti.task.dag`` assignment back inside ``try/except`` block (#24533) (#24592) +- Add missing types to ``FSHook`` (#24470) +- Mask secrets in ``stdout`` for ``airflow tasks test`` (#24362) +- ``DebugExecutor`` use ``ti.run()`` instead of ``ti._run_raw_task`` (#24357) +- Fix bugs in ``URI`` constructor for ``MySQL`` connection (#24320) +- Missing ``scheduleinterval`` nullable true added in ``openapi`` (#24253) +- Unify ``return_code`` interface for task runner (#24093) +- Handle occasional deadlocks in trigger with retries (#24071) +- Remove special serde logic for mapped ``op_kwargs`` (#23860) +- ``ExternalTaskSensor`` respects ``soft_fail`` if the external task enters a ``failed_state`` (#23647) +- Fix ``StatD`` timing metric units (#21106) +- Add ``cache_ok`` flag to sqlalchemy TypeDecorators. (#24499) - Allow for ``LOGGING_LEVEL=DEBUG`` (#23360) -- Fix grid date ticks (#24738) +- Fix grid date ticks (#24738, #24849) - Debounce status highlighting in Grid view (#24710) - Fix Grid vertical scrolling (#24684) - don't try to render child rows for closed groups (#24637) @@ -107,6 +121,7 @@ Bug Fixes Doc only changes +- Fix doc description of ``[core]`` parallelism config setting (#23768) - Update templates doc to mention ``extras`` and format Airflow ``Vars`` / ``Conns`` (#24735) - Document built in Timetables (#23099) - Alphabetizes two tables (#23923) @@ -121,6 +136,8 @@ Doc only changes Misc/Internal ^ +- Drop Python ``3.6`` compatibility objects/modules (#24048) +- Remove upper-binding for SQLAlchemy (#24819) - Remove internet explorer support (#24495) - Removing magic status code numbers from ``api_connexion`` (#24050) - Upgrade FAB to ``4.1.2`` (#24619) diff --git a/docs/docker-stack/index.rst b/docs/docker-stack/index.rst index ed52625031..87e1f4088c 100644 --- a/docs/docker-stack/index.rst +++ b/docs/docker-stack/index.rst @@ -59,7 +59,7 @@ Those are "reference" regular images. They contain the most common set of extras often used by the users and they are good to "try-things-out" when you want to just take Airflow for a spin, You can also use "slim" images that contain only core airflow and are about half the size of the "regular" images -but you need to add all the :doc:`extra-packages-ref` and providers that you need separately +but you need to add all the extra packages and providers that you need separately via :ref:`Building the image `. * :subst-code:`apache/airflow:slim-latest` - the latest released Airflow image with default Python version (3.7 currently) @@ -69,7 +69,7 @@ via :ref:`Building the image `. The Apache Airflow image provided as convenience package is optimized for size, and it provides just a bare minimal set of the extras and dependencies installed and in most cases -you want to either extend or customize the image. You can see all possible extras in :doc:`extra-packages-ref`. +you want to either extend or customize the image. You can see all possible extras in :doc:`apache-airflow:extra-packages-ref`. The set of extras used in Airflow Production image are available in the `Dockerfile <https://github.com/apache/airflow/blob/2c6c7fdb2308de98e142618836bdf414df9768c8/Dockerfile#L37>`_. diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index fd7a57694d..33a6acdc50 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -1319,6 +1319,7 @@ seealso seedlist segmentGranularity sendgrid +serde serialise serializable serverless diff --git a/newsfragments/23647.bugfix.rst b/newsfragments/23647.bugfix.rst deleted file mode 100644 index d12c1d7046..00 --- a/newsfragments/23647.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -``ExternalTaskSensor`` now supports the ``soft_fail`` flag to skip if external task or DAG enters a failed state.
