jason810496 commented on code in PR #71214:
URL: https://github.com/apache/airflow/pull/71214#discussion_r4618376776
##########
airflow-core/src/airflow/utils/db_cleanup.py:
##########
@@ -233,7 +233,7 @@ def readable_config(self):
keep_last=True,
keep_last_filters=[column("run_type") != DagRunType.MANUAL],
keep_last_group_by=["dag_id"],
- dependent_tables=["task_instance", "task_state_store", "deadline"],
+ dependent_tables=["task_instance", "task_state_store", "deadline",
"callback"],
Review Comment:
`callback` has no Dag scope, so `db clean -t dag_run --dag-ids foo` now
cleans callbacks of every Dag. Scope it like `deadline`:
```python
_TableConfig(
table_name="callback",
recency_column_name="created_at",
extra_columns=["id", "state", "dagrun_id"],
dag_id_scope=_IndirectDagScope(fk_column="dagrun_id",
referenced_table="dag_run"),
...
),
```
##########
airflow-core/src/airflow/models/deadline.py:
##########
@@ -327,9 +327,9 @@ def callback_data_with_context():
elif isinstance(self.callback, ExecutorCallback):
data = callback_data_with_context()
data["deadline_id"] = str(self.id)
- data["dag_run_id"] = str(self.dagrun.id)
data["dag_id"] = self.dagrun.dag_id
self.callback.data = data
+ self.callback.dagrun_id = self.dagrun.id
Review Comment:
Could `Deadline.__init__` set this too ([earlier
comment](https://github.com/apache/airflow/pull/71214#discussion_r4331773418))?
Until a miss, deleting the run leaves the SCHEDULED callback orphaned.
```python
self.callback.bundle_name = bundle_name
self.callback.dagrun_id = dagrun_id
```
##########
airflow-core/src/airflow/models/deadline.py:
##########
@@ -327,9 +327,9 @@ def callback_data_with_context():
elif isinstance(self.callback, ExecutorCallback):
data = callback_data_with_context()
data["deadline_id"] = str(self.id)
- data["dag_run_id"] = str(self.dagrun.id)
data["dag_id"] = self.dagrun.dag_id
Review Comment:
Dropping `data["dag_run_id"]` also drops the `dag_run_id` metric tag, since
`get_metric_info` spreads `data`. Is this Intended?
##########
airflow-core/src/airflow/migrations/versions/0158_3_5_0_add_dagrun_id_to_callback.py:
##########
@@ -0,0 +1,217 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Add dagrun_id foreign key to callback.
+
+Revision ID: 0f4c9a2b8e1d
+Revises: c64b10719069
+Create Date: 2026-11-11 00:00:00.000000
+
+"""
+
+from __future__ import annotations
+
+import json
+
+import sqlalchemy as sa
+from alembic import context, op
+
+from airflow.configuration import conf
+from airflow.migrations.utils import disable_sqlite_fkeys
+
+# revision identifiers, used by Alembic.
+revision = "0f4c9a2b8e1d"
+down_revision = "c64b10719069"
+branch_labels = None
+depends_on = None
+airflow_version = "3.5.0"
+
+
+def _deserialize_callback_data(data):
+ if isinstance(data, str):
+ try:
+ data = json.loads(data)
+ except json.JSONDecodeError:
+ return None
+ if not isinstance(data, dict):
+ return None
+ callback_data = data.get("__var", data)
+ return (data, callback_data) if isinstance(callback_data, dict) else None
+
+
+def _extract_dagrun_id(data) -> int | None:
+ deserialized_data = _deserialize_callback_data(data)
+ if deserialized_data is None:
+ return None
+ _, callback_data = deserialized_data
+ dagrun_id = callback_data.get("dag_run_id")
+ if dagrun_id is None:
+ return None
+ try:
+ return int(dagrun_id)
+ except (TypeError, ValueError):
+ return None
+
+
+def _backfill_dagrun_id_from_deadline(conn, batch_size: int) -> None:
+ callback = sa.table(
+ "callback",
+ sa.column("id", sa.Uuid()),
+ sa.column("type", sa.String(20)),
+ sa.column("dagrun_id", sa.Integer()),
+ )
+ deadline = sa.table(
+ "deadline",
+ sa.column("callback_id", sa.Uuid()),
+ sa.column("dagrun_id", sa.Integer()),
+ )
+ dag_run = sa.table("dag_run", sa.column("id", sa.Integer()))
+
+ last_id = None
+ while True:
+ query = (
+ sa.select(callback.c.id, deadline.c.dagrun_id)
+ .select_from(
+ callback.join(deadline, callback.c.id ==
deadline.c.callback_id).join(
+ dag_run, deadline.c.dagrun_id == dag_run.c.id
+ )
+ )
+ .where(callback.c.type == "executor",
callback.c.dagrun_id.is_(None))
+ .order_by(callback.c.id)
+ .limit(batch_size)
+ )
+ if last_id is not None:
+ query = query.where(callback.c.id > last_id)
+
+ rows = conn.execute(query).fetchall()
+ if not rows:
+ return
+
+ for callback_id, dagrun_id in rows:
+ conn.execute(callback.update().where(callback.c.id ==
callback_id).values(dagrun_id=dagrun_id))
+
+ last_id = rows[-1].id
+
+
+def _backfill_dagrun_id(conn, batch_size: int) -> None:
+ callback = sa.table(
+ "callback",
+ sa.column("id", sa.Uuid()),
+ sa.column("type", sa.String(20)),
+ sa.column("data", sa.Text()),
+ sa.column("dagrun_id", sa.Integer()),
+ )
+ dag_run = sa.table("dag_run", sa.column("id", sa.Integer()))
+
+ last_id = None
+ while True:
+ query = (
+ sa.select(callback.c.id, callback.c.data)
+ .where(callback.c.type == "executor",
callback.c.dagrun_id.is_(None))
+ .order_by(callback.c.id)
+ .limit(batch_size)
+ )
+ if last_id is not None:
+ query = query.where(callback.c.id > last_id)
+
+ rows = conn.execute(query).fetchall()
+ if not rows:
+ return
+
+ for callback_id, data in rows:
+ dagrun_id = _extract_dagrun_id(data)
+ if dagrun_id is None:
+ continue
+
+ dagrun_exists = conn.execute(
+ sa.select(dag_run.c.id).where(dag_run.c.id ==
dagrun_id).limit(1)
+ ).first()
+ if dagrun_exists:
+ conn.execute(
+ callback.update().where(callback.c.id ==
callback_id).values(dagrun_id=dagrun_id)
+ )
+
+ last_id = rows[-1].id
+
+
+def _restore_legacy_dagrun_id(conn, batch_size: int) -> None:
+ callback = sa.table(
+ "callback",
+ sa.column("id", sa.Uuid()),
+ sa.column("type", sa.String(20)),
+ sa.column("data", sa.Text()),
+ sa.column("dagrun_id", sa.Integer()),
+ )
+
+ last_id = None
+ while True:
+ query = (
+ sa.select(callback.c.id, callback.c.data, callback.c.dagrun_id)
+ .where(callback.c.type == "executor",
callback.c.dagrun_id.is_not(None))
+ .order_by(callback.c.id)
+ .limit(batch_size)
+ )
+ if last_id is not None:
+ query = query.where(callback.c.id > last_id)
+
+ rows = conn.execute(query).fetchall()
+ if not rows:
+ return
+
+ for callback_id, data, dagrun_id in rows:
+ deserialized_data = _deserialize_callback_data(data)
+ if deserialized_data is None:
+ continue
+ serialized_data, callback_data = deserialized_data
+ callback_data["dag_run_id"] = str(dagrun_id)
+ conn.execute(
+ callback.update().where(callback.c.id ==
callback_id).values(data=json.dumps(serialized_data))
+ )
+
+ last_id = rows[-1].id
+
+
+def upgrade():
+ """Add callback.dagrun_id and backfill it from legacy callback.data."""
+ with disable_sqlite_fkeys(op):
+ with op.batch_alter_table("callback") as batch_op:
+ batch_op.add_column(sa.Column("dagrun_id", sa.Integer(),
nullable=True))
+
+ if not context.is_offline_mode():
Review Comment:
Offline mode skips the backfill silently. Print a warning like
`0080_3_1_0_modify_deadline_callback_schema.py` does? Same for the downgrade.
```suggestion
if context.is_offline_mode():
print("-- WARNING: callback.dagrun_id is not backfilled in
offline mode.")
else:
```
##########
airflow-core/src/airflow/jobs/scheduler_job_runner.py:
##########
@@ -1453,25 +1454,19 @@ def _enqueue_executor_callbacks(self, session: Session)
-> None:
# Can't happen since we queried ExecutorCallback, but
satisfies mypy.
continue
- # TODO: Add dagrun_id as a proper ORM foreign key on the
callback table instead of storing in data dict.
- # This would eliminate this reconstruction step. For
now, all ExecutorCallbacks
- # are expected to have dag_run_id set in their data dict
(e.g., by Deadline.handle_miss).
- if not isinstance(callback.data, dict) or "dag_run_id" not in
callback.data:
+ if callback.dagrun_id is None:
self.log.error(
- "ExecutorCallback %s is missing required 'dag_run_id'
in data dict. "
- "This indicates a bug in callback creation. Skipping
callback.",
- callback.id,
+ "Executor callback is missing dagrun_id.",
+ callback_id=callback.id,
)
continue
Review Comment:
Nothing fills `dagrun_id` later, so this row is fetched and logged every
loop ([earlier
comment](https://github.com/apache/airflow/pull/71214#discussion_r4331773418)).
Shall we fail it instead?
```suggestion
if callback.dagrun_id is None:
self.log.error("Executor callback is missing
dagrun_id.", callback_id=callback.id)
callback.state = CallbackState.FAILED
callback.output = "Executor callback is missing
dagrun_id."
continue
```
##########
airflow-core/src/airflow/models/dagrun.py:
##########
@@ -455,6 +455,13 @@ class DagRun(Base, LoggingMixin):
cascade="all, delete, delete-orphan",
)
+ callbacks = relationship(
+ "Callback",
+ back_populates="dag_run",
+ uselist=True,
+ passive_deletes=True,
Review Comment:
Nit: deadline callbacks are deleted through the `Deadline.callback` cascade.
Any other callback in a loaded `callbacks` gets `dagrun_id` set to NULL
instead. Reproduced on SQLite.
```suggestion
uselist=True,
cascade="all, delete",
passive_deletes=True,
```
##########
airflow-core/src/airflow/models/callback.py:
##########
@@ -174,6 +179,7 @@ class Callback(Base, BaseWorkload):
postgresql_where=trigger_id.is_not(None),
sqlite_where=trigger_id.is_not(None),
),
+ Index("callback_dagrun_id_idx", dagrun_id, unique=False),
Review Comment:
Nit: only executor callbacks set `dagrun_id`. Make this partial like
`idx_callback_trigger_id`? The migration needs the same change.
```suggestion
Index(
"callback_dagrun_id_idx",
dagrun_id,
postgresql_where=dagrun_id.is_not(None),
sqlite_where=dagrun_id.is_not(None),
),
```
##########
airflow-core/tests/unit/migrations/test_0158_callback_dagrun_id_migration.py:
##########
@@ -0,0 +1,206 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import uuid
+from pathlib import Path
+from unittest.mock import patch
+
+import sqlalchemy as sa
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+
+from tests_common.test_utils.paths import AIRFLOW_CORE_SOURCES_PATH
+
+_MIGRATION_PATH = (
+ Path(AIRFLOW_CORE_SOURCES_PATH)
+ / "airflow"
+ / "migrations"
+ / "versions"
+ / "0158_3_5_0_add_dagrun_id_to_callback.py"
+)
+
+_spec = importlib.util.spec_from_file_location("migration_0158",
_MIGRATION_PATH)
+assert _spec is not None
+_migration = importlib.util.module_from_spec(_spec)
+assert _spec.loader is not None
+_spec.loader.exec_module(_migration)
+
+
+def _make_engine():
+ return sa.create_engine("sqlite:///:memory:")
+
+
+def _create_tables(conn):
+ conn.execute(sa.text("CREATE TABLE dag_run (id INTEGER PRIMARY KEY)"))
+ conn.execute(
+ sa.text(
+ """
+ CREATE TABLE callback (
+ id TEXT PRIMARY KEY,
+ type TEXT NOT NULL,
+ data TEXT NOT NULL,
+ dagrun_id INTEGER
+ )
+ """
+ )
+ )
+ conn.execute(
+ sa.text(
+ """
+ CREATE TABLE deadline (
+ id TEXT PRIMARY KEY,
+ callback_id TEXT NOT NULL,
+ dagrun_id INTEGER
+ )
+ """
+ )
+ )
+
+
+def _insert_callback(conn, callback_id: str, data: dict, callback_type: str =
"executor"):
+ conn.execute(
+ sa.text("INSERT INTO callback (id, type, data) VALUES (:id, :type,
:data)"),
+ {"id": callback_id, "type": callback_type, "data": json.dumps(data)},
+ )
+
+
+def _insert_deadline(conn, callback_id: str, dagrun_id: int):
+ conn.execute(
+ sa.text("INSERT INTO deadline (id, callback_id, dagrun_id) VALUES
(:id, :callback_id, :dagrun_id)"),
+ {"id": uuid.uuid4().hex, "callback_id": callback_id, "dagrun_id":
dagrun_id},
+ )
+
+
+def test_backfill_dagrun_id_from_legacy_callback_data():
+ engine = _make_engine()
+ extended_id = uuid.uuid4().hex
+ plain_id = uuid.uuid4().hex
+ deadline_id = uuid.uuid4().hex
+ invalid_id = uuid.uuid4().hex
+ triggerer_id = uuid.uuid4().hex
+ malformed_id = uuid.uuid4().hex
+
+ with engine.begin() as conn:
+ _create_tables(conn)
+ conn.execute(sa.text("INSERT INTO dag_run (id) VALUES (1), (2)"))
+ _insert_callback(conn, extended_id, {"__var": {"dag_run_id": "1"},
"__type": "dict"})
+ _insert_callback(conn, plain_id, {"dag_run_id": 2})
+ _insert_callback(conn, deadline_id, {"path":
"callback.without.dagrun.id"})
+ _insert_deadline(conn, deadline_id, 1)
+ _insert_callback(conn, invalid_id, {"__var": {"dag_run_id":
"missing"}, "__type": "dict"})
+ _insert_callback(conn, triggerer_id, {"__var": {"dag_run_id": "1"},
"__type": "dict"}, "triggerer")
+ conn.execute(
+ sa.text("INSERT INTO callback (id, type, data) VALUES (:id, :type,
:data)"),
+ {"id": malformed_id, "type": "executor", "data": "not-json"},
+ )
+
+ _migration._backfill_dagrun_id_from_deadline(conn, batch_size=2)
+ _migration._backfill_dagrun_id(conn, batch_size=2)
+
+ rows = conn.execute(sa.text("SELECT id, dagrun_id FROM
callback")).mappings().all()
+
+ dagrun_id_by_callback_id = {row["id"]: row["dagrun_id"] for row in rows}
+ assert dagrun_id_by_callback_id[extended_id] == 1
+ assert dagrun_id_by_callback_id[plain_id] == 2
+ assert dagrun_id_by_callback_id[deadline_id] == 1
+ assert dagrun_id_by_callback_id[invalid_id] is None
+ missing_dagrun_id = uuid.uuid4().hex
+ with engine.begin() as conn:
+ _insert_callback(conn, missing_dagrun_id, {"dag_run_id": 999})
+ _migration._backfill_dagrun_id(conn, batch_size=2)
+ assert (
+ conn.scalar(sa.text("SELECT dagrun_id FROM callback WHERE id =
:id"), {"id": missing_dagrun_id})
+ is None
+ )
+ assert dagrun_id_by_callback_id[triggerer_id] is None
+ assert dagrun_id_by_callback_id[malformed_id] is None
+
+
+def test_restore_legacy_dagrun_id_for_downgrade():
+ engine = _make_engine()
+ wrapped_id = uuid.uuid4().hex
+ plain_id = uuid.uuid4().hex
+
+ with engine.begin() as conn:
+ _create_tables(conn)
+ _insert_callback(conn, wrapped_id, {"__var": {"path":
"callback.path"}, "__type": "dict"})
+ _insert_callback(conn, plain_id, {"path": "callback.path"})
+ conn.execute(
+ sa.text("UPDATE callback SET dagrun_id = CASE id WHEN :wrapped_id
THEN 1 ELSE 2 END"),
+ {"wrapped_id": wrapped_id},
+ )
+
+ _migration._restore_legacy_dagrun_id(conn, batch_size=1)
+ rows = conn.execute(sa.text("SELECT id, data FROM
callback")).mappings().all()
+
+ data_by_callback_id = {row["id"]: json.loads(row["data"]) for row in rows}
+ assert data_by_callback_id[wrapped_id]["__var"]["dag_run_id"] == "1"
+ assert data_by_callback_id[plain_id]["dag_run_id"] == "2"
+
+
+def test_sqlite_upgrade_and_downgrade_preserve_deadline_callback():
+ engine = _make_engine()
+ callback_id = uuid.uuid4()
+ deadline_id = uuid.uuid4()
+
+ with engine.connect() as conn:
+ conn.execute(sa.text("PRAGMA foreign_keys=ON"))
+ conn.execute(sa.text("CREATE TABLE dag_run (id INTEGER PRIMARY KEY)"))
+ conn.execute(
+ sa.text("CREATE TABLE callback (id TEXT PRIMARY KEY, type TEXT NOT
NULL, data TEXT NOT NULL)")
+ )
+ conn.execute(
+ sa.text(
+ "CREATE TABLE deadline ("
+ "id TEXT PRIMARY KEY, "
+ "callback_id TEXT NOT NULL REFERENCES callback(id) ON DELETE
CASCADE, "
+ "dagrun_id INTEGER"
+ ")"
+ )
+ )
+ conn.execute(sa.text("INSERT INTO dag_run (id) VALUES (1)"))
+ conn.execute(
+ sa.text("INSERT INTO callback (id, type, data) VALUES (:id, :type,
:data)"),
+ {"id": str(callback_id), "type": "executor", "data":
json.dumps({"dag_run_id": "1"})},
+ )
+ conn.execute(
+ sa.text("INSERT INTO deadline (id, callback_id, dagrun_id) VALUES
(:id, :callback_id, 1)"),
+ {"id": str(deadline_id), "callback_id": str(callback_id)},
+ )
Review Comment:
`sa.Uuid()` binds `.hex` on SQLite, so with dashed ids every backfill UPDATE
matches 0 rows.
```suggestion
conn.execute(
sa.text("INSERT INTO callback (id, type, data) VALUES (:id,
:type, :data)"),
{"id": callback_id.hex, "type": "executor", "data":
json.dumps({"dag_run_id": "1"})},
)
conn.execute(
sa.text("INSERT INTO deadline (id, callback_id, dagrun_id)
VALUES (:id, :callback_id, 1)"),
{"id": str(deadline_id), "callback_id": callback_id.hex},
)
```
##########
airflow-core/tests/unit/models/test_callback.py:
##########
@@ -498,6 +499,7 @@ def test_polymorphic_serde(self, session):
assert retrieved.priority_weight == 1
assert retrieved.created_at is not None
assert retrieved.trigger_id is None
+ assert retrieved.dagrun_id is None
Review Comment:
No test deletes a `DagRun` and checks its executor callback goes with it.
Could we add something like this?
```python
def test_executor_callback_is_deleted_with_dag_run(dag_maker, session):
with dag_maker("cb_cascade"):
pass
dag_run = dag_maker.create_dagrun()
callback = ExecutorCallback(TEST_SYNC_CALLBACK,
fetch_method=CallbackFetchMethod.IMPORT_PATH)
callback.dagrun_id = dag_run.id
session.add(callback)
session.flush()
callback_id = callback.id
session.expunge(callback)
session.delete(dag_run)
session.flush()
assert session.get(Callback, callback_id) is None
```
##########
airflow-core/src/airflow/migrations/versions/0158_3_5_0_add_dagrun_id_to_callback.py:
##########
@@ -0,0 +1,217 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Add dagrun_id foreign key to callback.
+
+Revision ID: 0f4c9a2b8e1d
+Revises: c64b10719069
+Create Date: 2026-11-11 00:00:00.000000
+
+"""
+
+from __future__ import annotations
+
+import json
+
+import sqlalchemy as sa
+from alembic import context, op
+
+from airflow.configuration import conf
+from airflow.migrations.utils import disable_sqlite_fkeys
+
+# revision identifiers, used by Alembic.
+revision = "0f4c9a2b8e1d"
+down_revision = "c64b10719069"
+branch_labels = None
+depends_on = None
+airflow_version = "3.5.0"
+
+
+def _deserialize_callback_data(data):
+ if isinstance(data, str):
+ try:
+ data = json.loads(data)
+ except json.JSONDecodeError:
+ return None
+ if not isinstance(data, dict):
+ return None
+ callback_data = data.get("__var", data)
+ return (data, callback_data) if isinstance(callback_data, dict) else None
+
+
+def _extract_dagrun_id(data) -> int | None:
+ deserialized_data = _deserialize_callback_data(data)
+ if deserialized_data is None:
+ return None
+ _, callback_data = deserialized_data
+ dagrun_id = callback_data.get("dag_run_id")
+ if dagrun_id is None:
+ return None
+ try:
+ return int(dagrun_id)
+ except (TypeError, ValueError):
+ return None
+
+
+def _backfill_dagrun_id_from_deadline(conn, batch_size: int) -> None:
+ callback = sa.table(
+ "callback",
+ sa.column("id", sa.Uuid()),
+ sa.column("type", sa.String(20)),
+ sa.column("dagrun_id", sa.Integer()),
+ )
+ deadline = sa.table(
+ "deadline",
+ sa.column("callback_id", sa.Uuid()),
+ sa.column("dagrun_id", sa.Integer()),
+ )
+ dag_run = sa.table("dag_run", sa.column("id", sa.Integer()))
+
+ last_id = None
+ while True:
+ query = (
+ sa.select(callback.c.id, deadline.c.dagrun_id)
+ .select_from(
+ callback.join(deadline, callback.c.id ==
deadline.c.callback_id).join(
+ dag_run, deadline.c.dagrun_id == dag_run.c.id
+ )
+ )
+ .where(callback.c.type == "executor",
callback.c.dagrun_id.is_(None))
+ .order_by(callback.c.id)
+ .limit(batch_size)
+ )
+ if last_id is not None:
+ query = query.where(callback.c.id > last_id)
+
+ rows = conn.execute(query).fetchall()
+ if not rows:
+ return
+
+ for callback_id, dagrun_id in rows:
+ conn.execute(callback.update().where(callback.c.id ==
callback_id).values(dagrun_id=dagrun_id))
+
+ last_id = rows[-1].id
+
+
+def _backfill_dagrun_id(conn, batch_size: int) -> None:
+ callback = sa.table(
+ "callback",
+ sa.column("id", sa.Uuid()),
+ sa.column("type", sa.String(20)),
+ sa.column("data", sa.Text()),
+ sa.column("dagrun_id", sa.Integer()),
+ )
+ dag_run = sa.table("dag_run", sa.column("id", sa.Integer()))
+
+ last_id = None
+ while True:
+ query = (
+ sa.select(callback.c.id, callback.c.data)
+ .where(callback.c.type == "executor",
callback.c.dagrun_id.is_(None))
+ .order_by(callback.c.id)
+ .limit(batch_size)
+ )
+ if last_id is not None:
+ query = query.where(callback.c.id > last_id)
+
+ rows = conn.execute(query).fetchall()
+ if not rows:
+ return
+
+ for callback_id, data in rows:
+ dagrun_id = _extract_dagrun_id(data)
+ if dagrun_id is None:
+ continue
+
+ dagrun_exists = conn.execute(
+ sa.select(dag_run.c.id).where(dag_run.c.id ==
dagrun_id).limit(1)
+ ).first()
+ if dagrun_exists:
+ conn.execute(
+ callback.update().where(callback.c.id ==
callback_id).values(dagrun_id=dagrun_id)
+ )
+
+ last_id = rows[-1].id
+
+
+def _restore_legacy_dagrun_id(conn, batch_size: int) -> None:
+ callback = sa.table(
+ "callback",
+ sa.column("id", sa.Uuid()),
+ sa.column("type", sa.String(20)),
+ sa.column("data", sa.Text()),
+ sa.column("dagrun_id", sa.Integer()),
+ )
+
+ last_id = None
+ while True:
+ query = (
+ sa.select(callback.c.id, callback.c.data, callback.c.dagrun_id)
+ .where(callback.c.type == "executor",
callback.c.dagrun_id.is_not(None))
+ .order_by(callback.c.id)
+ .limit(batch_size)
+ )
+ if last_id is not None:
+ query = query.where(callback.c.id > last_id)
+
+ rows = conn.execute(query).fetchall()
+ if not rows:
+ return
+
+ for callback_id, data, dagrun_id in rows:
+ deserialized_data = _deserialize_callback_data(data)
+ if deserialized_data is None:
+ continue
+ serialized_data, callback_data = deserialized_data
+ callback_data["dag_run_id"] = str(dagrun_id)
+ conn.execute(
+ callback.update().where(callback.c.id ==
callback_id).values(data=json.dumps(serialized_data))
+ )
Review Comment:
With psycopg 3 this binds `data` as `::VARCHAR`, and PostgreSQL rejects that
for the `jsonb` column, so the downgrade fails. Write through a `sa.JSON()`
column instead?
```python
callback_json = sa.table("callback", sa.column("id", sa.Uuid()),
sa.column("data", sa.JSON()))
```
```suggestion
conn.execute(
callback_json.update().where(callback_json.c.id ==
callback_id).values(data=serialized_data)
)
```
A `db_test` that runs the real downgrade, like
`test_0153_add_index_on_callback_trigger_id.py`, would cover PostgreSQL and
MySQL.
---
Drafted-by: Claude Code (Opus 5.5); reviewed by @jason810496 before posting
##########
airflow-core/tests/unit/migrations/test_0158_callback_dagrun_id_migration.py:
##########
@@ -0,0 +1,206 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import uuid
+from pathlib import Path
+from unittest.mock import patch
+
+import sqlalchemy as sa
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+
+from tests_common.test_utils.paths import AIRFLOW_CORE_SOURCES_PATH
+
+_MIGRATION_PATH = (
+ Path(AIRFLOW_CORE_SOURCES_PATH)
+ / "airflow"
+ / "migrations"
+ / "versions"
+ / "0158_3_5_0_add_dagrun_id_to_callback.py"
+)
+
+_spec = importlib.util.spec_from_file_location("migration_0158",
_MIGRATION_PATH)
+assert _spec is not None
+_migration = importlib.util.module_from_spec(_spec)
+assert _spec.loader is not None
+_spec.loader.exec_module(_migration)
+
+
+def _make_engine():
+ return sa.create_engine("sqlite:///:memory:")
+
+
+def _create_tables(conn):
+ conn.execute(sa.text("CREATE TABLE dag_run (id INTEGER PRIMARY KEY)"))
+ conn.execute(
+ sa.text(
+ """
+ CREATE TABLE callback (
+ id TEXT PRIMARY KEY,
+ type TEXT NOT NULL,
+ data TEXT NOT NULL,
+ dagrun_id INTEGER
+ )
+ """
+ )
+ )
+ conn.execute(
+ sa.text(
+ """
+ CREATE TABLE deadline (
+ id TEXT PRIMARY KEY,
+ callback_id TEXT NOT NULL,
+ dagrun_id INTEGER
+ )
+ """
+ )
+ )
+
+
+def _insert_callback(conn, callback_id: str, data: dict, callback_type: str =
"executor"):
+ conn.execute(
+ sa.text("INSERT INTO callback (id, type, data) VALUES (:id, :type,
:data)"),
+ {"id": callback_id, "type": callback_type, "data": json.dumps(data)},
+ )
+
+
+def _insert_deadline(conn, callback_id: str, dagrun_id: int):
+ conn.execute(
+ sa.text("INSERT INTO deadline (id, callback_id, dagrun_id) VALUES
(:id, :callback_id, :dagrun_id)"),
+ {"id": uuid.uuid4().hex, "callback_id": callback_id, "dagrun_id":
dagrun_id},
+ )
+
+
+def test_backfill_dagrun_id_from_legacy_callback_data():
+ engine = _make_engine()
+ extended_id = uuid.uuid4().hex
+ plain_id = uuid.uuid4().hex
+ deadline_id = uuid.uuid4().hex
+ invalid_id = uuid.uuid4().hex
+ triggerer_id = uuid.uuid4().hex
+ malformed_id = uuid.uuid4().hex
+
+ with engine.begin() as conn:
+ _create_tables(conn)
+ conn.execute(sa.text("INSERT INTO dag_run (id) VALUES (1), (2)"))
+ _insert_callback(conn, extended_id, {"__var": {"dag_run_id": "1"},
"__type": "dict"})
+ _insert_callback(conn, plain_id, {"dag_run_id": 2})
+ _insert_callback(conn, deadline_id, {"path":
"callback.without.dagrun.id"})
+ _insert_deadline(conn, deadline_id, 1)
+ _insert_callback(conn, invalid_id, {"__var": {"dag_run_id":
"missing"}, "__type": "dict"})
+ _insert_callback(conn, triggerer_id, {"__var": {"dag_run_id": "1"},
"__type": "dict"}, "triggerer")
+ conn.execute(
+ sa.text("INSERT INTO callback (id, type, data) VALUES (:id, :type,
:data)"),
+ {"id": malformed_id, "type": "executor", "data": "not-json"},
+ )
+
+ _migration._backfill_dagrun_id_from_deadline(conn, batch_size=2)
+ _migration._backfill_dagrun_id(conn, batch_size=2)
+
+ rows = conn.execute(sa.text("SELECT id, dagrun_id FROM
callback")).mappings().all()
+
+ dagrun_id_by_callback_id = {row["id"]: row["dagrun_id"] for row in rows}
+ assert dagrun_id_by_callback_id[extended_id] == 1
+ assert dagrun_id_by_callback_id[plain_id] == 2
+ assert dagrun_id_by_callback_id[deadline_id] == 1
+ assert dagrun_id_by_callback_id[invalid_id] is None
+ missing_dagrun_id = uuid.uuid4().hex
+ with engine.begin() as conn:
+ _insert_callback(conn, missing_dagrun_id, {"dag_run_id": 999})
+ _migration._backfill_dagrun_id(conn, batch_size=2)
+ assert (
+ conn.scalar(sa.text("SELECT dagrun_id FROM callback WHERE id =
:id"), {"id": missing_dagrun_id})
+ is None
+ )
+ assert dagrun_id_by_callback_id[triggerer_id] is None
+ assert dagrun_id_by_callback_id[malformed_id] is None
+
+
+def test_restore_legacy_dagrun_id_for_downgrade():
+ engine = _make_engine()
+ wrapped_id = uuid.uuid4().hex
+ plain_id = uuid.uuid4().hex
+
+ with engine.begin() as conn:
+ _create_tables(conn)
+ _insert_callback(conn, wrapped_id, {"__var": {"path":
"callback.path"}, "__type": "dict"})
+ _insert_callback(conn, plain_id, {"path": "callback.path"})
+ conn.execute(
+ sa.text("UPDATE callback SET dagrun_id = CASE id WHEN :wrapped_id
THEN 1 ELSE 2 END"),
+ {"wrapped_id": wrapped_id},
+ )
+
+ _migration._restore_legacy_dagrun_id(conn, batch_size=1)
+ rows = conn.execute(sa.text("SELECT id, data FROM
callback")).mappings().all()
+
+ data_by_callback_id = {row["id"]: json.loads(row["data"]) for row in rows}
+ assert data_by_callback_id[wrapped_id]["__var"]["dag_run_id"] == "1"
+ assert data_by_callback_id[plain_id]["dag_run_id"] == "2"
+
+
+def test_sqlite_upgrade_and_downgrade_preserve_deadline_callback():
+ engine = _make_engine()
+ callback_id = uuid.uuid4()
+ deadline_id = uuid.uuid4()
+
+ with engine.connect() as conn:
+ conn.execute(sa.text("PRAGMA foreign_keys=ON"))
+ conn.execute(sa.text("CREATE TABLE dag_run (id INTEGER PRIMARY KEY)"))
+ conn.execute(
+ sa.text("CREATE TABLE callback (id TEXT PRIMARY KEY, type TEXT NOT
NULL, data TEXT NOT NULL)")
+ )
+ conn.execute(
+ sa.text(
+ "CREATE TABLE deadline ("
+ "id TEXT PRIMARY KEY, "
+ "callback_id TEXT NOT NULL REFERENCES callback(id) ON DELETE
CASCADE, "
+ "dagrun_id INTEGER"
+ ")"
+ )
+ )
+ conn.execute(sa.text("INSERT INTO dag_run (id) VALUES (1)"))
+ conn.execute(
+ sa.text("INSERT INTO callback (id, type, data) VALUES (:id, :type,
:data)"),
+ {"id": str(callback_id), "type": "executor", "data":
json.dumps({"dag_run_id": "1"})},
+ )
+ conn.execute(
+ sa.text("INSERT INTO deadline (id, callback_id, dagrun_id) VALUES
(:id, :callback_id, 1)"),
+ {"id": str(deadline_id), "callback_id": str(callback_id)},
+ )
+ conn.commit()
+
+ operations = Operations(MigrationContext.configure(conn))
+ with (
+ patch.object(_migration, "op", operations),
+ patch.object(_migration.context, "is_offline_mode",
return_value=False),
+ ):
+ _migration.upgrade()
+ assert (
+ conn.scalar(sa.text("SELECT COUNT(*) FROM deadline WHERE id =
:id"), {"id": str(deadline_id)})
+ == 1
+ )
Review Comment:
This passes even with a no-op backfill. Could we assert the backfilled value
too?
```suggestion
assert (
conn.scalar(sa.text("SELECT COUNT(*) FROM deadline WHERE id
= :id"), {"id": str(deadline_id)})
== 1
)
assert conn.scalar(sa.text("SELECT dagrun_id FROM callback")) ==
1
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]