aminghadersohi commented on code in PR #44926:
URL: https://github.com/apache/superset/pull/44926#discussion_r4203571540
##########
superset/commands/deletion_retention/purge_policy.py:
##########
@@ -1015,6 +1019,32 @@ def delete_associations(
phase=ExecutionPhase.ASSOCIATIONS,
permission_name=None,
)
+ if policy.entity_type == "dataset":
+ _detach_dataset_charts(session, entity_id)
+
+
+def _detach_dataset_charts(session: Session, entity_id: int) -> None:
+ """Clear the datasource reference and permission fields of a purged
+ dataset's charts, as an ORM hard delete of the dataset leaves them.
+
+ A chart kept pointing at the purged id would keep the dataset's stale
+ permission, and would adopt any later dataset that reuses the id. Only
+ ``slices`` rows are updated; no chart is deleted. Runs after the
+ dangling-chart impact snapshot, inside the purge's transaction.
+ """
+ # avoid circular import: model listener registration imports neutral event
helpers
+ from superset.models.slice import Slice
+ from superset.utils.core import DatasourceType
+
+ slices: sa.Table = Slice.__table__ # pylint: disable=no-member
+ session.execute(
+ sa.update(slices)
+ .where(
+ slices.c.datasource_type == DatasourceType.TABLE.value,
+ slices.c.datasource_id == entity_id,
+ )
+ .values(datasource_id=None, perm=None, schema_perm=None,
catalog_perm=None)
+ )
Review Comment:
Lost-update half doesn't hold: the ORM UPDATE carries only changed columns,
so a save of a matched chart waits on the row lock and the NULLs stand (checked
on Postgres 16). The phantom half holds, but an ORM dataset delete has the same
window and pre-PR every chart stayed dangling: not a regression.
##########
tests/unit_tests/commands/deletion_retention/test_purge_chart_detach.py:
##########
@@ -0,0 +1,211 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""SC-119912: purging a dataset detaches the charts that reference it."""
+
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+from unittest.mock import patch
+
+import pytest
+from sqlalchemy import create_engine, event
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm.session import Session
+
+from superset.commands.deletion_retention import purge_cascade
+from superset.commands.deletion_retention.purge_cascade import (
+ cascade_hard_delete,
+ CascadeResult,
+)
+from superset.connectors.sqla.models import SqlaTable
+from superset.models.core import Database
+from superset.models.slice import Slice
+from superset.semantic_layers.models import SemanticLayer, SemanticView
+
+
[email protected]
+def session_engine(tmp_path: Path) -> Engine:
+ """File-backed SQLite with working SAVEPOINTs, as the purge needs.
+
+ The purge's ``begin_nested`` needs real savepoints, which pysqlite's
+ implicit transaction handling breaks; SQLAlchemy's documented workaround
+ disables it and emits ``BEGIN``. A file database gives the purge's
+ ``has_table`` check its own connection instead of the test's.
+ """
+ engine: Engine = create_engine(f"sqlite:///{tmp_path / 'purge.db'}")
+
+ @event.listens_for(engine, "connect")
+ def _disable_pysqlite_transactions(dbapi_connection: Any, _: Any) -> None:
+ dbapi_connection.isolation_level = None
+
+ @event.listens_for(engine, "begin")
+ def _emit_begin(connection: Any) -> None:
+ connection.exec_driver_sql("BEGIN")
+
+ return engine
+
+
+def _trashed_dataset_with_charts(
+ session: Session,
+) -> tuple[SqlaTable, Slice, Slice, Database]:
+ """A soft-deleted dataset with one chart on it, and a semantic-view chart
+ whose datasource id equals the dataset's."""
+ SqlaTable.metadata.create_all(session.get_bind()) # pylint:
disable=no-member
+ database: Database = Database(database_name="examples",
sqlalchemy_uri="sqlite://")
+ dataset: SqlaTable = SqlaTable(
+ table_name="purged", database=database, schema="public"
+ )
+ session.add(dataset)
+ session.flush()
+ layer: SemanticLayer = SemanticLayer(name="layer", type="test",
configuration="{}")
+ session.add(layer)
+ session.flush()
+ view: SemanticView = SemanticView(
+ id=dataset.id, name="view", semantic_layer_uuid=layer.uuid,
configuration="{}"
+ )
+ session.add(view)
+ session.flush()
+ orphan: Slice = Slice(
+ slice_name="orphan",
+ datasource_type="table",
+ datasource_id=dataset.id,
+ viz_type="table",
+ )
+ other: Slice = Slice(
+ slice_name="semantic",
+ datasource_type="semantic_view",
+ datasource_id=view.id,
+ viz_type="table",
+ )
+ session.add_all([orphan, other])
+ session.flush()
+ assert orphan.perm == dataset.perm
+ assert other.perm == view.perm
+ dataset.deleted_at = datetime(2020, 1, 1)
+ # Production commits before the cascade (the task writes an audit row).
+ session.commit()
+ return dataset, orphan, other, database
+
+
+def _purge(session: Session, dataset: SqlaTable) -> CascadeResult:
+ result: CascadeResult = cascade_hard_delete(
+ session, dataset, enforce_window=False, require_archived=True
+ )
+ session.commit()
+ session.expire_all()
+ return result
+
+
+def test_purge_detaches_dataset_charts(session: Session, app_context: None) ->
None:
+ """The purged dataset's chart is kept with its datasource id and
+ permission fields cleared; a chart on another datasource is untouched."""
+ dataset: SqlaTable
+ orphan: Slice
+ other: Slice
+ dataset, orphan, other, _ = _trashed_dataset_with_charts(session)
+ other_before: tuple[Any, ...] = (
+ other.datasource_type,
+ other.datasource_id,
+ other.perm,
+ )
+
+ assert _purge(session, dataset).purged
Review Comment:
Dropping `slices.c.datasource_id == entity_id` from the detach UPDATE keeps
this suite green, since the fixture has only one table chart, yet it would
detach every table chart in the instance. A bystander chart pins it:
```suggestion
bystander_id: int = dataset.id + 1
bystander: Slice = Slice(
slice_name="bystander", datasource_type="table",
datasource_id=bystander_id
)
session.add(bystander)
session.commit()
assert _purge(session, dataset).purged
assert session.get(Slice, bystander.id).datasource_id == bystander_id
```
##########
tests/unit_tests/commands/deletion_retention/test_purge_chart_detach.py:
##########
@@ -0,0 +1,211 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""SC-119912: purging a dataset detaches the charts that reference it."""
+
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+from unittest.mock import patch
+
+import pytest
+from sqlalchemy import create_engine, event
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm.session import Session
+
+from superset.commands.deletion_retention import purge_cascade
+from superset.commands.deletion_retention.purge_cascade import (
+ cascade_hard_delete,
+ CascadeResult,
+)
+from superset.connectors.sqla.models import SqlaTable
+from superset.models.core import Database
+from superset.models.slice import Slice
+from superset.semantic_layers.models import SemanticLayer, SemanticView
+
+
[email protected]
+def session_engine(tmp_path: Path) -> Engine:
+ """File-backed SQLite with working SAVEPOINTs, as the purge needs.
+
+ The purge's ``begin_nested`` needs real savepoints, which pysqlite's
+ implicit transaction handling breaks; SQLAlchemy's documented workaround
+ disables it and emits ``BEGIN``. A file database gives the purge's
+ ``has_table`` check its own connection instead of the test's.
+ """
+ engine: Engine = create_engine(f"sqlite:///{tmp_path / 'purge.db'}")
+
+ @event.listens_for(engine, "connect")
+ def _disable_pysqlite_transactions(dbapi_connection: Any, _: Any) -> None:
+ dbapi_connection.isolation_level = None
+
+ @event.listens_for(engine, "begin")
+ def _emit_begin(connection: Any) -> None:
+ connection.exec_driver_sql("BEGIN")
+
+ return engine
+
+
+def _trashed_dataset_with_charts(
+ session: Session,
+) -> tuple[SqlaTable, Slice, Slice, Database]:
+ """A soft-deleted dataset with one chart on it, and a semantic-view chart
+ whose datasource id equals the dataset's."""
+ SqlaTable.metadata.create_all(session.get_bind()) # pylint:
disable=no-member
+ database: Database = Database(database_name="examples",
sqlalchemy_uri="sqlite://")
+ dataset: SqlaTable = SqlaTable(
+ table_name="purged", database=database, schema="public"
+ )
+ session.add(dataset)
+ session.flush()
+ layer: SemanticLayer = SemanticLayer(name="layer", type="test",
configuration="{}")
+ session.add(layer)
+ session.flush()
+ view: SemanticView = SemanticView(
+ id=dataset.id, name="view", semantic_layer_uuid=layer.uuid,
configuration="{}"
+ )
+ session.add(view)
+ session.flush()
+ orphan: Slice = Slice(
+ slice_name="orphan",
+ datasource_type="table",
+ datasource_id=dataset.id,
+ viz_type="table",
+ )
+ other: Slice = Slice(
+ slice_name="semantic",
+ datasource_type="semantic_view",
+ datasource_id=view.id,
+ viz_type="table",
+ )
+ session.add_all([orphan, other])
+ session.flush()
+ assert orphan.perm == dataset.perm
+ assert other.perm == view.perm
+ dataset.deleted_at = datetime(2020, 1, 1)
+ # Production commits before the cascade (the task writes an audit row).
+ session.commit()
+ return dataset, orphan, other, database
+
+
+def _purge(session: Session, dataset: SqlaTable) -> CascadeResult:
+ result: CascadeResult = cascade_hard_delete(
+ session, dataset, enforce_window=False, require_archived=True
+ )
+ session.commit()
+ session.expire_all()
+ return result
+
+
+def test_purge_detaches_dataset_charts(session: Session, app_context: None) ->
None:
+ """The purged dataset's chart is kept with its datasource id and
+ permission fields cleared; a chart on another datasource is untouched."""
+ dataset: SqlaTable
+ orphan: Slice
+ other: Slice
+ dataset, orphan, other, _ = _trashed_dataset_with_charts(session)
+ other_before: tuple[Any, ...] = (
+ other.datasource_type,
+ other.datasource_id,
+ other.perm,
+ )
+
+ assert _purge(session, dataset).purged
+
+ kept: Slice = session.get(Slice, orphan.id)
+ assert (kept.datasource_id, kept.perm, kept.schema_perm,
kept.catalog_perm) == (
+ None,
+ None,
+ None,
+ None,
+ )
+ untouched: Slice = session.get(Slice, other.id)
+ assert (
+ untouched.datasource_type,
+ untouched.datasource_id,
+ untouched.perm,
+ ) == other_before
+
+
+def test_purged_dataset_id_reuse_cannot_rebind_chart(
+ session: Session, app_context: None
+) -> None:
+ """A new dataset that reuses the purged id does not adopt the chart."""
+ dataset: SqlaTable
+ orphan: Slice
+ database: Database
+ dataset, orphan, _, database = _trashed_dataset_with_charts(session)
+ dataset_id: int = dataset.id
+ assert _purge(session, dataset).purged
+ stranger: SqlaTable = SqlaTable(
+ id=dataset_id, table_name="stranger", database=database,
schema="public"
+ )
+ session.add(stranger)
+ session.flush()
+
+ chart: Slice = session.get(Slice, orphan.id)
+ chart.slice_name = "orphan renamed"
+ session.flush()
+
+ assert chart.datasource_id is None
+ assert chart.perm is None
+ assert chart.resolved_datasource is None
+
+
+def test_failed_purge_rolls_back_chart_detach(
+ session: Session, app_context: None
+) -> None:
+ """A failure later in the cascade leaves the chart and dataset
unchanged."""
+ dataset: SqlaTable
+ orphan: Slice
+ dataset, orphan, _, _ = _trashed_dataset_with_charts(session)
+ before: tuple[Any, ...] = (orphan.datasource_id, orphan.perm,
orphan.schema_perm)
+
+ with (
+ patch.object(
+ purge_cascade,
+ "_delete_version_history",
+ side_effect=RuntimeError("boom"),
+ ),
+ pytest.raises(RuntimeError, match="boom"),
+ ):
+ cascade_hard_delete(
+ session, dataset, enforce_window=False, require_archived=True
+ )
+ session.rollback()
+ session.expire_all()
+
+ chart: Slice = session.get(Slice, orphan.id)
+ assert (chart.datasource_id, chart.perm, chart.schema_perm) == before
+ assert session.get(SqlaTable, dataset.id) is not None
+
+
+def test_dry_run_purge_changes_no_chart(session: Session, app_context: None)
-> None:
+ """A dry run counts eligible datasets and never touches their charts."""
+ from superset.tasks.deletion_retention import _purge_model
+
+ dataset: SqlaTable
+ orphan: Slice
+ dataset, orphan, _, _ = _trashed_dataset_with_charts(session)
+ before: tuple[Any, ...] = (orphan.datasource_id, orphan.perm,
orphan.schema_perm)
+
+ _, would_purge, _, _ = _purge_model(SqlaTable, datetime(2021, 1, 1), True)
+ session.expire_all()
+
+ assert would_purge == 1
Review Comment:
This still passes with the dry-run `continue` removed: the purge attempt
dies on the missing audit sentinel (`failures == 1`), so the chart is untouched
either way. Asserting the counters makes it bite:
```suggestion
purged, would_purge, failures, _ = _purge_model(
SqlaTable, datetime(2021, 1, 1), True
)
session.expire_all()
assert (purged, would_purge, failures) == (0, 1, 0)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
