mikebridge commented on code in PR #41549: URL: https://github.com/apache/superset/pull/41549#discussion_r3658926094
########## tests/integration_tests/deletion_retention/_base.py: ########## @@ -0,0 +1,228 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Shared base + self-contained builders for deletion-retention tests. + +These tests do not depend on the example datasets — each builds its own +``Database`` + ``SqlaTable`` so they run on a bare (schema-only) test DB. +Everything created is torn down (bypassing the soft-delete visibility +filter) so a leftover soft-deleted row never trips a later test. +""" + +from __future__ import annotations + +from datetime import datetime, timedelta +from typing import Any + +import sqlalchemy as sa + +from superset import db +from superset.connectors.sqla.models import SqlaTable, SqlMetric, TableColumn +from superset.constants import SKIP_VISIBILITY_FILTER_CLASSES +from superset.models.core import Database +from superset.models.dashboard import Dashboard +from superset.models.slice import Slice +from tests.integration_tests.base_tests import SupersetTestCase + +_PREFIX = "retention_it_" + + +def _bypass(model: type[Any]) -> dict[str, Any]: + return {"execution_options": {SKIP_VISIBILITY_FILTER_CLASSES: {model}}} + + +class DeletionRetentionTestBase(SupersetTestCase): + """Builds an isolated database + dataset and cleans up after itself.""" + + def setUp(self) -> None: + super().setUp() + self._cleanup() + self.database: Database = Database( + database_name=f"{_PREFIX}db", sqlalchemy_uri="sqlite://" + ) Review Comment: CI runs these integration tests serially (no xdist), and the fixed name mirrors the existing integration-suite pattern; making the name unique per instance would leak databases on hard crashes instead of converging on one. ########## tests/integration_tests/deletion_retention/_base.py: ########## @@ -0,0 +1,228 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Shared base + self-contained builders for deletion-retention tests. + +These tests do not depend on the example datasets — each builds its own +``Database`` + ``SqlaTable`` so they run on a bare (schema-only) test DB. +Everything created is torn down (bypassing the soft-delete visibility +filter) so a leftover soft-deleted row never trips a later test. +""" + +from __future__ import annotations + +from datetime import datetime, timedelta +from typing import Any + +import sqlalchemy as sa + +from superset import db +from superset.connectors.sqla.models import SqlaTable, SqlMetric, TableColumn +from superset.constants import SKIP_VISIBILITY_FILTER_CLASSES +from superset.models.core import Database +from superset.models.dashboard import Dashboard +from superset.models.slice import Slice +from tests.integration_tests.base_tests import SupersetTestCase + +_PREFIX = "retention_it_" + + +def _bypass(model: type[Any]) -> dict[str, Any]: + return {"execution_options": {SKIP_VISIBILITY_FILTER_CLASSES: {model}}} + + +class DeletionRetentionTestBase(SupersetTestCase): + """Builds an isolated database + dataset and cleans up after itself.""" + + def setUp(self) -> None: + super().setUp() + self._cleanup() + self.database: Database = Database( + database_name=f"{_PREFIX}db", sqlalchemy_uri="sqlite://" + ) + db.session.add(self.database) + db.session.commit() + self.dataset: SqlaTable = self.make_dataset("ds") + + def tearDown(self) -> None: + self._cleanup() + super().tearDown() + + # -- builders ----------------------------------------------------------- + + def make_dataset(self, name: str, with_children: bool = False) -> SqlaTable: + ds = SqlaTable(table_name=f"{_PREFIX}{name}", database=self.database) + db.session.add(ds) + db.session.commit() + if with_children: + db.session.add(TableColumn(column_name=f"{_PREFIX}col", table=ds)) + db.session.add( + SqlMetric( + metric_name=f"{_PREFIX}metric", expression="count(*)", table=ds + ) + ) + db.session.commit() + return ds + + def make_chart(self, name: str, dataset: SqlaTable | None = None) -> Slice: + dataset = dataset or self.dataset + chart = Slice( + slice_name=f"{_PREFIX}{name}", + datasource_type="table", + datasource_id=dataset.id, + viz_type="table", + ) + db.session.add(chart) + db.session.commit() + return chart + + def make_dashboard(self, name: str, slices: list[Slice] | None = None) -> Dashboard: + dash = Dashboard( + dashboard_title=f"{_PREFIX}{name}", + slug=f"{_PREFIX}{name}", + slices=slices or [], + ) + db.session.add(dash) + db.session.commit() + return dash + + def soft_delete(self, entity: Any, days_ago: int) -> None: + """Mark *entity* soft-deleted with a backdated ``deleted_at``.""" + entity.deleted_at = datetime.now() - timedelta(days=days_ago) + db.session.add(entity) + db.session.commit() + + # -- assertions / lookups ---------------------------------------------- + + def exists(self, model: type[Any], entity_id: int) -> bool: + row = ( + db.session.query(model) + .execution_options(**{SKIP_VISIBILITY_FILTER_CLASSES: {model}}) + .filter(model.id == entity_id) + .one_or_none() + ) + return row is not None + + def count(self, sql: str, params: dict[str, Any]) -> int: + return db.session.execute(sa.text(sql), params).scalar() or 0 + + # -- version-history forging ------------------------------------------- + + def forge_version_row(self, model: type[Any], entity_id: int, tx_id: int) -> None: + """Insert a version_transaction + parent shadow + version_changes row + for *entity_id* anchored at *tx_id* (so a purge has history to remove + without needing live capture to be enabled).""" + from superset.versioning.changes import ENTITY_KIND_BY_CLASS_NAME + + shadow = { + Slice: "slices_version", + Dashboard: "dashboards_version", + SqlaTable: "tables_version", + }[model] + kind = ENTITY_KIND_BY_CLASS_NAME[model.__name__] + # The transaction may be shared across entities — insert it once. + exists = db.session.execute( + sa.text("SELECT 1 FROM version_transaction WHERE id = :t"), {"t": tx_id} + ).first() + if not exists: + db.session.execute( + sa.text( + "INSERT INTO version_transaction (id, issued_at) VALUES (:t, :ts)" + ), + {"t": tx_id, "ts": datetime.utcnow()}, + ) + db.session.execute( + sa.text( + f"INSERT INTO {shadow} (id, transaction_id, operation_type) " # noqa: S608 + "VALUES (:i, :t, 0)" + ), + {"i": entity_id, "t": tx_id}, + ) + db.session.execute( + sa.text( + "INSERT INTO version_changes " + "(transaction_id, entity_kind, entity_id, sequence, kind, " + "operation, path) VALUES (:t, :k, :i, 1, 'set', 0, :p)" + ), + {"t": tx_id, "k": kind, "i": entity_id, "p": '["x"]'}, + ) + db.session.commit() + + # -- cleanup ------------------------------------------------------------ + + def _cleanup(self) -> None: + db.session.rollback() + from superset.connectors.sqla.models import RowLevelSecurityFilter + from superset.reports.models import ReportSchedule + + for report in db.session.query(ReportSchedule).filter( + ReportSchedule.name.like(f"{_PREFIX}%") + ): + db.session.delete(report) + for rule in db.session.query(RowLevelSecurityFilter).filter( + RowLevelSecurityFilter.name.like(f"{_PREFIX}%") + ): + db.session.delete(rule) + db.session.commit() + for model in (Dashboard, Slice, SqlaTable): + rows = ( + db.session.query(model) + .execution_options(**{SKIP_VISIBILITY_FILTER_CLASSES: {model}}) + .all() + ) + for row in rows: + name = ( + getattr(row, "slice_name", None) + or getattr(row, "dashboard_title", None) + or getattr(row, "table_name", "") + ) + if str(name).startswith(_PREFIX): Review Comment: Same serial-CI rationale as the database-name thread — the suite-scoped prefix keeps cleanup convergent across crashed runs, which per-instance prefixes would leak. ########## tests/unit_tests/tasks/test_deletion_retention.py: ########## @@ -0,0 +1,193 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Unit tests for deletion-retention configuration and window resolution. + +The shared value overrides config, an unset value falls back to config, ``0`` +is preserved as the disable value, and malformed shared values use the fallback. +""" + +import runpy +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from flask.config import Config + + [email protected] +def app_config(app_context: None) -> Config: + from flask import current_app + + current_app.config["SUPERSET_SOFT_DELETE_RETENTION_DAYS"] = 30 + return current_app.config + + +def _resolve() -> int: + from superset.commands.deletion_retention.window import resolve_retention_window + + return resolve_retention_window() + + +def test_unset_falls_back_to_config(app_config: Config) -> None: + with patch( + "superset.commands.deletion_retention.window.get_shared_value", + return_value=None, + ): + assert _resolve() == 30 + + +def test_shared_value_overrides_config(app_config: Config) -> None: + with patch( + "superset.commands.deletion_retention.window.get_shared_value", + return_value=7, + ): + assert _resolve() == 7 + + +def test_zero_shared_value_is_preserved_not_coerced(app_config: Config) -> None: + # `0` is a meaningful "disable"; it must survive (never `or`-coerced to 30). + with patch( + "superset.commands.deletion_retention.window.get_shared_value", + return_value=0, + ): + assert _resolve() == 0 + + +def test_malformed_shared_value_falls_back(app_config: Config) -> None: + for bad in ("oops", -3, True, 1.5): + with patch( + "superset.commands.deletion_retention.window.get_shared_value", + return_value=bad, + ): + assert _resolve() == 30 + + [email protected]("configured", ["oops", -3, True, None]) +def test_malformed_config_value_falls_back( + app_config: Config, configured: object +) -> None: + app_config["SUPERSET_SOFT_DELETE_RETENTION_DAYS"] = configured + with patch( + "superset.commands.deletion_retention.window.get_shared_value", + return_value=None, + ): + assert _resolve() == 30 + + +def test_window_zero_disables_the_task(app_context: None) -> None: + # A zero window short-circuits the purge entirely. + import superset.tasks.deletion_retention as mod + + with patch.object(mod, "_soft_delete_models") as models: + result = mod._purge_impl(0, dry_run=False) + assert result == {"skipped": 1} + models.assert_not_called() + + +def test_clock_uses_now_not_utcnow() -> None: + import superset.tasks.deletion_retention as mod + from superset.models.slice import Slice + + now = datetime(2026, 7, 13, 12, 0) Review Comment: Naive datetimes are the metadata-schema convention (columns are naive UTC), so the fixture deliberately matches the column semantics; a tz-aware value here would diverge from what production code writes. ########## superset/commands/deletion_retention/audit.py: ########## @@ -0,0 +1,225 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Write-ahead purge audit record. + +Every purge — time-based or force — writes an immutable record that +**survives** the entity it names, on a **dedicated session** outside the +purge transaction so it neither entangles with the ``DBEventLogger`` +(which shares ``db.session`` and commits mid-request) nor vanishes if the +purge rolls back. The record is written ``pending`` *before* the purge and +flipped to ``confirmed`` *after* it commits, so a crash leaves at most a +``pending`` row, never a missing one. ``pending`` rows are reconciled on the +next run (the purge is convergent). + +The dedicated ``purge_audit_log`` table is content-free (no name or PII; only +action, actor, UTC time, entity type, UUID, and affected referrers) and is never +removed by the purge cascade. +""" + +from __future__ import annotations + +import logging +from datetime import datetime, timedelta +from typing import Any, cast +from uuid import UUID, uuid4 + +import sqlalchemy as sa +from sqlalchemy import Column, DateTime, Integer, String, Text +from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy_utils import UUIDType + +from superset import db + +logger: logging.Logger = logging.getLogger(__name__) + + +def _dedicated_session() -> Session: + """A fresh session on its own connection, independent of the request / + task ``db.session``. The audit write must commit on its own so it survives + a rolled-back or crashed purge.""" + return sessionmaker(bind=db.engine)() + + +STATUS_PENDING = "pending" +STATUS_CONFIRMED = "confirmed" +STATUS_FAILED = "failed" +STATUS_BLOCKED = "blocked" + +_PENDING_STALE_AFTER = timedelta(hours=1) + +TRIGGER_RETENTION = "retention" +TRIGGER_FORCE = "force" + +ACTOR_SYSTEM = "system" + + +class PurgeAuditLog(db.Model): + """Immutable, content-free record of a purge.""" + + __tablename__ = "purge_audit_log" + + id = Column(UUIDType(binary=True), primary_key=True, default=uuid4) + status = Column(String(16), nullable=False, default=STATUS_PENDING) + trigger = Column(String(16), nullable=False) + actor = Column(String(256), nullable=False) + entity_type = Column(String(64), nullable=False) + entity_uuid = Column(String(36), nullable=True, index=True) + # Comma-joined UUIDs of charts left dangling / dashboards that lost a join + # row (force-purge visibility). Free text, content-free. + affected_referrers = Column(Text, nullable=True) + removed_dashboard_slices = Column(Integer, nullable=False, default=0) + created_on = Column(DateTime, nullable=False) + confirmed_on = Column(DateTime, nullable=True) + + +def write_ahead( + *, + trigger: str, + actor: str, + entity_type: str, + entity_uuid: str | None, + removed_dashboard_slices: int = 0, +) -> UUID | None: + """Insert a ``pending`` audit row on a dedicated session, before the + purge runs. Returns the row id to confirm later, or ``None`` if the audit + write itself fails (which must not block the purge).""" + session = _dedicated_session() + try: + record = PurgeAuditLog( + status=STATUS_PENDING, + trigger=trigger, + actor=actor, + entity_type=entity_type, + entity_uuid=entity_uuid, + removed_dashboard_slices=removed_dashboard_slices, + created_on=datetime.utcnow(), + ) + session.add(record) + session.commit() + return cast(UUID, record.id) + except Exception: # pylint: disable=broad-except + session.rollback() + logger.warning( + "deletion_retention: failed to write pending audit row", exc_info=True + ) + return None Review Comment: This is the documented availability tradeoff: the audit write must not wedge retention when the audit store is unavailable, and reconcile_pending() sweeps the gap on the next run. Happy to flip to fail-closed (skip that entity for the run) if maintainers prefer compliance-over-availability — it is a small change. ########## superset/commands/deletion_retention/force_purge.py: ########## @@ -0,0 +1,147 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Compliance force-purge of a single entity by UUID. + +Immediate, irreversible removal of one entity regardless of the retention +window or whether it is currently soft-deleted or live. Runs the same cascade +as the time-based task with ``enforce_window=False`` — identical dependent +handling with legacy hard-delete semantics: M:N join rows hard-deleted, +a referencing live chart's loose ``datasource_id`` left dangling (the chart is +never modified). Idempotent: a UUID that resolves to nothing is a no-op. +""" + +from __future__ import annotations + +import logging +from typing import Any, cast + +from superset import db +from superset.commands.deletion_retention import audit +from superset.commands.deletion_retention.purge_cascade import ( + cascade_hard_delete, + CascadeResult, + dashboard_slice_count, + suppress_purge_association_versions, +) +from superset.models.helpers import skip_visibility_filter, SoftDeleteMixin + +logger: logging.Logger = logging.getLogger(__name__) + + +class ForcePurgeCommand: + """Force-purge the entity identified by *uuid*, if any.""" + + def __init__(self, uuid: str, actor: str = "operator") -> None: + self._uuid: str = uuid + self._actor: str = actor + + def _resolve(self) -> SoftDeleteMixin | None: + """Find the entity across every soft-delete model by UUID, matching + live or soft-deleted rows (visibility-filter bypassed).""" + for model in SoftDeleteMixin._registered_subclasses: # noqa: SLF001 + if not hasattr(model, "uuid"): + continue + with skip_visibility_filter(db.session, model): + entity = ( + db.session.query(model).filter(model.uuid == self._uuid).first() + ) + if entity is not None: + return entity + return None + + def run(self) -> dict[str, Any]: + """Resolve + purge. Returns a summary; a no-op when nothing matches.""" + audit.reconcile_pending() + entity = self._resolve() + if entity is None: + logger.info("force_purge: no entity for uuid=%s (no-op)", self._uuid) + return {"purged": False, "reason": "not_found", "uuid": self._uuid} + + entity_type = str(cast(Any, type(entity)).__tablename__) + removed_dashboard_slices = dashboard_slice_count(db.session, entity) + # The audit row commits independently. Release the resolving read + # transaction first, then resolve again against post-audit state. + db.session.rollback() # pylint: disable=consider-using-transaction + record_id = audit.write_ahead( + trigger=audit.TRIGGER_FORCE, + actor=self._actor, + entity_type=entity_type, + entity_uuid=self._uuid, + removed_dashboard_slices=removed_dashboard_slices, + ) + entity = self._resolve() Review Comment: Same tradeoff as the retention-path thread: fail-open is the documented choice so an audit-store outage cannot block an operator-invoked purge; flagging the alternative (abort when write_ahead returns None) as an easy flip if maintainers prefer it. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
