michael-s-molina commented on code in PR #39171: URL: https://github.com/apache/superset/pull/39171#discussion_r3560991600
########## superset/extensions/storage/persistent_dao.py: ########## @@ -0,0 +1,313 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import hashlib +import hmac +import logging + +from flask import current_app +from flask_babel import gettext as _ +from sqlalchemy import and_, func, LargeBinary + +from superset import db +from superset.daos.base import BaseDAO +from superset.exceptions import SupersetGenericErrorException +from superset.extensions.storage.filters import ExtensionStorageFilter +from superset.extensions.storage.persistent_model import ExtensionStorage +from superset.utils.encrypt import ( + DEFAULT_ENCRYPTION_ENGINE_NAME, + EncryptedType, + resolve_encryption_engine, +) + +logger = logging.getLogger(__name__) + + +class ExtensionStorageQuotaExceeded(SupersetGenericErrorException): + """Raised when a write would exceed an extension's persistent storage quota.""" + + status = 413 + + def __init__(self, extension_id: str, quota: int) -> None: + super().__init__( + message=_( + "Extension '%(extension_id)s' has exceeded its persistent " + "storage quota of %(quota)d bytes.", + extension_id=extension_id, + quota=quota, + ), + ) + + +def _derive_key(secret: str, user_fk: int) -> str: + """Derive a per-user encryption key from the master secret and user ID. + + Uses HMAC-SHA256 so the derived key is cryptographically bound to both. + Ciphertext encrypted for one user cannot be decrypted as another. + """ + return hmac.new( + secret.encode(), + str(user_fk).encode(), + hashlib.sha256, + ).hexdigest() + + +def _enc_type(user_fk: int | None) -> EncryptedType: + """Return an EncryptedType for the given scope. + + User-scoped rows use a key derived from SECRET_KEY and the user ID, so + ciphertext written for one user cannot be decrypted as another. + Shared rows (user_fk=None) use SECRET_KEY directly. + """ + secret = current_app.config["SECRET_KEY"] + key = _derive_key(secret, user_fk) if user_fk is not None else secret + engine_name = current_app.config.get( + "SQLALCHEMY_ENCRYPTED_FIELD_ENGINE", DEFAULT_ENCRYPTION_ENGINE_NAME + ) + return EncryptedType( + LargeBinary, + key=key, + engine=resolve_encryption_engine(engine_name), + ) + + +def _get_quota() -> int | None: + """Return the configured per-extension persistent storage quota in bytes. + + Returns None when no quota is configured (unlimited). + """ + config = current_app.config.get("EXTENSIONS_PERSISTENT_STORAGE", {}) + return config.get("QUOTA_PER_EXTENSION") + + +def _check_quota(extension_id: str, new_size: int, existing_size: int = 0) -> None: + """Raise if writing `new_size` bytes would push the extension over quota. + + `existing_size` is the size of the row being replaced (0 for inserts), + subtracted from current usage so overwriting a key doesn't double-count + it against the quota it already occupies. + """ + quota = _get_quota() + if quota is None: + return + current_usage = ( + db.session.query( + func.coalesce(func.sum(func.length(ExtensionStorage.value)), 0) + ) + .filter(ExtensionStorage.extension_id == extension_id) + .scalar() + ) + if current_usage - existing_size + new_size > quota: + raise ExtensionStorageQuotaExceeded(extension_id, quota) Review Comment: To optimize quota calculations, I updated our strategy to store sizes at the time of insertion. I evaluated using a separate table or Redis, but the added complexity wasn't worth it. A simple SUM on a numeric column gives us the efficiency we need. ########## superset/extensions/storage/persistent_model.py: ########## @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import uuid as uuid_module + +from flask_appbuilder import Model +from sqlalchemy import ( + Boolean, + Column, + ForeignKey, + Index, + Integer, + LargeBinary, + String, + Text, + UniqueConstraint, +) +from sqlalchemy.orm import backref, relationship +from sqlalchemy_utils import UUIDType +from superset_core.extensions.storage.models import ( + ExtensionStorageEntry as CoreExtensionStorageEntry, +) + +from superset.models.helpers import AuditMixinNullable + +# 16 MB — matches the KeyValue store limit. +EXTENSION_STORAGE_MAX_SIZE = 2**24 - 1 + + +class ExtensionStorage(CoreExtensionStorageEntry, AuditMixinNullable, Model): + """Generic persistent key-value storage for extensions (Tier 3). + + Each row is identified by (extension_id, user_fk, resource_type, + resource_uuid, key): + + * Global scope — user_fk IS NULL, resource_type IS NULL + * User scope — user_fk set, resource_type IS NULL + * Resource scope — resource_type + resource_uuid set (user_fk optional) + + The payload is stored as raw bytes (value) with a MIME-type hint + (value_type). When is_encrypted is True the value has been encrypted + at the DAO layer using Fernet and must be decrypted before use. + """ + + __tablename__ = "extension_storage" + + id = Column(Integer, primary_key=True, autoincrement=True) + uuid = Column( + UUIDType(binary=True), + default=uuid_module.uuid4, + unique=True, + nullable=False, + ) + + # Extension identity + extension_id = Column(String(255), nullable=False) + + # Scope discriminators — all nullable; NULLs define the scope (see docstring) + user_fk = Column( + Integer, + ForeignKey( + "ab_user.id", + ondelete="SET NULL", + name="fk_extension_storage_user_fk_ab_user", + ), + nullable=True, + ) + resource_type = Column(String(64), nullable=True) + resource_uuid = Column(String(36), nullable=True) Review Comment: Done -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
