sadpandajoe commented on code in PR #43771: URL: https://github.com/apache/superset/pull/43771#discussion_r3927462037
########## superset/mcp_service/chart/sunburst.py: ########## @@ -0,0 +1,329 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared native Sunburst result-role resolution and validation.""" + +import math +from collections.abc import Mapping +from dataclasses import dataclass +from decimal import Decimal +from typing import Any + +from superset.mcp_service.chart.schemas import ChartError +from superset.mcp_service.common.error_schemas import DatasetContext + + +@dataclass(frozen=True) +class SunburstResultRoles: + """Resolved query-output fields required to render a Sunburst.""" + + hierarchy: tuple[str, ...] + primary_metric: str + secondary_metric: str | None = None + + +def _column_result_label(column: Any) -> str | None: + """Resolve a bounded native column reference to its query output label.""" + if isinstance(column, str) and column: + return column + if not isinstance(column, Mapping) or not 0 < len(column) <= 20: + return None + for key in ("label", "column_name", "columnName", "sqlExpression"): + value = column.get(key) + if isinstance(value, str) and value: + return value + if value not in (None, ""): + return None + return None + + +def _metric_result_label(metric: Any) -> str | None: + """Resolve saved, SIMPLE, and SQL metric aliases like the frontend.""" + if isinstance(metric, str) and metric: + return metric + if not isinstance(metric, Mapping) or not 0 < len(metric) <= 20: + return None + + label = metric.get("label") + if isinstance(label, str) and label: + return label + if label not in (None, ""): + return None + + expression_type = metric.get("expressionType") + if expression_type == "SIMPLE": + aggregate = metric.get("aggregate") + column = metric.get("column") + if ( + not isinstance(aggregate, str) + or not aggregate + or len(aggregate) > 100 + or not isinstance(column, Mapping) + or not 0 < len(column) <= 50 + ): + return None + column_name = column.get("column_name") or column.get("columnName") + if not isinstance(column_name, str) or not column_name: + return None + return f"{aggregate}({column_name})" + if expression_type == "SQL": + sql_expression = metric.get("sqlExpression") + if ( + isinstance(sql_expression, str) + and sql_expression + and len(sql_expression) <= 2000 + ): + return sql_expression + return None + + +def _metric_query_identity(metric: Any) -> tuple[str, ...] | None: + """Return a hook-free identity for one exact native metric value.""" + if type(metric) is str and metric: + return ("saved", metric) + if type(metric) is not dict: + return None + expression_type = dict.get(metric, "expressionType") + if expression_type == "SQL": + expression = dict.get(metric, "sqlExpression") + return ("sql", expression) if type(expression) is str and expression else None + if expression_type == "SIMPLE": + aggregate = dict.get(metric, "aggregate") + column = dict.get(metric, "column") + if type(aggregate) is not str: + return None + column_name: Any + if type(column) is str: + column_name = column + elif type(column) is dict: + column_name = dict.get(column, "column_name") + if column_name is None: + column_name = dict.get(column, "columnName") + else: + return None + if type(column_name) is str and column_name: + return ("simple", aggregate, column_name) + return None + + +def resolve_sunburst_result_roles( + form_data: Mapping[str, Any], +) -> tuple[SunburstResultRoles | None, ChartError | None]: + """Resolve and validate all native Sunburst query-result roles.""" + columns = form_data.get("columns") + if not isinstance(columns, list) or not columns: + return None, ChartError( + error="Sunburst form data requires one or more hierarchy columns.", + error_type="InvalidSunburstFormData", + ) + hierarchy: list[str] = [] + for index, column in enumerate(columns): + label = _column_result_label(column) + if label is None: + return None, ChartError( + error=f"Sunburst hierarchy column {index + 1} is malformed.", + error_type="InvalidSunburstFormData", + ) + hierarchy.append(label) + + primary = _metric_result_label(form_data.get("metric")) + if primary is None: + return None, ChartError( + error="Sunburst primary metric is missing or malformed.", + error_type="InvalidSunburstFormData", + ) + primary_metric = form_data.get("metric") + secondary: str | None = None + if (secondary_metric := form_data.get("secondary_metric")) is not None: + secondary = _metric_result_label(secondary_metric) + if secondary is None: + return None, ChartError( + error="Sunburst secondary metric is malformed.", + error_type="InvalidSunburstFormData", + ) + if ( + secondary.casefold() == primary.casefold() + and _metric_query_identity(primary_metric) + == _metric_query_identity(secondary_metric) + and _metric_query_identity(primary_metric) is not None + ): + # The frontend interprets a repeated primary metric as categorical + # color mode and the query has one physical metric output. + secondary = None + + labels = [*hierarchy, primary, *([secondary] if secondary else [])] + seen: dict[str, str] = {} + for label in labels: + folded = label.casefold() + if previous := seen.get(folded): + return None, ChartError( + error=( + f"Sunburst result label {label!r} is ambiguous with {previous!r}." + ), + error_type="InvalidSunburstFormData", + ) + seen[folded] = label + + return SunburstResultRoles(tuple(hierarchy), primary, secondary), None + + +def _finite_number(value: Any) -> bool: + """Return whether a normalized database value is exactly numeric and finite.""" + value_type = type(value) + if value_type is int: + return True + if value_type is float: + return math.isfinite(value) + if value_type is Decimal: + return Decimal.is_finite(value) + return False + + +def _valid_hierarchy_value(value: Any) -> bool: + """Reject nested/container values that cannot form a stable node label.""" + return not isinstance(value, (Mapping, list, tuple, set)) + + +def validate_sunburst_result_data( + data: Any, form_data: Mapping[str, Any] +) -> tuple[SunburstResultRoles | None, ChartError | None]: + """Validate every Sunburst row and all resolved result aliases.""" + roles, error = resolve_sunburst_result_roles(form_data) + if error is not None: + return None, error + assert roles is not None + if not isinstance(data, list): + return None, ChartError( + error="Sunburst query result data is not an array of rows.", + error_type="InvalidSunburstResult", + ) + + required_fields = [ + *roles.hierarchy, + roles.primary_metric, + *([roles.secondary_metric] if roles.secondary_metric else []), + ] + for index, row in enumerate(data, start=1): + if not isinstance(row, Mapping): + return None, ChartError( + error=f"Sunburst result row {index} is not an object.", + error_type="InvalidSunburstResult", + ) + missing = [field for field in required_fields if field not in row] + if missing: + return None, ChartError( + error=( + f"Sunburst result row {index} is missing required field(s): " + f"{', '.join(str(field) for field in missing)}." + ), + error_type="InvalidSunburstResult", + ) + for field in roles.hierarchy: + if not _valid_hierarchy_value(row[field]): + return None, ChartError( + error=( + f"Sunburst result row {index} has a malformed hierarchy " + f"value for {field!r}." + ), + error_type="InvalidSunburstResult", + ) + for field in ( + roles.primary_metric, + *([roles.secondary_metric] if roles.secondary_metric else []), + ): + assert field is not None + if not _finite_number(row[field]): Review Comment: A NULL aggregate value now rejects the entire Sunburst result even though the frontend treats a non-numeric metric as zero. A group whose `SUM(revenue)` is NULL can therefore make generate/preview fail (and compile sampling makes it intermittent); could this normalize missing metric values to the frontend behavior before validating? ########## superset/mcp_service/chart/query_result.py: ########## @@ -0,0 +1,1520 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Canonicalize and validate ``ChartDataCommand`` result envelopes.""" + +import math +import time as system_time +from bisect import bisect_right +from dataclasses import dataclass +from datetime import date, datetime, time, timedelta, timezone +from decimal import Decimal +from enum import Enum +from typing import Any +from uuid import UUID +from zoneinfo import ZoneInfo + +import numpy as np +import pandas as pd +import pytz +from dateutil import tz as dateutil_tz, zoneinfo as dateutil_zoneinfo +from pydantic import BaseModel +from pydantic_core import to_json + +from superset.common.chart_data import ChartDataResultFormat +from superset.common.db_query_status import QueryStatus +from superset.mcp_service.chart.schemas import ChartError +from superset.utils.core import ( + ExtraFiltersReasonType, + ExtraFiltersTimeColumnType, + GenericDataType, +) + +FAILED_QUERY_STATUSES = frozenset( + {"error", "failed", "stopped", "timed_out", "cancelled", "canceled"} +) + +# These are aggregate envelope limits, not per-query allowances. In particular, +# splitting a result across the maximum number of queries must not multiply the +# permitted rows, nodes, or encoded bytes. +MAX_QUERY_RESULTS = 32 +MAX_QUERY_RESULT_ROWS_PER_QUERY = 50_000 +MAX_QUERY_RESULT_ROWS = 100_000 +MAX_QUERY_RESULT_COLUMNS = 4_096 +MAX_QUERY_RESULT_VALUES = 2_500_000 +MAX_QUERY_RESULT_VALUE_BYTES = 16 * 1024 * 1024 +MAX_QUERY_RESULT_METADATA_BYTES = 1024 * 1024 +MAX_QUERY_RESULT_METADATA_ITEMS = 32_768 +MAX_RESULT_VALUE_ITEMS = 4_096 +MAX_RESULT_VALUE_DEPTH = 32 +MAX_RESULT_STRING_LENGTH = 65_536 +MAX_RESULT_KEY_LENGTH = 4_096 +MAX_RESULT_INTEGER_BITS = 4_096 +MAX_RESULT_INTEGER_DIGITS = 1_234 +MAX_RESULT_DECIMAL_DIGITS = 1_024 +MAX_RESULT_DECIMAL_MAGNITUDE = 4_096 +MAX_RESULT_DECIMAL_STORAGE = 2_048 +MAX_QUERY_RESULT_ROWCOUNT = 2**63 - 1 +MAX_QUERY_RESULT_CACHE_TIMEOUT = 2**31 - 1 +MAX_QUERY_RESULT_TIMESTAMP_LENGTH = 64 + +_ERROR_KEYS = ("error", "errors", "error_message", "message", "detail") +_MAX_ERROR_TEXT_BYTES = 2_000 +_TRUSTED_TIMEZONE_TYPES = (timezone, ZoneInfo) +_SAFE_RESULT_ENUM_TYPES = frozenset( + { + ChartDataResultFormat, + QueryStatus, + ExtraFiltersReasonType, + ExtraFiltersTimeColumnType, + GenericDataType, + } +) +_RESULT_FORMAT_VALUES = frozenset( + object.__getattribute__(member, "_value_") for member in ChartDataResultFormat +) +_COLTYPE_VALUES = frozenset( + object.__getattribute__(member, "_value_") for member in GenericDataType +) +_NUMPY_INTEGER_TYPES = frozenset( + type(value) + for value in ( + np.int8(0), + np.int16(0), + np.int32(0), + np.int64(0), + np.uint8(0), + np.uint16(0), + np.uint32(0), + np.uint64(0), + ) +) +_NUMPY_FLOAT_TYPES = frozenset( + type(value) + for value in (np.float16(0), np.float32(0), np.float64(0), np.longdouble(0)) +) +_PANDAS_NAT_TYPE = type(pd.NaT) +_PANDAS_NA_TYPE = type(pd.NA) +_PANDAS_PERIOD_TYPE = type(pd.Period("2000-01", freq="M")) +_PANDAS_INTERVAL_TYPE = type(pd.Interval(0, 1)) +_DATEUTIL_FIXED_TIMEZONE_TYPES = frozenset( + {type(dateutil_tz.tzoffset(None, 0)), type(dateutil_tz.tzutc())} +) +_DATEUTIL_NAMED_TIMEZONE_TYPES = frozenset( + {dateutil_tz.tzfile, dateutil_zoneinfo.tzfile} +) +_DATEUTIL_TTINFO_TYPE = type( + object.__getattribute__(dateutil_tz.gettz("UTC"), "__dict__")["_ttinfo_std"] +) +_DATEUTIL_LOCAL_TIMEZONE_TYPE = type(dateutil_tz.tzlocal()) +_PYTZ_FIXED_TIMEZONE_TYPES = frozenset({type(pytz.FixedOffset(1))}) +_MAX_DATEUTIL_TRANSITIONS = 4_096 +_MAX_DATEUTIL_TTINFOS = 256 +_MAX_DATEUTIL_TRANSITION_MAGNITUDE = 10**12 + + +@dataclass +class _ResultBudget: + """Aggregate counters shared by every query and metadata value.""" + + rows: int = 0 + values: int = 0 + json_bytes: int = 0 + metadata_items: int = 0 + metadata_bytes: int = 0 + + +@dataclass(frozen=True) +class _DateutilTimezoneState: + """Hook-free subset of a validated exact dateutil tzfile transition table.""" + + transitions: tuple[int, ...] + transition_offsets: tuple[int, ...] + standard_offset: int + before_offset: int | None + + +def _invalid_result(message: str) -> ChartError: + return ChartError( + error=f"Chart query returned {message}.", + error_type="InvalidQueryResult", + ) + + +def _invalid_metadata(label: str) -> ChartError: + return ChartError( + error=f"{label} returned hostile or malformed metadata.", + error_type="InvalidQueryResult", + ) + + +def _safe_enum_value(value: Any, expected: frozenset[type[Any]]) -> Any | None: + """Read trusted enum storage without invoking public conversion hooks.""" + if type(value) not in expected or type(value) not in _SAFE_RESULT_ENUM_TYPES: + return None + return object.__getattribute__(value, "_value_") + + +def _bounded_utf8_length(value: str, maximum: int) -> int | None: + """Return the exact UTF-8 size while bounding pre-encoding work.""" + if str.__len__(value) > maximum: + return None + try: + encoded = str.encode(value, "utf-8", errors="strict") + except UnicodeEncodeError: + return None + size = bytes.__len__(encoded) + return size if size <= maximum else None + + +def _json_string_size(value: str, maximum: int) -> int | None: + """Return compact UTF-8 JSON string size without serializing the value.""" + raw_size = _bounded_utf8_length(value, maximum) + if raw_size is None: + return None + escaped_size = raw_size + 2 + for character in value: + codepoint = ord(character) + if character in {'"', "\\", "\b", "\t", "\n", "\f", "\r"}: + escaped_size += 1 + elif codepoint < 0x20: + escaped_size += 5 + return escaped_size + + +def _integer_json_size(value: int) -> int: + """Return exact decimal JSON size without rendering the bounded integer.""" + magnitude = -value if value < 0 else value + if magnitude == 0: + digits = 1 + else: + bits = int.bit_length(magnitude) + digits = ((bits - 1) * 30103) // 100000 + 1 + if magnitude >= 10**digits: + digits += 1 + return digits + (value < 0) + + +def _container_json_syntax_size(item_count: int, *, mapping: bool) -> int: + """Return braces/brackets plus compact separators and mapping colons.""" + if item_count == 0: + return 2 + return 2 + item_count - 1 + (item_count if mapping else 0) + + +def _normalized_scalar_json_size(value: Any) -> int: + """Return exact compact JSON size for a normalized scalar.""" + value_type = type(value) + if value is None: + return 4 + if value_type is bool: + return 4 if value else 5 + if value_type is str: + size = _json_string_size(value, MAX_RESULT_STRING_LENGTH) + assert size is not None + return size + if value_type is int: + return _integer_json_size(value) + if value_type is float: + return len(float.__repr__(value)) + if value_type is Decimal: + # Pydantic serializes Decimal values as JSON strings so their exact + # finite value survives the wire projection without binary rounding. + text = Decimal.__str__(value) + size = _json_string_size(text, MAX_RESULT_STRING_LENGTH) + assert size is not None + return size + raise AssertionError("result scalar was not normalized") + + +def _pydantic_scalar_json_size(value: Any) -> int: + """Return the scalar size emitted by Pydantic's JSON serializer.""" + if type(value) is float: + # pydantic-core uses the shortest exponent (``1e-7``), while Python's + # repr retains a leading zero (``1e-07``). + return len(to_json(value)) + return _normalized_scalar_json_size(value) + + +def _charge_json_bytes( + budget: _ResultBudget, size: int, *, metadata: bool = False +) -> str | None: + budget.json_bytes += size + if budget.json_bytes > MAX_QUERY_RESULT_VALUE_BYTES: + return "too many aggregate JSON bytes" + if metadata: + budget.metadata_bytes += size + if budget.metadata_bytes > MAX_QUERY_RESULT_METADATA_BYTES: + return "too many aggregate metadata JSON bytes" + return None + + +def _charge_value(budget: _ResultBudget, *, metadata: bool = False) -> str | None: + budget.values += 1 + if budget.values > MAX_QUERY_RESULT_VALUES: + return "too many aggregate values" + if metadata: + budget.metadata_items += 1 + if budget.metadata_items > MAX_QUERY_RESULT_METADATA_ITEMS: + return "too many aggregate metadata values" + return None + + +def _charge_text( + value: str, + budget: _ResultBudget, + *, + key: bool = False, + metadata: bool = False, +) -> str | None: + maximum = ( + MAX_RESULT_KEY_LENGTH + if key + else MAX_QUERY_RESULT_METADATA_BYTES + if metadata + else MAX_RESULT_STRING_LENGTH + ) + size = _json_string_size(value, maximum) + if size is None: + return "an invalid or oversized object key" if key else "invalid text data" + return _charge_json_bytes(budget, size, metadata=metadata) + + +def _integer_failure(value: int) -> str | None: + bits = int.bit_length(value) + if bits > MAX_RESULT_INTEGER_BITS: + return "an oversized integer" + digits = 1 if bits == 0 else ((bits - 1) * 30103) // 100000 + 1 + if digits > MAX_RESULT_INTEGER_DIGITS: + return "an oversized integer" + return None + + +def _decimal_failure(value: Decimal) -> str | None: + if Decimal.__sizeof__(value) > MAX_RESULT_DECIMAL_STORAGE: + return "an oversized Decimal" + if not Decimal.is_finite(value): + return "a non-finite Decimal" + parts = Decimal.as_tuple(value) + if tuple.__len__(parts.digits) > MAX_RESULT_DECIMAL_DIGITS: + return "an oversized Decimal" + exponent = parts.exponent + if type(exponent) is not int or abs(exponent) > MAX_RESULT_DECIMAL_MAGNITUDE: + return "an oversized Decimal" + return None + + +def _type_mro(value_type: type[Any]) -> tuple[type[Any], ...]: + """Read a concrete type's MRO without consulting metaclass overrides.""" + try: + mro = type.__getattribute__(value_type, "__mro__") + except (AttributeError, TypeError): # pragma: no cover - defensive metaclass + return () + return mro if type(mro) is tuple else () + + +def _timezone_name_without_hooks(tzinfo: Any) -> str | None: # noqa: C901 + """Read common pytz/dateutil zone state without dispatching timezone hooks.""" + value_mro = _type_mro(type(tzinfo)) + if any(base is pytz.tzinfo.BaseTzInfo for base in value_mro): + for base in value_mro: + try: + namespace = type.__getattribute__(base, "__dict__") + except (AttributeError, TypeError): # pragma: no cover + continue + zone = namespace.get("zone") + if type(zone) is str and _bounded_utf8_length(zone, 256) is not None: + try: + canonical = pytz.timezone(zone) + except (KeyError, ValueError): + return None + # Generated pytz types are trusted; arbitrary subclasses that + # inherit their internal fields are not. + return zone if type(canonical) is type(tzinfo) else None + + if type(tzinfo) not in _DATEUTIL_NAMED_TIMEZONE_TYPES: + return None + + try: + namespace = object.__getattribute__(tzinfo, "__dict__") + except (AttributeError, TypeError): + return None + if type(namespace) is not dict: + return None + filename = dict.get(namespace, "_filename") + if type(filename) is not str or _bounded_utf8_length(filename, 4_096) is None: + return None + marker = "/zoneinfo/" + if (offset := str.find(filename, marker)) >= 0: + name = str.__getitem__(filename, slice(offset + len(marker), None)) + elif not str.startswith(filename, "/") and str.find(filename, "\\") < 0: + name = filename + else: + return None + parts = str.split(name, "/") + if not parts or any(part in {"", ".", ".."} for part in parts): + return None + return name if _bounded_utf8_length(name, 256) is not None else None + + +def _object_namespace(value: Any) -> dict[str, Any] | None: + """Read exact instance storage without descriptor dispatch.""" + try: + namespace = object.__getattribute__(value, "__dict__") + except (AttributeError, TypeError): + return None + return namespace if type(namespace) is dict else None + + +def _dateutil_ttinfo_offset_without_hooks(value: Any) -> int | None: + """Validate one exact dateutil transition record and return its offset.""" + if type(value) is not _DATEUTIL_TTINFO_TYPE: + return None + try: + offset = object.__getattribute__(value, "offset") + delta = object.__getattribute__(value, "delta") + isdst = object.__getattribute__(value, "isdst") + abbreviation = object.__getattribute__(value, "abbr") + is_standard = object.__getattribute__(value, "isstd") + is_gmt = object.__getattribute__(value, "isgmt") + dst_offset = object.__getattribute__(value, "dstoffset") + except (AttributeError, TypeError): + return None + if type(offset) is not int or not -86_400 < offset < 86_400: + return None + if type(delta) is not timedelta or delta != timedelta(seconds=offset): + return None + if type(isdst) is not int or isdst not in {0, 1}: + return None + if abbreviation is not None and ( + type(abbreviation) is not str or _bounded_utf8_length(abbreviation, 256) is None + ): + return None + if type(is_standard) is not bool or type(is_gmt) is not bool: + return None + if type(dst_offset) is not timedelta: + return None + if not -timedelta(days=1) < dst_offset < timedelta(days=1): + return None + return offset + + +def _dateutil_named_state_without_hooks( # noqa: C901 + tzinfo: Any, +) -> _DateutilTimezoneState | None: + """Validate bounded exact dateutil tzfile state without timezone hooks.""" + if type(tzinfo) not in _DATEUTIL_NAMED_TIMEZONE_TYPES: + return None + if _timezone_name_without_hooks(tzinfo) is None: + return None + namespace = _object_namespace(tzinfo) + if namespace is None: + return None + transitions = dict.get(namespace, "_trans_list") + utc_transitions = dict.get(namespace, "_trans_list_utc") + transition_info = dict.get(namespace, "_trans_idx") + info_list = dict.get(namespace, "_ttinfo_list") + standard_info = dict.get(namespace, "_ttinfo_std") + before_info = dict.get(namespace, "_ttinfo_before") + first_info = dict.get(namespace, "_ttinfo_first") + if ( + type(transitions) is not tuple + or type(utc_transitions) is not tuple + or type(transition_info) is not tuple + or type(info_list) is not list + or tuple.__len__(transitions) > _MAX_DATEUTIL_TRANSITIONS + or tuple.__len__(utc_transitions) != tuple.__len__(transitions) + or tuple.__len__(transition_info) != tuple.__len__(transitions) + or list.__len__(info_list) == 0 + or list.__len__(info_list) > _MAX_DATEUTIL_TTINFOS + ): + return None + + previous_transition: int | None = None + previous_utc_transition: int | None = None + for index in range(tuple.__len__(transitions)): + transition = tuple.__getitem__(transitions, index) + utc_transition = tuple.__getitem__(utc_transitions, index) + if ( + type(transition) is not int + or type(utc_transition) is not int + or abs(transition) > _MAX_DATEUTIL_TRANSITION_MAGNITUDE + or abs(utc_transition) > _MAX_DATEUTIL_TRANSITION_MAGNITUDE + or (previous_transition is not None and transition <= previous_transition) + or ( + previous_utc_transition is not None + and utc_transition <= previous_utc_transition + ) + ): + return None + previous_transition = transition + previous_utc_transition = utc_transition + + known_info_ids: set[int] = set() + for index in range(list.__len__(info_list)): + info = list.__getitem__(info_list, index) + if _dateutil_ttinfo_offset_without_hooks(info) is None: + return None + known_info_ids.add(id(info)) + for info in (standard_info, before_info, first_info): + if info is not None and id(info) not in known_info_ids: + return None + for index in range(tuple.__len__(transition_info)): + if id(tuple.__getitem__(transition_info, index)) not in known_info_ids: + return None + if not transitions: + if ( + standard_info is not list.__getitem__(info_list, 0) + or first_info is not standard_info + or before_info is not None + ): + return None + else: + expected_standard = None + expected_dst = None + for index in range(tuple.__len__(transition_info) - 1, -1, -1): + info = tuple.__getitem__(transition_info, index) + is_dst = object.__getattribute__(info, "isdst") + if expected_standard is None and not is_dst: + expected_standard = info + elif expected_dst is None and is_dst: + expected_dst = info + if expected_standard is not None and expected_dst is not None: + break + if expected_standard is None: + expected_standard = expected_dst + expected_before = None + for index in range(list.__len__(info_list)): + info = list.__getitem__(info_list, index) + if not object.__getattribute__(info, "isdst"): + expected_before = info + break + if expected_before is None: + expected_before = list.__getitem__(info_list, 0) + if standard_info is not expected_standard or before_info is not expected_before: + return None + standard_offset = _dateutil_ttinfo_offset_without_hooks(standard_info) + if standard_offset is None: + return None + before_offset = ( + _dateutil_ttinfo_offset_without_hooks(before_info) + if before_info is not None + else None + ) + transition_offsets: list[int] = [] + previous_offset: int | None = None + previous_base_offset: int | None = None + previous_is_dst: int | None = None + previous_dst_offset = 0 + for index in range(tuple.__len__(transition_info)): + info = tuple.__getitem__(transition_info, index) + if id(info) not in known_info_ids: + return None + offset = _dateutil_ttinfo_offset_without_hooks(info) + if offset is None: + return None + is_dst = object.__getattribute__(info, "isdst") + dst_offset_seconds = 0 + if previous_is_dst is not None and is_dst: + if not previous_is_dst: + assert previous_offset is not None + dst_offset_seconds = offset - previous_offset + if not dst_offset_seconds and previous_dst_offset: + dst_offset_seconds = previous_dst_offset + previous_dst_offset = dst_offset_seconds + base_offset = offset - dst_offset_seconds + adjustment = base_offset + if ( + previous_base_offset is not None + and base_offset != previous_base_offset + and is_dst != previous_is_dst + ): + adjustment = previous_base_offset + if ( + tuple.__getitem__(transitions, index) + != tuple.__getitem__(utc_transitions, index) + adjustment + ): + return None + transition_offsets.append(offset) + previous_offset = offset + previous_base_offset = base_offset + previous_is_dst = is_dst + if transitions and before_offset is None: + return None + return _DateutilTimezoneState( + transitions=transitions, + transition_offsets=tuple(transition_offsets), + standard_offset=standard_offset, + before_offset=before_offset, + ) + + +def _dateutil_named_offset_without_hooks( # noqa: C901 + value: datetime, tzinfo: Any +) -> timezone | None: + """Preserve dateutil's source-selected wall offset from validated state.""" + state = _dateutil_named_state_without_hooks(tzinfo) + if state is None: + return None + epoch_ordinal = date.toordinal(date(1970, 1, 1)) + wall_timestamp = ( + (datetime.toordinal(value) - epoch_ordinal) * 86_400 + + value.hour * 3_600 + + value.minute * 60 + + value.second + ) + transitions = state.transitions + selected_offset: int | None + if not transitions: + selected_offset = state.standard_offset + else: + index = bisect_right(transitions, wall_timestamp) - 1 + + def offset_at(transition_index: int | None) -> int | None: + if transition_index is None or transition_index + 1 >= len(transitions): + return state.standard_offset + if transition_index < 0: + return state.before_offset + return state.transition_offsets[transition_index] + + if index > 0: + selected_offset = offset_at(index) + previous_offset = offset_at(index - 1) + if selected_offset is None or previous_offset is None: + return None + is_ambiguous = wall_timestamp < transitions[index] + ( + previous_offset - selected_offset + ) + if not value.fold and is_ambiguous: + index -= 1 + selected_offset = offset_at(index) + if selected_offset is None: + return None + try: + return timezone(timedelta(seconds=selected_offset)) + except (OverflowError, ValueError): + return None + + +def _pytz_named_offset_without_hooks(tzinfo: Any) -> timezone | None: + """Return a localized pytz zone's stored offset without calling hooks.""" + if _timezone_name_without_hooks(tzinfo) is None: + return None + namespace = _object_namespace(tzinfo) + offset = dict.get(namespace, "_utcoffset") if namespace is not None else None + if type(offset) is not timedelta: + return None + try: + return timezone(offset) + except ValueError: + return None + + +def _dateutil_local_offset_without_hooks( + value: datetime, tzinfo: Any +) -> timezone | None: + """Select a dateutil local offset using builtin system-time data.""" + if type(tzinfo) is not _DATEUTIL_LOCAL_TIMEZONE_TYPE: + return None + namespace = _object_namespace(tzinfo) + if namespace is None: + return None + standard_offset = dict.get(namespace, "_std_offset") + daylight_offset = dict.get(namespace, "_dst_offset") + has_daylight = dict.get(namespace, "_hasdst") + if ( + type(standard_offset) is not timedelta + or type(daylight_offset) is not timedelta + or type(has_daylight) is not bool + ): + return None + selected_offset = standard_offset + if has_daylight: + epoch = datetime(1970, 1, 1) + naive = datetime( + value.year, + value.month, + value.day, + value.hour, + value.minute, + value.second, + value.microsecond, + ) + timestamp = (naive - epoch).total_seconds() + try: + is_daylight = bool( + system_time.localtime(timestamp + system_time.timezone).tm_isdst + ) + daylight_saved = daylight_offset - standard_offset + previous_is_daylight = bool( + system_time.localtime( + timestamp + - timedelta.total_seconds(daylight_saved) + + system_time.timezone + ).tm_isdst + ) + except (OverflowError, OSError, ValueError): + return None + if not is_daylight and is_daylight != previous_is_daylight: + is_daylight = not bool(value.fold) + selected_offset = daylight_offset if is_daylight else standard_offset + try: + return timezone(selected_offset) + except ValueError: + return None + + +def _canonical_timezone(tzinfo: Any) -> timezone | ZoneInfo | None: # noqa: C901 + if any(type(tzinfo) is trusted for trusted in _TRUSTED_TIMEZONE_TYPES): + return tzinfo + if type(tzinfo) in _DATEUTIL_FIXED_TIMEZONE_TYPES: + try: + namespace = object.__getattribute__(tzinfo, "__dict__") + except (AttributeError, TypeError): + return timezone.utc if type(tzinfo) is type(dateutil_tz.tzutc()) else None + if type(namespace) is not dict: + return None + offset = dict.get(namespace, "_offset") + if type(offset) is not timedelta: + return timezone.utc if type(tzinfo) is type(dateutil_tz.tzutc()) else None + if abs(offset) >= timedelta(days=1): + return None + return timezone(offset) + if type(tzinfo) in _PYTZ_FIXED_TIMEZONE_TYPES: + try: + namespace = object.__getattribute__(tzinfo, "__dict__") + except (AttributeError, TypeError): + return None + if type(namespace) is not dict: + return None + minutes = dict.get(namespace, "_minutes") + if type(minutes) is not int or not -1_440 < minutes < 1_440: + return None + return timezone(timedelta(minutes=minutes)) + return None + + +def _timestamp_offset_without_hooks(value: pd.Timestamp) -> timezone | None: + """Recover a timestamp's stored wall-clock offset without timezone hooks.""" + multipliers = {"s": 1_000_000_000, "ms": 1_000_000, "us": 1_000, "ns": 1} + multiplier = multipliers.get(value.unit) + if multiplier is None: + return None + try: + instant_ns = int(value.asm8.view("i8")) * multiplier + epoch_ordinal = date.toordinal(date(1970, 1, 1)) + wall_ns = ( + ( + (datetime.toordinal(value) - epoch_ordinal) * 86_400 + + value.hour * 3600 + + value.minute * 60 + + value.second + ) + * 1_000_000_000 + + value.microsecond * 1000 + + value.nanosecond + ) + offset_ns = wall_ns - instant_ns + if offset_ns % 1000: + return None + return timezone(timedelta(microseconds=offset_ns // 1000)) + except (OverflowError, TypeError, ValueError): + return None + + +def _canonical_datetime(value: datetime) -> tuple[str | None, str | None]: + """Serialize an exact datetime through trusted timezone state only.""" + tzinfo = value.tzinfo + canonical_value = value + if tzinfo is not None and not any( + type(tzinfo) is trusted for trusted in _TRUSTED_TIMEZONE_TYPES + ): + canonical_tz = ( + _pytz_named_offset_without_hooks(tzinfo) + or _dateutil_local_offset_without_hooks(value, tzinfo) + or _dateutil_named_offset_without_hooks(value, tzinfo) + or _canonical_timezone(tzinfo) + ) + if canonical_tz is None: + return None, "a datetime with an unsupported timezone" + canonical_value = datetime( + value.year, + value.month, + value.day, + value.hour, + value.minute, + value.second, + value.microsecond, + tzinfo=canonical_tz, + fold=value.fold, + ) + try: + return datetime.isoformat(canonical_value), None + except (OverflowError, TypeError, ValueError): + return None, "an invalid datetime" + + +def _canonical_time(value: time) -> tuple[str | None, str | None]: + """Serialize an exact time through trusted timezone state only.""" + tzinfo = value.tzinfo + canonical_value = value + if tzinfo is not None and not any( + type(tzinfo) is trusted for trusted in _TRUSTED_TIMEZONE_TYPES + ): + if _dateutil_named_state_without_hooks(tzinfo) is not None: + canonical_value = time( + value.hour, + value.minute, + value.second, + value.microsecond, + fold=value.fold, + ) + else: + canonical_tz = _canonical_timezone(tzinfo) + if canonical_tz is None and type(tzinfo) is _DATEUTIL_LOCAL_TIMEZONE_TYPE: + namespace = _object_namespace(tzinfo) + if namespace is not None and dict.get(namespace, "_hasdst") is False: + offset = dict.get(namespace, "_std_offset") + if type(offset) is timedelta: + try: + canonical_tz = timezone(offset) + except ValueError: + canonical_tz = None + if canonical_tz is None: + return None, "a time with an unsupported timezone" + canonical_value = time( + value.hour, + value.minute, + value.second, + value.microsecond, + tzinfo=canonical_tz, + fold=value.fold, + ) + try: + return time.isoformat(canonical_value), None + except (OverflowError, TypeError, ValueError): + return None, "an invalid time" + + +def _canonical_timestamp(value: pd.Timestamp) -> tuple[str | None, str | None]: + """Preserve a trusted timestamp's instant, offset, nanoseconds, and fold.""" + try: + tzinfo = value.tzinfo + if tzinfo is not None and not any( + type(tzinfo) is trusted for trusted in _TRUSTED_TIMEZONE_TYPES + ): + supported_timezone = ( + _canonical_timezone(tzinfo) is not None + or type(tzinfo) is _DATEUTIL_LOCAL_TIMEZONE_TYPE + or _dateutil_named_state_without_hooks(tzinfo) is not None + ) + if not supported_timezone: + return None, "a timestamp with an unsupported timezone" + canonical_tz = _timestamp_offset_without_hooks(value) + if canonical_tz is None: + return None, "an invalid timestamp" + raw_value = value.asm8.view("i8") + value = pd.Timestamp(raw_value, unit=value.unit, tz="UTC").tz_convert( + canonical_tz + ) + return pd.Timestamp.isoformat(value), None + except (KeyError, OverflowError, TypeError, ValueError): + return None, "an invalid timestamp" + + +def _normalize_scalar(value: Any) -> tuple[Any, str | None]: # noqa: C901 + """Convert one exact trusted producer scalar to a JSON-safe scalar.""" + value_type = type(value) + if value is None or value_type is bool or value_type is str: + return value, None + if value_type is int: + return value, _integer_failure(value) + if value_type is float: + if math.isnan(value): + return None, None + return (value, None) if math.isfinite(value) else (None, "a non-finite number") + if value_type is Decimal: + return value, _decimal_failure(value) + if value_type is datetime: + return _canonical_datetime(value) + if value_type is time: + return _canonical_time(value) + if value_type is date: + return date.isoformat(value), None + if value_type is timedelta: + try: + return pd.Timedelta(value).isoformat(), None + except (OverflowError, TypeError, ValueError): + return None, "an invalid duration" + if value_type is UUID: + return UUID.__str__(value), None + + if value_type is _PANDAS_NAT_TYPE or value_type is _PANDAS_NA_TYPE: + return None, None + if value_type is pd.Timestamp: + return _canonical_timestamp(value) + if value_type is pd.Timedelta: + if pd.isna(value): + return None, None + try: + return pd.Timedelta.isoformat(value), None + except (OverflowError, TypeError, ValueError): + return None, "an invalid pandas duration" + if value_type is _PANDAS_PERIOD_TYPE or value_type is _PANDAS_INTERVAL_TYPE: + # These concrete immutable pandas extension scalars are trusted. Exact + # type checks deliberately exclude subclasses with conversion hooks. + try: + normalized_text = str(value) + except (OverflowError, TypeError, ValueError): + return None, "an invalid pandas scalar" + if _bounded_utf8_length(normalized_text, MAX_RESULT_STRING_LENGTH) is None: + return None, "an oversized pandas scalar" + return normalized_text, None + if value_type in _NUMPY_INTEGER_TYPES: + normalized = int(value) + return normalized, _integer_failure(normalized) + if value_type in _NUMPY_FLOAT_TYPES: + normalized_float = float(value) + if math.isnan(normalized_float): + return None, None + return ( + (normalized_float, None) + if math.isfinite(normalized_float) + else (None, "a non-finite NumPy number") + ) + if value_type is np.bool_: + return bool(value), None + if value_type is np.str_: + return str(value), None + if value_type is np.datetime64: + if np.isnat(value): + return None, None + try: + return _canonical_timestamp(pd.Timestamp(value)) + except (OverflowError, TypeError, ValueError): + return None, "an invalid NumPy timestamp" + if value_type is np.timedelta64: + if np.isnat(value): + return None, None + try: + return pd.Timedelta(value).isoformat(), None + except (OverflowError, TypeError, ValueError): + return None, "an invalid NumPy duration" + return None, "an unsupported or subclassed value" + + +def _normalize_value( # noqa: C901 + value: Any, + budget: _ResultBudget, + *, + enum_types: frozenset[type[Any]] = frozenset(), + metadata: bool = False, +) -> tuple[Any, str | None]: + """Iteratively normalize one bounded exact-container value tree.""" + stack: list[ + tuple[Any, list[Any] | dict[str, Any] | None, int | str | None, int, bool] + ] = [(value, None, None, 0, False)] + active_containers: set[int] = set() + root = value + + while stack: + item, parent, slot, depth, leaving = stack.pop() + if leaving: + active_containers.remove(id(item)) + continue + if depth > MAX_RESULT_VALUE_DEPTH: + return None, "excessively nested data" + if reason := _charge_value(budget, metadata=metadata): + return None, reason + + if type(item) is list: + identity = id(item) + if identity in active_containers: + return None, "cyclic containers" + active_containers.add(identity) + width = list.__len__(item) + if width > MAX_RESULT_VALUE_ITEMS: + return None, "an oversized array" + if reason := _charge_json_bytes( + budget, + _container_json_syntax_size(width, mapping=False), + metadata=metadata, + ): + return None, reason + stack.append((item, None, None, depth, True)) + stack.extend( + (list.__getitem__(item, index), item, index, depth + 1, False) + for index in range(width - 1, -1, -1) + ) + continue + + if type(item) is dict: + identity = id(item) + if identity in active_containers: + return None, "cyclic containers" + active_containers.add(identity) + width = dict.__len__(item) + if width > MAX_RESULT_VALUE_ITEMS: + return None, "an oversized object" + if reason := _charge_json_bytes( + budget, + _container_json_syntax_size(width, mapping=True), + metadata=metadata, + ): + return None, reason + children: list[tuple[Any, dict[str, Any], str, int, bool]] = [] + for key, child in dict.items(item): + if type(key) is not str: + return None, "a non-string object key" + if reason := _charge_text(key, budget, key=True, metadata=metadata): + return None, reason + children.append((child, item, key, depth + 1, False)) + stack.append((item, None, None, depth, True)) + stack.extend(reversed(children)) + continue + + source_item = item + if type(item) in enum_types: + normalized = _safe_enum_value(item, enum_types) + if normalized is None: + return None, "an unsupported enum" + item = normalized + elif any(base is Enum for base in _type_mro(type(item))): + return None, "an enum outside its expected metadata slot" + + normalized, reason = _normalize_scalar(item) + if reason is not None: + return None, reason + max_string_bytes = ( + MAX_QUERY_RESULT_METADATA_BYTES if metadata else MAX_RESULT_STRING_LENGTH + ) + if type(normalized) is str: + scalar_size = _json_string_size(normalized, max_string_bytes) + if scalar_size is None: + return None, "invalid text data" + else: + scalar_size = _normalized_scalar_json_size(normalized) + if reason := _charge_json_bytes( + budget, + scalar_size, + metadata=metadata, + ): + return None, reason + if parent is None: + root = normalized + elif normalized is not source_item: + if type(parent) is list: + assert type(slot) is int + list.__setitem__(parent, slot, normalized) + else: + assert type(parent) is dict + assert type(slot) is str + dict.__setitem__(parent, slot, normalized) + return root, None + + +def _normalize_metadata_value( + payload: dict[str, Any], key: str, budget: _ResultBudget +) -> str | None: + enum_slots: dict[str, frozenset[type[Any]]] = { + "status": frozenset({QueryStatus}), + "result_format": frozenset({ChartDataResultFormat}), + "coltypes": frozenset({GenericDataType}), + "applied_filters": frozenset({ExtraFiltersTimeColumnType}), + "rejected_filters": frozenset( + {ExtraFiltersReasonType, ExtraFiltersTimeColumnType} + ), + } + value = dict.__getitem__(payload, key) + normalized, reason = _normalize_value( + value, + budget, + enum_types=enum_slots.get(key, frozenset()), + metadata=True, + ) + if reason is None and normalized is not value: + dict.__setitem__(payload, key, normalized) + return reason + + +def _error_text(value: Any) -> str | None: + """Extract a bounded error from an already validated primitive tree.""" + stack: list[Any] = [value] + parts: list[str] = [] + used = 0 + while stack and len(parts) < 3 and used < _MAX_ERROR_TEXT_BYTES: + item = stack.pop() + if type(item) is dict: + stack.extend( + reversed( + [dict.__getitem__(item, key) for key in _ERROR_KEYS if key in item] + ) + ) + continue + if type(item) is list: + stack.extend( + list.__getitem__(item, index) + for index in range(list.__len__(item) - 1, -1, -1) + ) + continue + if item is None or item is False: + continue + if type(item) is str: + remaining = _MAX_ERROR_TEXT_BYTES - used - (2 if parts else 0) + encoded = str.encode(item, "utf-8")[:remaining] + text = bytes.decode(encoded, "utf-8", errors="ignore") + if text: + parts.append(text) + used += bytes.__len__(encoded) + (2 if len(parts) > 1 else 0) + return "; ".join(parts) or None + + +def _failure_for_payload(payload: dict[str, Any], label: str) -> ChartError | None: + for key in ("error", "errors", "error_message"): + if key in payload and (message := _error_text(dict.__getitem__(payload, key))): + return ChartError( + error=f"{label} failed: {message}", error_type="QueryError" + ) + + raw_status = dict.get(payload, "status") + status = raw_status if type(raw_status) is str else "" + normalized_status = status.strip().casefold().replace("-", "_").replace(" ", "_") + if normalized_status in FAILED_QUERY_STATUSES: + message = ( + _error_text(dict.get(payload, "message")) + or _error_text(dict.get(payload, "error_message")) + or normalized_status + ) + return ChartError(error=f"{label} failed: {message}", error_type="QueryError") + if dict.get(payload, "success") is False: + message = _error_text(dict.get(payload, "message")) or "request failed" + return ChartError(error=f"{label} failed: {message}", error_type="QueryError") + if raw_status is None and "data" not in payload and "queries" not in payload: + if message := _error_text(dict.get(payload, "message")): + return ChartError( + error=f"{label} failed: {message}", error_type="QueryError" + ) + return None + + +def _metadata_shape_error( # noqa: C901 + payload: dict[str, Any], label: str, budget: _ResultBudget +) -> ChartError | None: + if "success" in payload and type(dict.__getitem__(payload, "success")) is not bool: + return _invalid_metadata(label) + if "status" in payload and type(dict.__getitem__(payload, "status")) is not str: + return _invalid_metadata(label) + if ( + "result_format" in payload + and dict.__getitem__(payload, "result_format") not in _RESULT_FORMAT_VALUES + ): + return _invalid_metadata(label) + for count_key in ("rowcount", "sql_rowcount", "total_rows"): + if count_key in payload: + count = dict.__getitem__(payload, count_key) + if count is not None and not ( + type(count) is int and 0 <= count <= MAX_QUERY_RESULT_ROWCOUNT + ): + return _invalid_metadata(label) + if "is_cached" in payload: + cached = dict.__getitem__(payload, "is_cached") + if cached is None: + if reason := _charge_json_bytes( + budget, 1, metadata=True + ): # ``false`` vs ``null`` + return _invalid_result(reason) + dict.__setitem__(payload, "is_cached", False) + elif type(cached) is not bool: + return _invalid_metadata(label) + if "cache_timeout" in payload: + timeout = dict.__getitem__(payload, "cache_timeout") + if timeout is not None and not ( Review Comment: This rejects Superset's supported `cache_timeout = -1` sentinel, which `QueryContextProcessor` includes whenever caching is disabled. Any MCP chart query using that valid setting now fails the new result-envelope check; could the validator allow `CACHE_DISABLED_TIMEOUT`? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
