aminghadersohi commented on code in PR #43870:
URL: https://github.com/apache/superset/pull/43870#discussion_r4162516989
##########
superset-frontend/src/explore/components/controls/FilterControl/AdhocFilterOption/AdhocFilterOption.test.tsx:
##########
@@ -108,3 +108,115 @@ test('should render the Popover on clicking the right
caret', async () => {
await userEvent.click(rightCaret);
expect(screen.getByRole('tooltip')).toBeInTheDocument();
});
+
+// A monotonic transform, so ranges mirror alongside equality.
+const MONOTONIC_MAPPING = {
+ partition_column: 'dt_epoch',
+ mapped_column: 'value',
+ active: true,
+ is_monotonic: true,
+ mirrorable_operators: ['<', '<=', '==', '>', '>=', 'IN', 'TEMPORAL_RANGE'],
+};
+
+const withMapping = (
+ mapping: Record<string, unknown> | null,
+ adhocFilter = simpleAdhocFilter,
+) =>
+ setup({
+ ...mockedProps,
+ adhocFilter,
+ datasource: { partition_filter_mapping: mapping },
+ });
+
+const glyph = () => screen.queryByTestId('partition-pruning-indicator');
+
+test('a filter on the mapped column carries the partition pruning glyph', ()
=> {
+ // Wireframe 1d: the chart author never configured any of this and only sees
+ // an explanation of why the query got faster.
+ render(withMapping(MONOTONIC_MAPPING));
+
+ expect(glyph()).toBeInTheDocument();
+});
+
+test('a filter on any other column carries no glyph', () => {
+ render(
+ withMapping({ ...MONOTONIC_MAPPING, mapped_column: 'some_other_column' }),
+ );
+
+ expect(glyph()).not.toBeInTheDocument();
+});
+
+test('an inactive mapping leaves the chip alone', () => {
+ render(withMapping({ ...MONOTONIC_MAPPING, active: false }));
+
+ expect(glyph()).not.toBeInTheDocument();
+});
+
+test('a negated filter on the mapped column carries no glyph', () => {
+ // The query path never mirrors a negation -- the transform need not be
+ // injective, so `value != 10` would mirror to a predicate that drops rows
the
+ // original filter keeps. Labelling it would promise a speed-up the SQL does
+ // not contain.
+ render(
+ withMapping(
+ MONOTONIC_MAPPING,
+ simpleAdhocFilter.duplicateWith({ operator: '!=' }),
+ ),
+ );
+
+ expect(glyph()).not.toBeInTheDocument();
+});
+
+test('a range filter waits on the monotonicity declaration', () => {
+ render(
+ withMapping({
+ ...MONOTONIC_MAPPING,
+ is_monotonic: false,
+ mirrorable_operators: ['==', 'IN'],
+ }),
+ );
+
+ expect(glyph()).not.toBeInTheDocument();
+});
+
+test('an equality filter mirrors under a non-monotonic transform', () => {
+ render(
+ withMapping(
+ {
+ ...MONOTONIC_MAPPING,
+ is_monotonic: false,
+ mirrorable_operators: ['==', 'IN'],
+ },
+ simpleAdhocFilter.duplicateWith({ operator: '==' }),
+ ),
+ );
+
+ expect(glyph()).toBeInTheDocument();
+});
+
+test('a free-form SQL filter on the mapped column carries no glyph', () => {
+ // It lands in `extras.where`, where the backend sees no operator to mirror.
+ render(
+ withMapping(
+ MONOTONIC_MAPPING,
+ new AdhocFilter({
+ expressionType: ExpressionTypes.Sql,
+ sqlExpression: 'value > 10',
+ clause: Clauses.Where,
+ }),
+ ),
+ );
+
+ expect(glyph()).not.toBeInTheDocument();
+});
+
+test('a filter with no comparator yet carries no glyph', () => {
+ render(
+ withMapping(
+ MONOTONIC_MAPPING,
+ simpleAdhocFilter.duplicateWith({ comparator: '' }),
+ ),
+ );
+
+ expect(glyph()).not.toBeInTheDocument();
+});
Review Comment:
Red at head: `1e132aefc9` flipped the `''` case in
`PartitionPruningIndicator.test.tsx` but not this sibling, and `isValid()`
accepts `''`, so the glyph is right and this test is stale. With the flip below
the file is 14/14.
```suggestion
test('an empty-string comparator still carries the glyph', () => {
render(
withMapping(
MONOTONIC_MAPPING,
simpleAdhocFilter.duplicateWith({ comparator: '' }),
),
);
expect(glyph()).toBeInTheDocument();
});
```
##########
superset/connectors/sqla/partition_mapping.py:
##########
@@ -0,0 +1,1278 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Partition filter mapping.
+
+Datasets on Hadoop-family engines are commonly partitioned on a *technical*
+column -- an epoch integer, a lowercased region key -- that no analyst would
+filter on. Unless a query carries a predicate on that column the engine scans
+every partition.
+
+A dataset owner names one partition column ``p``, one business column that
+filters are mirrored from, and a value transform ``T`` (a SQL expression
+containing a ``:value`` placeholder). Superset then appends an equivalent
+predicate on ``p`` to every query, so chart authors change nothing and queries
+prune.
+
+The load-bearing assumption
+---------------------------
+Everything here reasons about ``T(col) op T(v)``, but what is emitted is
+``p op T(v)`` -- a predicate on a *physically different column*. The step from
+one to the other is::
+
+ p = T(mapped_col) for every row in the table
+
+Superset cannot verify that; it is a property of whatever ETL populates the
+partition column. If that job lags, backfills with different logic, or writes
+the partition key in a different timezone, mirrored predicates silently drop
+real rows. The mapping is only as trustworthy as the pipeline behind it.
+
+Operator safety
+---------------
+A mirrored predicate ``P2`` may only be ``AND``-ed onto a query when the
+original predicate ``P1`` *implies* it:
+
+=========================================== =============================
+Original Safe when
+=========================================== =============================
+``col = v``, ``col IN (...)`` always -- ``T`` is a function
+``col >=|>|<|<= v``, ``TEMPORAL_RANGE`` only if ``T`` is monotonic
+``col != v``, ``NOT IN``, ``LIKE``, ... never
+=========================================== =============================
+
+Negations are never safe because ``T`` need not be injective:
+``lower(:value)`` with ``country != 'US'`` mirrors to ``region_key != 'us'``,
+which wrongly excludes rows whose ``country`` is already lowercase ``'us'`` --
+rows the original filter *keeps*.
+
+Monotonicity is a property of the transform, not of the column's data type:
+``hour(:value)``, ``date_format(:value, 'dd')`` and ``dayofweek(:value)`` are
+all reasonable transforms on a ``TIMESTAMP`` column and none of them preserve
+ordering. It is therefore declared by the owner, not inferred.
+
+Time grains
+-----------
+A grained filter -- drill-to-detail, mostly -- compares the *truncated* column,
+``trunc(col) op v``, so the raw bounds it carries do not describe the rows it
+keeps. A row in the final partial bucket satisfies ``trunc(col) < until`` while
+``col < until`` excludes it.
+
+Which direction a grain rounds is not knowable from the duration alone, and the
+obvious guess is wrong: ``WEEK_ENDING_SATURDAY`` rounds *forward* on Hive and
+Presto, and Ocient's grains are ``ROUND``, i.e. to nearest. What every grain
+does satisfy is a bound on the displacement::
+
+ |trunc(ts) - ts| < width(grain)
+
+so widening *both* bounds by one bucket width is no narrower than the real
+predicate whichever way the grain rounds -- and it needs no monotonicity of
+``trunc`` itself, which is what rescues Hive's oddly-anchored ``P1W``. Width is
+a property of the grain, not of the engine, which is what makes this
+maintainable; see `grain_bucket_width`. A grain whose SQL an operator supplied
+has no known width, and does not mirror at all.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import re
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Any, cast, TYPE_CHECKING
+
+import numpy as np
+import pandas as pd
+import sqlalchemy as sa
+from dateutil.relativedelta import relativedelta
+from flask import current_app as app
+from flask_babel import lazy_gettext as _
+from sqlalchemy.engine.interfaces import Dialect
+from sqlalchemy.sql.elements import ColumnElement
+
+from superset.constants import LRU_CACHE_MAX_SIZE, TimeGrain
+from superset.exceptions import (
+ QueryClauseValidationException,
+ SupersetParseError,
+ SupersetSecurityException,
+)
+from superset.extensions import cache_manager, feature_flag_manager
+from superset.sql.parse import SQLStatement
+from superset.utils import json
+from superset.utils.core import FilterOperator
+
+if TYPE_CHECKING:
+ from superset.connectors.sqla.models import SqlaTable, TableColumn
+ from superset.models.core import Database
+
+logger = logging.getLogger(__name__)
+
+FEATURE_FLAG = "PARTITION_FILTER_MAPPING"
+
+#: Longest transform accepted anywhere. A transform is one SQL expression an
+#: owner types by hand, so this is generous; the point is that it is bounded.
+#: Every entry point enforces it -- the typed column field, the import schema
+#: and the preview request -- because `is_transform_active` parses the stored
+#: value on each Explore load, and an unbounded string would make that parse
+#: the expensive part of rendering a chart.
+MAX_TRANSFORM_LENGTH = 1024
+
+#: Placeholder the owner writes in the transform, e.g.
``unix_timestamp(:value)``.
+#: Matched with word boundaries so ``:values`` is not mistaken for it.
+VALUE_PLACEHOLDER_RE = re.compile(r":value\b")
+
+#: Balanced Jinja blocks. The probe would render these in a different context
+#: at a different time from the chart query, so they are rejected at save time.
+JINJA_BLOCK_RE = re.compile(r"\{\{.*?\}\}|\{%.*?%\}|\{#.*?#\}", re.DOTALL)
+
+#: Substituted for ``:value`` before parsing -- sqlglot rejects a bare
``:value``
+#: on most dialects. Mirrors the ``_JINJA_BLOCK_RE`` -> ``NULL`` trick used by
+#: ``validate_stored_expression``.
+_PARSE_STANDIN = "NULL"
+
+#: Functions whose value depends on wall-clock time or randomness. The probe
+#: runs in a different session at a different moment from the chart query and
+#: its result is then cached, so any of these freezes a snapshot of probe time
+#: into the emitted predicate.
+NON_DETERMINISTIC_FUNCTIONS = {
+ "CURRENT_DATE",
+ "CURRENT_TIME",
+ "CURRENT_TIMESTAMP",
+ "NOW",
+ "RAND",
+ "RANDOM",
+ "UUID",
+}
+
+#: Functions that mean "now" only in their zero-argument form. On Hive and
+#: Impala ``unix_timestamp()`` is the current time while ``unix_timestamp(x)``
+#: -- the canonical transform for this feature -- is pure.
+NON_DETERMINISTIC_WHEN_NILADIC = {"UNIX_TIMESTAMP"}
+
+#: Safe for any function ``T``.
+MIRRORABLE_ALWAYS = {FilterOperator.EQUALS, FilterOperator.IN}
+
+#: Safe only when ``T`` preserves ordering.
+MIRRORABLE_IF_MONOTONIC = {
+ FilterOperator.GREATER_THAN,
+ FilterOperator.GREATER_THAN_OR_EQUALS,
+ FilterOperator.LESS_THAN,
+ FilterOperator.LESS_THAN_OR_EQUALS,
+ FilterOperator.TEMPORAL_RANGE,
+}
+
+
+#: Every operator that can be mirrored under *some* transform. The preview
+#: endpoint accepts these; whether a given one actually mirrors still depends
on
+#: the monotonicity declaration.
+MIRRORABLE_OPERATORS = MIRRORABLE_ALWAYS | MIRRORABLE_IF_MONOTONIC
+
+#: Operators the preview endpoint can construct. `TEMPORAL_RANGE` is mirrored
by
+#: the query path, but as *two* bounds that `_collect_partition_mirror_range`
+#: decomposes a time range into -- the preview request carries sample values,
not
+#: a since/until pair, so there is no range for it to build. The editor never
+#: asks for one either: `previewOperatorFor` sends `>=`, `=` or `IN`.
+PREVIEWABLE_OPERATORS = MIRRORABLE_OPERATORS - {FilterOperator.TEMPORAL_RANGE}
+
+
+def mirrorable_operators(is_monotonic: bool) -> set[FilterOperator]:
+ """
+ The operators whose predicates may be mirrored onto the partition column.
+
+ :param is_monotonic: whether the owner declared the transform
+ order-preserving
+ """
+ if is_monotonic:
+ return MIRRORABLE_ALWAYS | MIRRORABLE_IF_MONOTONIC
+ return set(MIRRORABLE_ALWAYS)
+
+
+#: How wide one bucket of each built-in time grain is.
+#:
+#: Keyed on the ISO duration a filter carries in its ``grain``. Written out
+#: rather than parsed: four of the week grains are ISO *intervals* with an
+#: anchor (``P1W/1970-01-03T00:00:00Z``) that ``isodate.parse_duration``
+#: rejects outright, and ``PT0.5H`` / ``P0.25Y`` are fractional. A literal
+#: table is also the thing a reviewer can check a line at a time.
+#:
+#: ``relativedelta`` rather than ``timedelta`` so the calendar grains stay
+#: calendar arithmetic: a month is not 30 days.
+GRAIN_BUCKET_WIDTHS: dict[str, relativedelta] = {
+ TimeGrain.SECOND: relativedelta(seconds=1),
+ TimeGrain.FIVE_SECONDS: relativedelta(seconds=5),
+ TimeGrain.THIRTY_SECONDS: relativedelta(seconds=30),
+ TimeGrain.MINUTE: relativedelta(minutes=1),
+ TimeGrain.FIVE_MINUTES: relativedelta(minutes=5),
+ TimeGrain.TEN_MINUTES: relativedelta(minutes=10),
+ TimeGrain.FIFTEEN_MINUTES: relativedelta(minutes=15),
+ TimeGrain.THIRTY_MINUTES: relativedelta(minutes=30),
+ TimeGrain.HALF_HOUR: relativedelta(minutes=30),
+ TimeGrain.HOUR: relativedelta(hours=1),
+ TimeGrain.SIX_HOURS: relativedelta(hours=6),
+ TimeGrain.DAY: relativedelta(days=1),
+ TimeGrain.WEEK: relativedelta(days=7),
+ TimeGrain.WEEK_STARTING_SUNDAY: relativedelta(days=7),
+ TimeGrain.WEEK_STARTING_MONDAY: relativedelta(days=7),
+ TimeGrain.WEEK_ENDING_SATURDAY: relativedelta(days=7),
+ TimeGrain.WEEK_ENDING_SUNDAY: relativedelta(days=7),
+ TimeGrain.MONTH: relativedelta(months=1),
+ TimeGrain.QUARTER: relativedelta(months=3),
+ TimeGrain.QUARTER_YEAR: relativedelta(months=3),
+ TimeGrain.YEAR: relativedelta(years=1),
+}
+
+
+def grain_bucket_width(grain: str | None, engine: str) -> relativedelta | None:
+ """
+ How far a grain's truncation can move a timestamp, or ``None`` if unknown.
+
+ A grained filter compares the *truncated* column, so the raw bounds it
+ carries do not describe the rows it keeps. Widening both bounds by one
+ bucket recovers a predicate that is no narrower than the real one -- see
+ `_collect_partition_mirror_range` for the argument. That only works for a
+ grain whose bucket width Superset knows, which excludes anything an
+ operator supplied.
+
+ :param grain: the ISO duration from the filter's ``grain``
+ :param engine: the engine spec's ``engine``, to check per-engine overrides
+ """
+ if not grain:
+ return None
+
+ # An operator-declared grain carries whatever duration string they typed,
+ # and `TIME_GRAIN_ADDON_EXPRESSIONS` can redefine a *built-in* grain's SQL
+ # per engine -- `P1D` could be anything at all. Neither has a width we can
+ # claim to know, so both fall back to not mirroring.
+ if grain in app.config["TIME_GRAIN_ADDONS"]:
+ return None
+ if grain in app.config["TIME_GRAIN_ADDON_EXPRESSIONS"].get(engine, {}):
+ return None
+
+ return GRAIN_BUCKET_WIDTHS.get(grain)
+
+
+@dataclass(frozen=True)
+class PartitionMapping:
+ """A resolved, usable partition filter mapping."""
+
+ partition_column: str
+ mapped_column: str
+ value_transform: str
+ is_monotonic: bool
+
+ def mirrors(self, operator: FilterOperator) -> bool:
+ return operator in mirrorable_operators(self.is_monotonic)
+
+
+def contains_value_placeholder(transform: str | None) -> bool:
+ """Whether the transform contains the ``:value`` placeholder."""
+ return bool(transform) and VALUE_PLACEHOLDER_RE.search(transform or "") is
not None
+
+
+def contains_jinja(transform: str | None) -> bool:
+ """Whether the transform contains a balanced Jinja block."""
+ return bool(transform) and JINJA_BLOCK_RE.search(transform or "") is not
None
+
+
+def parse_skeleton(transform: str) -> str:
+ """
+ The transform with ``:value`` substituted out, ready for a SQL parser.
+
+ ``sanitize_clause`` / sqlglot choke on a bare ``:value`` on most dialects,
+ so the placeholder is swapped for a benign literal first -- the same trick
+ ``validate_stored_expression`` uses for Jinja blocks.
+ """
+ return VALUE_PLACEHOLDER_RE.sub(_PARSE_STANDIN, transform)
+
+
+#: Prefix the transform is wrapped in before parsing. Its length is subtracted
+#: from any reported column so positions refer to what the owner actually
typed.
+_SELECT_PREFIX = "SELECT "
+
+
+def _parse_skeleton(transform: str, engine: str) -> SQLStatement | None:
+ """
+ Parse ``SELECT <transform>`` with the placeholder substituted out.
+
+ Returns ``None`` when the transform does not parse.
+ """
+ try:
+ return SQLStatement(f"{_SELECT_PREFIX}{parse_skeleton(transform)}",
engine)
+ except SupersetParseError:
+ return None
+
+
+def parse_error_detail(transform: str, engine: str) -> str | None:
+ """
+ Where the parser gave up on the transform.
+
+ Returns ``None`` when it parses, or when the parser offered no position.
+ Note that sqlglot parses unknown functions happily -- a misspelled function
+ name is not a parse error, it is an engine error, and surfaces only when
the
+ transform is evaluated.
+
+ The parser's own ``highlight`` is deliberately dropped: it would name the
+ ``NULL`` we substituted for ``:value``, which is not a token the owner
typed.
+ """
+ try:
+ SQLStatement(f"{_SELECT_PREFIX}{parse_skeleton(transform)}", engine)
+ except SupersetParseError as ex:
+ column = (ex.error.extra or {}).get("column")
+ if not isinstance(column, int):
+ return None
+ return str(
+ _(
+ "syntax error at position %(position)d.",
+ position=_position_in_transform(transform, column),
+ )
+ )
+ return None
+
+
+def _position_in_transform(transform: str, parsed_column: int) -> int:
+ """
+ Map a column in the parsed skeleton back to the transform as typed.
+
+ Two substitutions stand between them: the ``SELECT`` prefix, and every
+ ``:value`` that became a shorter ``NULL``. Without unwinding both, a
+ reported position drifts left by two characters per placeholder ahead of it
+ -- which is worst exactly where transforms usually break, at the end.
+ """
+ position = max(parsed_column - len(_SELECT_PREFIX), 0)
+ shift = len(":value") - len(_PARSE_STANDIN)
+ preceding = sum(
+ 1
+ for index, match in enumerate(VALUE_PLACEHOLDER_RE.finditer(transform))
+ if match.start() - index * shift < position
+ )
+ return position + preceding * shift
+
+
+def is_parseable(transform: str | None, engine: str) -> bool:
+ """
+ Whether the transform parses as a single select expression.
+
+ "Single" is the load-bearing word. `SELECT lower(:value), 'x'` parses just
+ as happily as `SELECT lower(:value)`, but it returns two columns per input
+ value, and the probe reads one column per input -- so an `IN` filter would
+ get a predicate built from the wrong halves of the wrong rows. Rejecting
the
+ list here is what lets the probe trust its own column count.
+ """
+ if not transform or not transform.strip():
+ return False
+ statement = _parse_skeleton(transform, engine)
+ return statement is not None and statement.count_select_expressions() == 1
+
+
+def find_non_deterministic_functions(transform: str, engine: str) -> set[str]:
+ """
+ Names of non-deterministic functions the transform calls.
+
+ ``UNIX_TIMESTAMP`` is only reported in its zero-argument form, which means
+ "now" on Hive and Impala; the one-argument form is the canonical temporal
+ transform and stays allowed.
+ """
+ statement = _parse_skeleton(transform, engine)
+ if statement is None:
+ return set()
+
+ found = {
+ name
+ for name in NON_DETERMINISTIC_FUNCTIONS
+ if statement.check_functions_present({name})
+ }
+ return found | _find_niladic_calls(statement)
+
+
+def _find_niladic_calls(statement: SQLStatement) -> set[str]:
+ """
+ Names from ``NON_DETERMINISTIC_WHEN_NILADIC`` called with no arguments.
+
+ Note some dialects resolve the zero-argument form themselves -- Hive parses
+ ``unix_timestamp()`` straight to ``CURRENT_TIMESTAMP`` -- in which case the
+ name-based check above has already caught it. This is the backstop for the
+ dialects that do not.
+ """
+ return NON_DETERMINISTIC_WHEN_NILADIC & statement.get_niladic_functions()
+
+
+def resolve_partition_mapping(datasource: SqlaTable) -> PartitionMapping |
None:
+ """
+ Resolve the dataset's mapping, or ``None`` when nothing may be mirrored.
+
+ Every bail-out here is defensive as well as functional: save-time
validation
+ rejects most of these, but rows predating the validation can still violate
+ the invariants, and a column sync can invalidate a mapping that was fine
+ when it was written.
+
+ The transform gate is `is_transform_active`, the same function the Explore
+ indicator reads, which is `validate_transform` with the messages discarded.
+ Anything narrower here would be a second, weaker statement of the same
rule:
+ a transform calling `now()` that reached storage without passing
+ `UpdateDatasetCommand` -- through import, or through a bundle written by
hand
+ -- would be reported inactive by the editor and still mirrored by this
+ function, freezing a snapshot of probe time into the predicate with nothing
+ on screen to say so.
+ """
+ if not feature_flag_manager.is_feature_enabled(FEATURE_FLAG):
+ return None
+
+ partition_column = getattr(datasource, "partition_column", None)
+ if not partition_column:
+ return None
+
+ columns_by_name = {column.column_name: column for column in
datasource.columns}
+ if partition_column not in columns_by_name:
+ # The partition column was dropped by a column sync or at the source.
+ return None
+
+ mapped_column_name = (
+ getattr(datasource, "partition_mapped_column", None) or
datasource.main_dttm_col
+ )
+ if not mapped_column_name or mapped_column_name not in columns_by_name:
+ return None
+
+ if mapped_column_name == partition_column:
+ # Self-mapping: the mirrored predicate would duplicate the original.
+ return None
+
+ mapped_column = columns_by_name[mapped_column_name]
+ transform = getattr(mapped_column, "partition_value_transform", None)
+ if not is_transform_active(transform, datasource.database.backend):
+ return None
+
+ if has_active_advanced_data_type(mapped_column):
+ # `translate_filter` builds its own predicate shape from *translated*
+ # values, so the `(operator, value)` pair the operator matrix reasons
+ # about does not exist and mirroring would apply the wrong values.
+ return None
+
+ return PartitionMapping(
+ partition_column=str(partition_column),
+ mapped_column=str(mapped_column_name),
+ value_transform=cast(str, transform),
+ is_monotonic=bool(
+ getattr(mapped_column, "partition_transform_is_monotonic", False)
+ ),
+ )
+
+
+def has_active_advanced_data_type(column: TableColumn) -> bool:
+ """
+ Whether the column's advanced data type is configured and switched on.
+
+ Such a column is never mirrored: ``translate_filter`` builds its own
+ predicate shape from translated values, so the ``(operator, value)`` pair
+ the operator matrix reasons about does not exist.
+ """
+ advanced_data_type = getattr(column, "advanced_data_type", None)
+ if not advanced_data_type:
+ return False
+ if not
feature_flag_manager.is_feature_enabled("ENABLE_ADVANCED_DATA_TYPES"):
+ return False
+ return advanced_data_type in app.config.get("ADVANCED_DATA_TYPES", {})
+
+
+def build_probe_sql(
+ transform: str,
+ values: list[Any],
+ dialect: Dialect | None = None,
+ from_suffix: str = "",
+) -> str:
+ """
+ Compile a single ``SELECT`` that evaluates the transform at every value.
+
+ Values are attacker-controlled (a Gamma user picks filter values), so they
+ are bound as parameters and rendered by the dialect's own literal processor
+ rather than interpolated into the SQL text.
+
+ ``from_suffix`` comes from the engine spec's
``select_without_from_suffix``.
+ A ``SELECT`` with no ``FROM`` is not universal SQL: Oracle and Db2 need a
+ one-row table to select from, and without it every probe on those engines
+ raises -- which `_run_probe` swallows, so the only symptom is a correctly
+ configured mapping that silently never prunes.
+
+ Note this deliberately does *not* go through ``BaseEngineSpec``'s text
+ helper, which escapes ``:`` on every engine but Athena and would destroy
the
+ ``:value`` placeholder before it can be bound.
+ """
+ selections = []
+ for index, value in enumerate(values):
+ clause = sa.text(transform).bindparams(sa.bindparam("value",
value=value))
+ compiled = clause.compile(
+ dialect=dialect,
+ compile_kwargs={"literal_binds": True},
+ )
+ selections.append(f"{compiled} AS v{index}")
+ return "SELECT " + ", ".join(selections) + from_suffix
+
+
+def evaluate_transform(
+ database: Database,
+ catalog: str | None,
+ schema: str | None,
+ transform: str,
+ values: list[Any],
+ *,
+ errors: list[str] | None = None,
+) -> list[Any] | None:
+ """
+ Evaluate ``transform`` against the engine once per distinct value.
+
+ Returns one result per input value, positionally aligned with ``values``,
or
+ ``None`` if anything at all goes wrong. Failing open costs pruning, never
+ correctness: the chart query still runs, it just scans more partitions.
+
+ The probe is pinned to the dataset's catalog and schema so session settings
+ match the chart query as closely as the connection pool allows. It still
+ runs in a *different* session, which is why transforms calling
+ session-dependent functions are rejected at save time.
+
+ :param errors: optional sink for the engine's own account of a failure. The
+ query path passes nothing and stays silent; the editor's preview passes
+ a list so it can tell the owner *why* the transform did not evaluate --
+ a misspelled function is the common case and sqlglot parses it happily.
+ """
+ if not values:
+ return None
+
+ # Dedupe so a 200-value `IN` list costs one column, not 200.
+ distinct: list[Any] = []
+ seen: set[Any] = set()
+ for value in values:
+ key = _hashable(value)
+ if key not in seen:
+ seen.add(key)
+ distinct.append(value)
+
+ cache_key = _probe_cache_key(database, catalog, schema, transform,
distinct)
+ cached = _cache_get(cache_key)
+ if cached is None:
+ cached = _run_probe(
+ database, catalog, schema, transform, distinct, errors=errors
+ )
+ if cached is None:
+ # Deliberately not cached: a transient engine blip would otherwise
+ # keep the dataset pruning-free for the whole cache timeout.
+ return None
+ _cache_set(cache_key, cached)
+
+ evaluated = dict(
+ zip((_hashable(value) for value in distinct), cached, strict=False)
+ )
+ return [evaluated[_hashable(value)] for value in values]
+
+
+def _run_probe(
+ database: Database,
+ catalog: str | None,
+ schema: str | None,
+ transform: str,
+ distinct: list[Any],
+ *,
+ errors: list[str] | None = None,
+) -> list[Any] | None:
+ try:
+ sql = build_probe_sql(
+ transform,
+ distinct,
+ _dialect_for(database),
+ database.db_engine_spec.select_without_from_suffix,
+ )
+ frame = database.get_df(sql=sql, catalog=catalog, schema=schema)
+ if frame is None or frame.empty:
+ logger.warning(
+ "Partition transform probe returned no rows; skipping
mirroring"
+ )
+ return None
+ row = frame.iloc[0]
+ if len(row) != len(distinct):
+ # The results cannot be aligned back to their inputs; skipping
+ # beats guessing which value produced which column. Too *many*
+ # columns is the dangerous direction and the reason this is `!=`
+ # rather than `<`: a transform whose select list holds two
+ # expressions returns 2N columns for N inputs, and reading the
+ # first N of them interleaves the expressions instead of selecting
+ # one per value -- a predicate built from the wrong values rather
+ # than one that is merely short. `is_parseable` rejects that
+ # transform before it is ever stored; this is the backstop for one
+ # that predates the check.
+ logger.warning(
+ "Partition transform probe returned %d values for %d inputs",
+ len(row),
+ len(distinct),
+ )
+ return None
+ return [_to_python_scalar(row.iloc[index]) for index in
range(len(distinct))]
+ except Exception as ex: # pylint: disable=broad-except
+ logger.warning(
+ "Partition transform probe failed; queries will not prune",
+ exc_info=True,
+ )
+ if errors is not None:
+ errors.append(str(ex))
+ return None
+
+
+def _to_python_scalar(value: Any) -> Any:
+ """
+ A probed value as a type SQLAlchemy can render and bind.
+
+ The probe reads its results out of a pandas frame, so a numeric column
+ arrives as a `numpy.int64` and a temporal one as a `pandas.Timestamp`.
+ SQLAlchemy has no literal renderer for either: `sa.literal(np.int64(...))`
+ infers `NullType` and raises `CompileError`, which is a 500 from the
preview
+ endpoint for the canonical epoch transform -- the most ordinary mapping
+ there is.
+
+ Pandas' missing-value sentinels are folded into `None` on the way through.
+ `NaT` is not a `Timestamp` and has no renderer either, and a transform that
+ returns nothing for an input it cannot convert is exactly the NULL case the
+ mirror already admits.
+ """
+ if value is None:
+ return None
+ try:
+ if pd.isna(value):
+ return None
+ except (TypeError, ValueError):
+ # Not something `isna` can judge (a list, a dict); pass it through and
+ # let the dialect decide.
+ pass
+ if isinstance(value, pd.Timestamp):
+ return value.to_pydatetime()
+ if isinstance(value, np.generic):
+ return value.item()
+ return value
+
+
+def _probe_cache_key(
+ database: Database,
+ catalog: str | None,
+ schema: str | None,
+ transform: str,
+ values: list[Any],
+) -> str:
+ """
+ Key on everything that can change the answer.
+
+ Note this cache is independent of the chart-data cache: it is keyed on the
+ transform and its inputs, so it is correct to share across every chart on
+ every dataset that happens to use the same transform.
+ """
+ payload = json.dumps(
+ [
+ database.id,
+ database.backend,
+ # The probe asks this connection to evaluate the transform, so the
+ # answer belongs to it. `id` outlives an edit to the URI or to
+ # `extra` (a session timezone, say), which would otherwise serve
+ # values computed against the old environment until the entry
+ # expires. `changed_on` covers the edits that do not show up here,
+ # such as a rotated password, without putting a secret in the key.
+ database.sqlalchemy_uri,
+ database.extra,
+ str(database.changed_on),
+ catalog,
+ schema,
+ transform,
+ [repr(value) for value in values],
+ ],
+ default=repr,
+ )
+ digest = hashlib.md5(payload.encode("utf-8")).hexdigest() # noqa: S324
+ return f"partition_transform_probe:{digest}"
+
+
+def _cache_get(key: str) -> list[Any] | None:
+ try:
+ return cache_manager.cache.get(key)
+ except Exception: # pylint: disable=broad-except # noqa: BLE001
+ return None
+
+
+def _cache_set(key: str, value: list[Any]) -> None:
+ timeout = app.config.get("PARTITION_TRANSFORM_PROBE_CACHE_TIMEOUT", 24 *
60 * 60)
+ try:
+ cache_manager.cache.set(key, value, timeout=timeout)
+ except Exception: # pylint: disable=broad-except
+ logger.warning("Could not cache partition transform probe",
exc_info=True)
+
+
+def _dialect_for(database: Database) -> Dialect | None:
+ """
+ The dialect used to render literals in the probe SQL.
+
+ Falls back to SQLAlchemy's default dialect if the database cannot produce
+ one -- the probe is best-effort and a rendering mismatch surfaces as a
+ failed probe, which fails open to no pruning.
+ """
+ try:
+ dialect = database.get_dialect()
+ except Exception: # pylint: disable=broad-except # noqa: BLE001
+ return None
+ return dialect if isinstance(dialect, Dialect) else None
+
+
+def _hashable(value: Any) -> Any:
+ """Values arrive from user filters and are not guaranteed hashable."""
+ try:
+ hash(value)
+ except TypeError:
+ return repr(value)
+ return value
+
+
+@dataclass(frozen=True)
+class MappingValidationIssue:
+ """
+ One problem found with a mapping at save time.
+
+ ``blocking`` issues reject the save (400). The rest save fine and leave the
+ mapping inactive -- the PRD is explicit that a mapping "stays inactive
until
+ it parses", so a half-written transform must not cost the owner the rest of
+ their edits.
+ """
+
+ field: str
+ message: str
+ blocking: bool
+
+
+def validate_partition_mapping( # pylint: disable=too-many-arguments
+ *,
+ column_names: set[str],
+ partition_column: str | None,
+ partition_mapped_column: str | None,
+ main_dttm_col: str | None,
+ transform: str | None,
+ engine: str,
+) -> list[MappingValidationIssue]:
+ """
+ Validate a dataset's partition mapping, in two tiers.
+
+ Tier 1 (``blocking=True``) is structural and safety: the columns have to
+ exist, a column cannot be mapped onto itself, and the transform cannot
carry
+ Jinja or call a non-deterministic function. Tier 2 (``blocking=False``) is
+ everything that merely leaves the mapping inactive.
+
+ The Tier-1 transform checks need a successful parse to inspect anything, so
+ an unparseable transform falls through to Tier 2. That leaves them
+ unreachable in exactly the case where it doesn't matter: an unparseable
+ transform is never executed.
+ """
+ if not partition_column:
+ return []
+
+ issues: list[MappingValidationIssue] = []
+
+ if partition_column not in column_names:
+ issues.append(
+ MappingValidationIssue(
+ field="partition_column",
+ message=_(
+ "Partition column %(name)s is not a column on this
dataset.",
+ name=partition_column,
+ ),
+ blocking=True,
+ )
+ )
+
+ if partition_mapped_column and partition_mapped_column not in column_names:
+ issues.append(
+ MappingValidationIssue(
+ field="partition_mapped_column",
+ message=_(
+ "Mapped column %(name)s is not a column on this dataset.",
+ name=partition_mapped_column,
+ ),
+ blocking=True,
+ )
+ )
+
+ effective_mapped_column = partition_mapped_column or main_dttm_col
+ if effective_mapped_column and effective_mapped_column == partition_column:
+ issues.append(
+ MappingValidationIssue(
+ field="partition_column",
+ message=_(
+ "The partition column cannot be mapped onto itself. "
+ "%(name)s is both the partition column and the mapped "
+ "column.",
+ name=partition_column,
+ ),
+ blocking=True,
+ )
+ )
+
+ issues.extend(validate_transform(transform, engine))
+ return issues
+
+
+def validate_transform(
+ transform: str | None,
+ engine: str,
+) -> list[MappingValidationIssue]:
+ """Validate the value transform on its own. See
`validate_partition_mapping`."""
+ field = "partition_value_transform"
+
+ if contains_jinja(transform):
+ return [
+ MappingValidationIssue(
+ field=field,
+ message=_(
+ "Jinja templating is not supported in a partition value "
+ "transform. The transform is evaluated in a different "
+ "context and at a different time from the chart query, so "
+ "a template would not render the same way."
+ ),
+ blocking=True,
+ )
+ ]
+
+ if not transform or not transform.strip():
+ return [
+ MappingValidationIssue(
+ field=field,
+ message=_(
+ "No value transform is set, so no filter will be mirrored "
+ "onto the partition column."
+ ),
+ blocking=False,
+ )
+ ]
+
+ if not is_parseable(transform, engine):
+ detail = parse_error_detail(cast(str, transform), engine)
+ return [
+ MappingValidationIssue(
+ field=field,
+ message=(
+ _(
+ "The value transform could not be parsed: %(detail)s "
+ "The mapping is saved but stays inactive until it
does.",
+ detail=detail,
+ )
+ if detail
+ else _(
+ "The value transform could not be parsed. The mapping "
+ "is saved but stays inactive until it does."
+ )
+ ),
+ blocking=False,
+ )
+ ]
+
+ if not contains_value_placeholder(transform):
+ return [
+ MappingValidationIssue(
+ field=field,
+ message=_(
+ "The value transform must contain the :value placeholder, "
+ "which stands for the filter value being mirrored."
+ ),
+ blocking=False,
+ )
+ ]
+
+ if functions := find_non_deterministic_functions(transform, engine):
+ return [
+ MappingValidationIssue(
+ field=field,
+ message=_(
+ "The value transform calls %(functions)s, whose result "
+ "depends on when and where it runs. The transform is "
+ "evaluated in a separate session and the result is cached,
"
+ "so the emitted predicate would freeze a snapshot of that "
+ "moment.",
+ functions=", ".join(sorted(functions)),
+ ),
+ blocking=True,
+ )
+ ]
+
+ return []
+
+
+@lru_cache(maxsize=LRU_CACHE_MAX_SIZE)
+def is_transform_active(transform: str | None, engine: str) -> bool:
+ """
+ Whether Superset will mirror filters through this transform.
+
+ `validate_transform` with the messages discarded, so the Explore indicator
+ cannot advertise a mapping the save path recorded as inactive. Anything
+ cheaper here would be a second, weaker statement of the same rule, free to
+ drift from the one the save path applies.
+
+ Blocking issues count too. Jinja and non-deterministic transforms are
+ rejected on PUT, but `CreateDatasetCommand` and import do not validate the
+ mapping, so one can still reach the database -- and it would never mirror a
+ filter either.
+
+ Memoized because `partition_filter_mapping_summary` is serialized on every
+ Explore and dashboard load and this parses the transform. The key is
+ everything the answer depends on: `validate_transform` reads no session, no
+ locale and no config beyond `SQL_MAX_PARSE_LENGTH`, which bounds the parser
+ rather than changing a verdict within that bound.
+ """
+ return not validate_transform(transform, engine)
+
+
+def preview_partition_mapping( # pylint: disable=too-many-return-statements
+ datasource: SqlaTable,
+ *,
+ mapped_column: str,
+ value_transform: str | None,
+ sample_values: list[str],
+ operator: FilterOperator = FilterOperator.EQUALS,
+ is_monotonic: bool = False,
+ partition_column: str | None = None,
+) -> dict[str, Any]:
+ """
+ Evaluate a candidate mapping and describe the predicate it would emit.
+
+ Shares the evaluator -- and therefore the probe cache -- with the query
+ path, so preview and runtime cannot drift and a previewed transform warms
+ the chart path for free. The predicate itself comes from
+ `build_mirrored_predicates`, the same builder the query path uses, so
+ operator shapes cannot drift either: what the panel shows is what a chart
+ would emit.
+
+ Validation runs first and the engine second: a half-typed transform is by
+ definition unparseable, so most of what a text input produces costs zero
+ queries.
+
+ Every part of the mapping is taken from the request rather than the stored
+ dataset. The editor previews while the owner is still editing -- a mapping
+ that has to be saved before it can be checked is not a preview.
+ """
+ partition_column = partition_column or datasource.partition_column
+ if not partition_column:
+ return {
+ "valid": False,
+ "reason": "unconfigured",
+ "error": _("No partition column is set."),
+ }
+
+ column_names = {str(column.column_name) for column in datasource.columns}
+ if mapped_column not in column_names:
+ return {
+ "valid": False,
+ "reason": "validation",
+ "error": _("%(name)s is not a column on this dataset.",
name=mapped_column),
+ }
+
+ engine = datasource.database.backend
+ for issue in validate_partition_mapping(
+ column_names=column_names,
+ partition_column=str(partition_column),
+ partition_mapped_column=mapped_column,
+ main_dttm_col=datasource.main_dttm_col,
+ transform=value_transform,
+ engine=engine,
+ ):
+ return {
+ "valid": False,
+ # The parse failure is the one an owner sees constantly, and it is
+ # the only one the mockup gives its own headline to.
+ "reason": (
+ "parse"
+ if issue.field == "partition_value_transform"
+ and not is_parseable(value_transform, engine)
+ else "validation"
+ ),
+ "error": str(issue.message),
+ }
+
+ # The same subquery, function-denylist and RLS policy every other stored
+ # expression goes through. The save path applies it in
+ # `UpdateDatasetCommand._validate_partition_mapping`, but preview evaluates
+ # a candidate transform that has not been saved, so without this the gate
+ # had a door around it: `build_probe_sql` binds only `:value` and splices
+ # the rest of the transform in as SQL text, so a dataset editor without SQL
+ # Lab could submit `(SELECT secret FROM protected_table LIMIT 1) || :value`
+ # and read the answer back out of `emitted_predicate`.
+ #
+ # Imported here rather than at module scope: `connectors.sqla.models`
+ # imports this module, so the dependency only runs one way at import time.
+ from superset.connectors.sqla.models import ( # pylint:
disable=import-outside-toplevel,cyclic-import
+ validate_stored_expression,
+ )
+
+ try:
+ validate_stored_expression(
+ datasource.database,
+ datasource.catalog,
+ datasource.schema,
+ parse_skeleton(cast(str, value_transform)),
+ )
+ except (SupersetSecurityException, QueryClauseValidationException) as ex:
+ return {
+ "valid": False,
+ "reason": "validation",
+ "error": (
+ ex.error.message
+ if isinstance(ex, SupersetSecurityException)
+ else ex.message
+ ),
+ }
+
+ mapping = PartitionMapping(
+ partition_column=str(partition_column),
+ mapped_column=mapped_column,
+ value_transform=cast(str, value_transform),
+ is_monotonic=is_monotonic,
+ )
+ sample_input = _render_sample_input(
+ datasource, mapped_column, operator, sample_values
+ )
+
+ if operator not in PREVIEWABLE_OPERATORS:
+ # Checked before `mirrors`, which lets `TEMPORAL_RANGE` through when
the
+ # transform is declared monotonic -- and the single sample value would
+ # then reach `handle_comparison_filter`, which has no case for a range
+ # and raises. The request schema rejects it too; this keeps the
function
+ # honest for any other caller.
+ return {
+ "valid": False,
+ "reason": "operator",
+ "sample_input": sample_input,
+ "error": _(
+ "A %(operator)s filter is mirrored as a pair of bounds, which "
+ "a preview of a single value cannot describe.",
+ operator=operator.value,
+ ),
+ }
+
+ if not mapping.mirrors(operator):
+ return {
+ "valid": False,
+ "reason": "operator",
+ "sample_input": sample_input,
+ "error": _(
+ "A %(operator)s filter is only mirrored when the transform "
+ "preserves ordering, which this one is not declared to do.",
+ operator=operator.value,
+ ),
+ }
+
+ value: Any = sample_values if operator == FilterOperator.IN else
sample_values[0]
+ errors: list[str] = []
+ predicates = build_mirrored_predicates(
+ datasource, mapping, [(operator, value)], errors=errors
+ )
+ if not predicates:
+ return {
+ "valid": False,
+ "reason": "engine",
+ "sample_input": sample_input,
+ "error": (
+ _(
+ "The transform could not be evaluated against the "
+ "database: %(reason)s",
+ reason=errors[0],
+ )
+ if errors
+ else _("The transform could not be evaluated against the
database.")
+ ),
+ }
+
+ return {
+ "valid": True,
+ "sample_input": sample_input,
+ "emitted_predicate": _render_predicate(datasource, predicates[0]),
+ }
+
+
+def _render_sample_input(
+ datasource: SqlaTable,
+ mapped_column: str,
+ operator: FilterOperator,
+ values: list[str],
+) -> str:
+ """
+ The filter being previewed, written the way an owner would read it.
+
+ Display-only -- it is never executed -- but it is still read as SQL, so the
+ column is quoted and the values are rendered by the dialect rather than
+ concatenated raw. A column named ``order date`` would otherwise come out as
+ two bare words.
+ """
+ quoted = datasource.quote_identifier(mapped_column)
+ if operator == FilterOperator.IN:
+ rendered = ", ".join(
+ _render_literal(datasource.database, value) for value in values
+ )
+ return f"{quoted} IN ({rendered})"
+ return (
+ f"{quoted} {operator.value} {_render_literal(datasource.database,
values[0])}"
+ )
+
+
+def _render_predicate(datasource: SqlaTable, predicate: ColumnElement[Any]) ->
str:
+ """
+ Compile a mirrored predicate to the text a reader would see in View query.
+
+ Every value in it is a probed constant by this point, so ``literal_binds``
+ renders the same literals the chart query carries.
+ """
+ return str(
+ predicate.compile(
+ dialect=_dialect_for(datasource.database),
+ compile_kwargs={"literal_binds": True},
+ )
+ ).replace("\n", " ")
+
+
+def _render_literal(database: Database, value: Any) -> str:
+ """
+ Render a value the way it appears in the generated SQL.
+
+ Compiled by the dialect rather than formatted by hand. `str()` would render
+ a NULL as the Python repr `None` and leave a date unquoted, neither of
which
+ is SQL, and hand-rolled quote-doubling would only ever have been right for
+ strings.
+ """
+ return str(
+ sa.literal(value).compile(
+ dialect=_dialect_for(database),
+ compile_kwargs={"literal_binds": True},
+ )
+ )
+
+
+def build_mirrored_predicates(
+ datasource: SqlaTable,
+ mapping: PartitionMapping,
+ requests: list[tuple[FilterOperator, Any]],
+ *,
+ errors: list[str] | None = None,
+) -> list[ColumnElement[Any]]:
+ """
+ Turn collected ``(operator, value)`` mirror requests into predicates.
+
+ ``requests`` is expected to be deduplicated by the caller. Every value is
+ resolved in a single probe round trip -- one per chart query at most --
then
+ emitted as a literal constant, so "View query" shows the reader an ordinary
+ ``WHERE`` clause rather than an inline expression.
+
+ :param errors: optional sink; see `evaluate_transform`.
+ """
+ if not requests:
+ return []
+
+ partition_column = next(
+ (
+ column
+ for column in datasource.columns
+ if column.column_name == mapping.partition_column
+ ),
+ None,
+ )
+ if partition_column is None:
+ return []
+
+ # Flatten every value that needs probing into one list, remembering how
many
+ # each request consumed so the results can be handed back out.
+ flat: list[Any] = []
+ spans: list[tuple[FilterOperator, int, int]] = []
+ for operator, value in requests:
+ values = list(value) if operator == FilterOperator.IN else [value]
+ spans.append((operator, len(flat), len(values)))
+ flat.extend(values)
+
+ evaluated = evaluate_transform(
+ datasource.database,
+ datasource.catalog,
+ datasource.schema,
+ mapping.value_transform,
+ flat,
+ errors=errors,
+ )
+ if evaluated is None:
+ return []
+
+ if not _bounds_are_ordered(evaluated, spans):
+ return []
+
+ sqla_col = datasource.convert_tbl_column_to_sqla_col(partition_column)
+ db_engine_spec = datasource.db_engine_spec
+
+ predicates: list[ColumnElement[Any]] = []
+ for operator, start, length in spans:
+ chunk = evaluated[start : start + length]
+ if any(value is None for value in chunk):
+ continue
+ if operator == FilterOperator.IN:
+ predicates.append(sqla_col.in_(chunk))
+ else:
+ predicates.append(
+ db_engine_spec.handle_comparison_filter(sqla_col, operator,
chunk[0])
+ )
Review Comment:
Strict `>`/`<` is narrower than the filter when T is monotonic but not
strict, e.g. `date(:value)`. Real probe on SQLite: `event_time > '2026-01-01
00:00'` became `dt > '2026-01-01'` and dropped 09:30 Jan 1; a time range ending
12:00 dropped that morning. 8 mirroring tests pin the strict ops.
```suggestion
else:
# A monotonic T need only be non-decreasing: x > v gives T(x) >=
T(v).
operator = {
FilterOperator.GREATER_THAN:
FilterOperator.GREATER_THAN_OR_EQUALS,
FilterOperator.LESS_THAN: FilterOperator.LESS_THAN_OR_EQUALS,
}.get(operator, operator)
predicates.append(
db_engine_spec.handle_comparison_filter(sqla_col, operator,
chunk[0])
)
```
##########
docs/admin_docs/configuration/partition-filter-mapping.mdx:
##########
@@ -0,0 +1,189 @@
+---
+title: Partition Filter Mapping
+hide_title: true
+sidebar_position: 15
+version: 1
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Partition Filter Mapping
+
+Tables on Hadoop-family engines are often partitioned on a *technical* column
— an epoch
+integer, a lowercased region key — that no analyst would ever filter on.
Unless a query
+carries a predicate on that column, the engine scans every partition.
+
+Partition filter mapping makes this a dataset setting instead of a per-chart
chore. A
+dataset owner names the partition column, the business column whose filters
should be
+mirrored onto it, and a value transform. Superset then appends an equivalent
predicate on
+the partition column to every query. Chart authors change nothing; queries
prune.
+
+:::caution Experimental
+This feature is behind the `PARTITION_FILTER_MAPPING` feature flag and is off
by default.
+:::
+
+## Enabling it
+
+```python
+FEATURE_FLAGS = {
+ "PARTITION_FILTER_MAPPING": True,
+}
+```
+
+Configure it as a **static boolean**. `FEATURE_FLAGS` also accepts per-request
callables,
+but a flag that resolves differently per user or tenant would let a user with
the feature
+off read a cached chart result that was produced from pruned SQL by a user
with it on.
+
+## Configuring a mapping
+
+In the dataset editor's **Columns** tab, under *Default Column Settings*, pick
a
+**Partition column**. By default the mapping follows the dataset's default
datetime column,
+so re-pointing that column moves the mapping with it; set an explicit override
if you want it
+pinned to a different column.
+
+Expand the mapped column's row in *Column Settings* and set the **value
transform**: a SQL
+expression containing a `:value` placeholder, which stands for the filter
value being
+mirrored. The **Transform preserves ordering** checkbox sits directly beneath
it.
+
+| Mapped column | Partition column | Transform |
+|---|---|---|
+| `event_time` (`TIMESTAMP`) | `dt_epoch` (`BIGINT`) |
`unix_timestamp(:value)` |
+| `country` (`VARCHAR`) | `region_key` (`VARCHAR`) | `lower(:value)` |
+
+A filter of `event_time >= '2026-01-01'` then adds `dt_epoch >= 1767225600` to
the query.
+The added predicate is an ordinary `WHERE` clause and shows up in **View
query**.
+
+### Transform preserves ordering
+
+Range filters — including the Explore time range, the most important case —
are only
+mirrored when you check **Transform preserves ordering**.
+
+Monotonicity is a property of the *transform*, not of the column's data type.
+`unix_timestamp(:value)` preserves ordering. `hour(:value)`,
+`date_format(:value, 'dd')` and `dayofweek(:value)` are all perfectly
reasonable
+partition transforms on a `TIMESTAMP` column and none of them do:
`hour('2026-01-01 23:00')`
+is greater than `hour('2026-01-02 01:00')` even though the first instant is
earlier. Mirroring
+a range through one of those would silently return wrong numbers, so Superset
asks you to
+declare it rather than guessing.
+
+When the box is unchecked, `=` and `IN` filters still mirror; ranges do not.
+
+## What is and isn't mirrored
+
+| Filter | Mirrored |
+|---|---|
+| `=`, `IN` | Always |
+| `>`, `>=`, `<`, `<=`, time ranges | Only when the transform preserves
ordering |
+| `!=`, `NOT IN`, `LIKE`, `ILIKE`, `IS NULL`, `IS TRUE` | Never |
+
+Negations are never safe. A transform need not be injective: `lower(:value)`
with
+`country != 'US'` would mirror to `region_key != 'us'`, which excludes rows
whose `country`
+is already lowercase `'us'` — rows the original filter *keeps*.
+
+**Time grains.** A filter that carries a time grain — drill-to-detail, mostly
— compares the
+*truncated* column, so the raw bounds it carries do not describe the rows it
keeps: a row in
+the final partial bucket satisfies `DATE_TRUNC(...) < until` while `col <
until` excludes it.
+Grained *ranges* still mirror, with both bounds widened by one bucket so the
mirror stays no
+narrower than the real filter. A `P1D` drill therefore reads three days of
partitions rather
+than one, instead of scanning the table. Grained `=` and `IN` filters do not
mirror.
+
+Widening is only applied to grains whose bucket width Superset knows, which
means the
+built-in ones. A grain you added through `TIME_GRAIN_ADDONS`, or a built-in
grain whose SQL
+you replaced through `TIME_GRAIN_ADDON_EXPRESSIONS`, has no width Superset can
rely on and
+simply does not mirror.
+
+Known gaps, all of which are out of scope rather than bugs:
+
+- **Filter-value dropdowns do not prune.** Populating a filter's value list
runs its own
+ `SELECT DISTINCT`, which never goes through the chart query path. There is
no filter to
+ mirror from.
+- **Row-level security predicates do not mirror.** They are stored as raw SQL
and appended
+ downstream of the structured filters.
+- **Custom SQL `WHERE` clauses do not mirror**, for the same reason.
+- **Columns with an active advanced data type do not mirror.** Those build
their own
+ predicate shape from translated values, so there is no operator/value pair
to mirror.
+- Dashboard native filters and cross-filters *do* mirror — they arrive as
ordinary filters —
+ they just carry no visual indicator in the filter bar.
Review Comment:
The stated limitation is only in a code comment; this list already covers
indicator gaps, so a line here makes it user-visible:
```suggestion
- Dashboard native filters and cross-filters *do* mirror — they arrive as
ordinary filters —
they just carry no visual indicator in the filter bar.
- **The Explore glyph is advisory.** A virtual dataset whose SQL consumes a
filter through
`get_filters(..., remove_filter=True)` gets neither the filter nor its
mirror, yet the
glyph still shows. **View query** is the authority.
```
##########
superset/connectors/sqla/partition_mapping.py:
##########
@@ -0,0 +1,1278 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Partition filter mapping.
+
+Datasets on Hadoop-family engines are commonly partitioned on a *technical*
+column -- an epoch integer, a lowercased region key -- that no analyst would
+filter on. Unless a query carries a predicate on that column the engine scans
+every partition.
+
+A dataset owner names one partition column ``p``, one business column that
+filters are mirrored from, and a value transform ``T`` (a SQL expression
+containing a ``:value`` placeholder). Superset then appends an equivalent
+predicate on ``p`` to every query, so chart authors change nothing and queries
+prune.
+
+The load-bearing assumption
+---------------------------
+Everything here reasons about ``T(col) op T(v)``, but what is emitted is
+``p op T(v)`` -- a predicate on a *physically different column*. The step from
+one to the other is::
+
+ p = T(mapped_col) for every row in the table
+
+Superset cannot verify that; it is a property of whatever ETL populates the
+partition column. If that job lags, backfills with different logic, or writes
+the partition key in a different timezone, mirrored predicates silently drop
+real rows. The mapping is only as trustworthy as the pipeline behind it.
+
+Operator safety
+---------------
+A mirrored predicate ``P2`` may only be ``AND``-ed onto a query when the
+original predicate ``P1`` *implies* it:
+
+=========================================== =============================
+Original Safe when
+=========================================== =============================
+``col = v``, ``col IN (...)`` always -- ``T`` is a function
+``col >=|>|<|<= v``, ``TEMPORAL_RANGE`` only if ``T`` is monotonic
+``col != v``, ``NOT IN``, ``LIKE``, ... never
+=========================================== =============================
+
+Negations are never safe because ``T`` need not be injective:
+``lower(:value)`` with ``country != 'US'`` mirrors to ``region_key != 'us'``,
+which wrongly excludes rows whose ``country`` is already lowercase ``'us'`` --
+rows the original filter *keeps*.
+
+Monotonicity is a property of the transform, not of the column's data type:
+``hour(:value)``, ``date_format(:value, 'dd')`` and ``dayofweek(:value)`` are
+all reasonable transforms on a ``TIMESTAMP`` column and none of them preserve
+ordering. It is therefore declared by the owner, not inferred.
+
+Time grains
+-----------
+A grained filter -- drill-to-detail, mostly -- compares the *truncated* column,
+``trunc(col) op v``, so the raw bounds it carries do not describe the rows it
+keeps. A row in the final partial bucket satisfies ``trunc(col) < until`` while
+``col < until`` excludes it.
+
+Which direction a grain rounds is not knowable from the duration alone, and the
+obvious guess is wrong: ``WEEK_ENDING_SATURDAY`` rounds *forward* on Hive and
+Presto, and Ocient's grains are ``ROUND``, i.e. to nearest. What every grain
+does satisfy is a bound on the displacement::
+
+ |trunc(ts) - ts| < width(grain)
+
+so widening *both* bounds by one bucket width is no narrower than the real
+predicate whichever way the grain rounds -- and it needs no monotonicity of
+``trunc`` itself, which is what rescues Hive's oddly-anchored ``P1W``. Width is
+a property of the grain, not of the engine, which is what makes this
+maintainable; see `grain_bucket_width`. A grain whose SQL an operator supplied
+has no known width, and does not mirror at all.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import re
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Any, cast, TYPE_CHECKING
+
+import numpy as np
+import pandas as pd
+import sqlalchemy as sa
+from dateutil.relativedelta import relativedelta
+from flask import current_app as app
+from flask_babel import lazy_gettext as _
+from sqlalchemy.engine.interfaces import Dialect
+from sqlalchemy.sql.elements import ColumnElement
+
+from superset.constants import LRU_CACHE_MAX_SIZE, TimeGrain
+from superset.exceptions import (
+ QueryClauseValidationException,
+ SupersetParseError,
+ SupersetSecurityException,
+)
+from superset.extensions import cache_manager, feature_flag_manager
+from superset.sql.parse import SQLStatement
+from superset.utils import json
+from superset.utils.core import FilterOperator
+
+if TYPE_CHECKING:
+ from superset.connectors.sqla.models import SqlaTable, TableColumn
+ from superset.models.core import Database
+
+logger = logging.getLogger(__name__)
+
+FEATURE_FLAG = "PARTITION_FILTER_MAPPING"
+
+#: Longest transform accepted anywhere. A transform is one SQL expression an
+#: owner types by hand, so this is generous; the point is that it is bounded.
+#: Every entry point enforces it -- the typed column field, the import schema
+#: and the preview request -- because `is_transform_active` parses the stored
+#: value on each Explore load, and an unbounded string would make that parse
+#: the expensive part of rendering a chart.
+MAX_TRANSFORM_LENGTH = 1024
+
+#: Placeholder the owner writes in the transform, e.g.
``unix_timestamp(:value)``.
+#: Matched with word boundaries so ``:values`` is not mistaken for it.
+VALUE_PLACEHOLDER_RE = re.compile(r":value\b")
+
+#: Balanced Jinja blocks. The probe would render these in a different context
+#: at a different time from the chart query, so they are rejected at save time.
+JINJA_BLOCK_RE = re.compile(r"\{\{.*?\}\}|\{%.*?%\}|\{#.*?#\}", re.DOTALL)
+
+#: Substituted for ``:value`` before parsing -- sqlglot rejects a bare
``:value``
+#: on most dialects. Mirrors the ``_JINJA_BLOCK_RE`` -> ``NULL`` trick used by
+#: ``validate_stored_expression``.
+_PARSE_STANDIN = "NULL"
+
+#: Functions whose value depends on wall-clock time or randomness. The probe
+#: runs in a different session at a different moment from the chart query and
+#: its result is then cached, so any of these freezes a snapshot of probe time
+#: into the emitted predicate.
+NON_DETERMINISTIC_FUNCTIONS = {
+ "CURRENT_DATE",
+ "CURRENT_TIME",
+ "CURRENT_TIMESTAMP",
+ "NOW",
+ "RAND",
+ "RANDOM",
+ "UUID",
+}
+
+#: Functions that mean "now" only in their zero-argument form. On Hive and
+#: Impala ``unix_timestamp()`` is the current time while ``unix_timestamp(x)``
+#: -- the canonical transform for this feature -- is pure.
+NON_DETERMINISTIC_WHEN_NILADIC = {"UNIX_TIMESTAMP"}
+
+#: Safe for any function ``T``.
+MIRRORABLE_ALWAYS = {FilterOperator.EQUALS, FilterOperator.IN}
+
+#: Safe only when ``T`` preserves ordering.
+MIRRORABLE_IF_MONOTONIC = {
+ FilterOperator.GREATER_THAN,
+ FilterOperator.GREATER_THAN_OR_EQUALS,
+ FilterOperator.LESS_THAN,
+ FilterOperator.LESS_THAN_OR_EQUALS,
+ FilterOperator.TEMPORAL_RANGE,
+}
+
+
+#: Every operator that can be mirrored under *some* transform. The preview
+#: endpoint accepts these; whether a given one actually mirrors still depends
on
+#: the monotonicity declaration.
+MIRRORABLE_OPERATORS = MIRRORABLE_ALWAYS | MIRRORABLE_IF_MONOTONIC
+
+#: Operators the preview endpoint can construct. `TEMPORAL_RANGE` is mirrored
by
+#: the query path, but as *two* bounds that `_collect_partition_mirror_range`
+#: decomposes a time range into -- the preview request carries sample values,
not
+#: a since/until pair, so there is no range for it to build. The editor never
+#: asks for one either: `previewOperatorFor` sends `>=`, `=` or `IN`.
+PREVIEWABLE_OPERATORS = MIRRORABLE_OPERATORS - {FilterOperator.TEMPORAL_RANGE}
+
+
+def mirrorable_operators(is_monotonic: bool) -> set[FilterOperator]:
+ """
+ The operators whose predicates may be mirrored onto the partition column.
+
+ :param is_monotonic: whether the owner declared the transform
+ order-preserving
+ """
+ if is_monotonic:
+ return MIRRORABLE_ALWAYS | MIRRORABLE_IF_MONOTONIC
+ return set(MIRRORABLE_ALWAYS)
+
+
+#: How wide one bucket of each built-in time grain is.
+#:
+#: Keyed on the ISO duration a filter carries in its ``grain``. Written out
+#: rather than parsed: four of the week grains are ISO *intervals* with an
+#: anchor (``P1W/1970-01-03T00:00:00Z``) that ``isodate.parse_duration``
+#: rejects outright, and ``PT0.5H`` / ``P0.25Y`` are fractional. A literal
+#: table is also the thing a reviewer can check a line at a time.
+#:
+#: ``relativedelta`` rather than ``timedelta`` so the calendar grains stay
+#: calendar arithmetic: a month is not 30 days.
+GRAIN_BUCKET_WIDTHS: dict[str, relativedelta] = {
+ TimeGrain.SECOND: relativedelta(seconds=1),
+ TimeGrain.FIVE_SECONDS: relativedelta(seconds=5),
+ TimeGrain.THIRTY_SECONDS: relativedelta(seconds=30),
+ TimeGrain.MINUTE: relativedelta(minutes=1),
+ TimeGrain.FIVE_MINUTES: relativedelta(minutes=5),
+ TimeGrain.TEN_MINUTES: relativedelta(minutes=10),
+ TimeGrain.FIFTEEN_MINUTES: relativedelta(minutes=15),
+ TimeGrain.THIRTY_MINUTES: relativedelta(minutes=30),
+ TimeGrain.HALF_HOUR: relativedelta(minutes=30),
+ TimeGrain.HOUR: relativedelta(hours=1),
+ TimeGrain.SIX_HOURS: relativedelta(hours=6),
+ TimeGrain.DAY: relativedelta(days=1),
+ TimeGrain.WEEK: relativedelta(days=7),
+ TimeGrain.WEEK_STARTING_SUNDAY: relativedelta(days=7),
+ TimeGrain.WEEK_STARTING_MONDAY: relativedelta(days=7),
+ TimeGrain.WEEK_ENDING_SATURDAY: relativedelta(days=7),
+ TimeGrain.WEEK_ENDING_SUNDAY: relativedelta(days=7),
+ TimeGrain.MONTH: relativedelta(months=1),
+ TimeGrain.QUARTER: relativedelta(months=3),
+ TimeGrain.QUARTER_YEAR: relativedelta(months=3),
+ TimeGrain.YEAR: relativedelta(years=1),
+}
+
+
+def grain_bucket_width(grain: str | None, engine: str) -> relativedelta | None:
+ """
+ How far a grain's truncation can move a timestamp, or ``None`` if unknown.
+
+ A grained filter compares the *truncated* column, so the raw bounds it
+ carries do not describe the rows it keeps. Widening both bounds by one
+ bucket recovers a predicate that is no narrower than the real one -- see
+ `_collect_partition_mirror_range` for the argument. That only works for a
+ grain whose bucket width Superset knows, which excludes anything an
+ operator supplied.
+
+ :param grain: the ISO duration from the filter's ``grain``
+ :param engine: the engine spec's ``engine``, to check per-engine overrides
+ """
+ if not grain:
+ return None
+
+ # An operator-declared grain carries whatever duration string they typed,
+ # and `TIME_GRAIN_ADDON_EXPRESSIONS` can redefine a *built-in* grain's SQL
+ # per engine -- `P1D` could be anything at all. Neither has a width we can
+ # claim to know, so both fall back to not mirroring.
+ if grain in app.config["TIME_GRAIN_ADDONS"]:
+ return None
+ if grain in app.config["TIME_GRAIN_ADDON_EXPRESSIONS"].get(engine, {}):
+ return None
+
+ return GRAIN_BUCKET_WIDTHS.get(grain)
+
+
+@dataclass(frozen=True)
+class PartitionMapping:
+ """A resolved, usable partition filter mapping."""
+
+ partition_column: str
+ mapped_column: str
+ value_transform: str
+ is_monotonic: bool
+
+ def mirrors(self, operator: FilterOperator) -> bool:
+ return operator in mirrorable_operators(self.is_monotonic)
+
+
+def contains_value_placeholder(transform: str | None) -> bool:
+ """Whether the transform contains the ``:value`` placeholder."""
+ return bool(transform) and VALUE_PLACEHOLDER_RE.search(transform or "") is
not None
+
+
+def contains_jinja(transform: str | None) -> bool:
+ """Whether the transform contains a balanced Jinja block."""
+ return bool(transform) and JINJA_BLOCK_RE.search(transform or "") is not
None
+
+
+def parse_skeleton(transform: str) -> str:
+ """
+ The transform with ``:value`` substituted out, ready for a SQL parser.
+
+ ``sanitize_clause`` / sqlglot choke on a bare ``:value`` on most dialects,
+ so the placeholder is swapped for a benign literal first -- the same trick
+ ``validate_stored_expression`` uses for Jinja blocks.
+ """
+ return VALUE_PLACEHOLDER_RE.sub(_PARSE_STANDIN, transform)
+
+
+#: Prefix the transform is wrapped in before parsing. Its length is subtracted
+#: from any reported column so positions refer to what the owner actually
typed.
+_SELECT_PREFIX = "SELECT "
+
+
+def _parse_skeleton(transform: str, engine: str) -> SQLStatement | None:
+ """
+ Parse ``SELECT <transform>`` with the placeholder substituted out.
+
+ Returns ``None`` when the transform does not parse.
+ """
+ try:
+ return SQLStatement(f"{_SELECT_PREFIX}{parse_skeleton(transform)}",
engine)
+ except SupersetParseError:
+ return None
+
+
+def parse_error_detail(transform: str, engine: str) -> str | None:
+ """
+ Where the parser gave up on the transform.
+
+ Returns ``None`` when it parses, or when the parser offered no position.
+ Note that sqlglot parses unknown functions happily -- a misspelled function
+ name is not a parse error, it is an engine error, and surfaces only when
the
+ transform is evaluated.
+
+ The parser's own ``highlight`` is deliberately dropped: it would name the
+ ``NULL`` we substituted for ``:value``, which is not a token the owner
typed.
+ """
+ try:
+ SQLStatement(f"{_SELECT_PREFIX}{parse_skeleton(transform)}", engine)
+ except SupersetParseError as ex:
+ column = (ex.error.extra or {}).get("column")
+ if not isinstance(column, int):
+ return None
+ return str(
+ _(
+ "syntax error at position %(position)d.",
+ position=_position_in_transform(transform, column),
+ )
+ )
+ return None
+
+
+def _position_in_transform(transform: str, parsed_column: int) -> int:
+ """
+ Map a column in the parsed skeleton back to the transform as typed.
+
+ Two substitutions stand between them: the ``SELECT`` prefix, and every
+ ``:value`` that became a shorter ``NULL``. Without unwinding both, a
+ reported position drifts left by two characters per placeholder ahead of it
+ -- which is worst exactly where transforms usually break, at the end.
+ """
+ position = max(parsed_column - len(_SELECT_PREFIX), 0)
+ shift = len(":value") - len(_PARSE_STANDIN)
+ preceding = sum(
+ 1
+ for index, match in enumerate(VALUE_PLACEHOLDER_RE.finditer(transform))
+ if match.start() - index * shift < position
+ )
+ return position + preceding * shift
+
+
+def is_parseable(transform: str | None, engine: str) -> bool:
+ """
+ Whether the transform parses as a single select expression.
+
+ "Single" is the load-bearing word. `SELECT lower(:value), 'x'` parses just
+ as happily as `SELECT lower(:value)`, but it returns two columns per input
+ value, and the probe reads one column per input -- so an `IN` filter would
+ get a predicate built from the wrong halves of the wrong rows. Rejecting
the
+ list here is what lets the probe trust its own column count.
+ """
+ if not transform or not transform.strip():
+ return False
+ statement = _parse_skeleton(transform, engine)
+ return statement is not None and statement.count_select_expressions() == 1
+
+
+def find_non_deterministic_functions(transform: str, engine: str) -> set[str]:
+ """
+ Names of non-deterministic functions the transform calls.
+
+ ``UNIX_TIMESTAMP`` is only reported in its zero-argument form, which means
+ "now" on Hive and Impala; the one-argument form is the canonical temporal
+ transform and stays allowed.
+ """
+ statement = _parse_skeleton(transform, engine)
+ if statement is None:
+ return set()
+
+ found = {
+ name
+ for name in NON_DETERMINISTIC_FUNCTIONS
+ if statement.check_functions_present({name})
+ }
+ return found | _find_niladic_calls(statement)
+
+
+def _find_niladic_calls(statement: SQLStatement) -> set[str]:
+ """
+ Names from ``NON_DETERMINISTIC_WHEN_NILADIC`` called with no arguments.
+
+ Note some dialects resolve the zero-argument form themselves -- Hive parses
+ ``unix_timestamp()`` straight to ``CURRENT_TIMESTAMP`` -- in which case the
+ name-based check above has already caught it. This is the backstop for the
+ dialects that do not.
+ """
+ return NON_DETERMINISTIC_WHEN_NILADIC & statement.get_niladic_functions()
+
+
+def resolve_partition_mapping(datasource: SqlaTable) -> PartitionMapping |
None:
+ """
+ Resolve the dataset's mapping, or ``None`` when nothing may be mirrored.
+
+ Every bail-out here is defensive as well as functional: save-time
validation
+ rejects most of these, but rows predating the validation can still violate
+ the invariants, and a column sync can invalidate a mapping that was fine
+ when it was written.
+
+ The transform gate is `is_transform_active`, the same function the Explore
+ indicator reads, which is `validate_transform` with the messages discarded.
+ Anything narrower here would be a second, weaker statement of the same
rule:
+ a transform calling `now()` that reached storage without passing
+ `UpdateDatasetCommand` -- through import, or through a bundle written by
hand
+ -- would be reported inactive by the editor and still mirrored by this
+ function, freezing a snapshot of probe time into the predicate with nothing
+ on screen to say so.
+ """
+ if not feature_flag_manager.is_feature_enabled(FEATURE_FLAG):
+ return None
+
+ partition_column = getattr(datasource, "partition_column", None)
+ if not partition_column:
+ return None
+
+ columns_by_name = {column.column_name: column for column in
datasource.columns}
+ if partition_column not in columns_by_name:
+ # The partition column was dropped by a column sync or at the source.
+ return None
+
+ mapped_column_name = (
+ getattr(datasource, "partition_mapped_column", None) or
datasource.main_dttm_col
+ )
+ if not mapped_column_name or mapped_column_name not in columns_by_name:
+ return None
+
+ if mapped_column_name == partition_column:
+ # Self-mapping: the mirrored predicate would duplicate the original.
+ return None
+
+ mapped_column = columns_by_name[mapped_column_name]
+ transform = getattr(mapped_column, "partition_value_transform", None)
+ if not is_transform_active(transform, datasource.database.backend):
Review Comment:
Import keeps a subquery transform (`drop_unusable_partition_transforms`
drops only blocking issues) and this gate skips the stored-expression policy.
Run: `(SELECT secret FROM vault LIMIT 1) || :value` put `'TOPSECRET_us'` into
chart SQL; the same subquery as a calculated column is rejected.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
