github-actions[bot] commented on code in PR #68499: URL: https://github.com/apache/doris/pull/68499#discussion_r4314503941
########## fe/fe-core/src/main/java/org/apache/doris/nereids/spm/builder/SPMExprSqlBuilder.java: ########## @@ -0,0 +1,807 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.spm.builder; + +import org.apache.doris.nereids.analyzer.UnboundFunction; +import org.apache.doris.nereids.spm.matcher.SPMFrozenTreeReplacer; +import org.apache.doris.nereids.spm.placeholder.SPMSubquerySupport; +import org.apache.doris.nereids.trees.expressions.Add; +import org.apache.doris.nereids.trees.expressions.AggregateExpression; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.And; +import org.apache.doris.nereids.trees.expressions.Between; +import org.apache.doris.nereids.trees.expressions.BinaryArithmetic; +import org.apache.doris.nereids.trees.expressions.BinaryOperator; +import org.apache.doris.nereids.trees.expressions.BitAnd; +import org.apache.doris.nereids.trees.expressions.BitNot; +import org.apache.doris.nereids.trees.expressions.BitOr; +import org.apache.doris.nereids.trees.expressions.BitXor; +import org.apache.doris.nereids.trees.expressions.CaseWhen; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; +import org.apache.doris.nereids.trees.expressions.CompoundPredicate; +import org.apache.doris.nereids.trees.expressions.Divide; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.Exists; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.GreaterThan; +import org.apache.doris.nereids.trees.expressions.GreaterThanEqual; +import org.apache.doris.nereids.trees.expressions.InPredicate; +import org.apache.doris.nereids.trees.expressions.InSubquery; +import org.apache.doris.nereids.trees.expressions.IntegralDivide; +import org.apache.doris.nereids.trees.expressions.IsFalse; +import org.apache.doris.nereids.trees.expressions.IsNull; +import org.apache.doris.nereids.trees.expressions.IsTrue; +import org.apache.doris.nereids.trees.expressions.LessThan; +import org.apache.doris.nereids.trees.expressions.LessThanEqual; +import org.apache.doris.nereids.trees.expressions.Like; +import org.apache.doris.nereids.trees.expressions.Mod; +import org.apache.doris.nereids.trees.expressions.Multiply; +import org.apache.doris.nereids.trees.expressions.Not; +import org.apache.doris.nereids.trees.expressions.NullSafeEqual; +import org.apache.doris.nereids.trees.expressions.OrderExpression; +import org.apache.doris.nereids.trees.expressions.SessionVarGuardExpr; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.SubqueryExpr; +import org.apache.doris.nereids.trees.expressions.Subtract; +import org.apache.doris.nereids.trees.expressions.TryCast; +import org.apache.doris.nereids.trees.expressions.WhenClause; +import org.apache.doris.nereids.trees.expressions.WindowExpression; +import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; +import org.apache.doris.nereids.trees.expressions.functions.Function; +import org.apache.doris.nereids.trees.expressions.functions.Udf; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.GroupConcat; +import org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinctGroupConcat; +import org.apache.doris.nereids.trees.expressions.functions.window.Lag; +import org.apache.doris.nereids.trees.expressions.functions.window.Lead; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.LargeIntType; +import org.apache.doris.nereids.util.ExpressionUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Expression printer of the decompiler. + * + * When SPMPlan2SQLBuilder decompiles a physical plan into SQL, it must print the + * expressions in the plan (SlotReference, predicates, function calls, etc.) as SQL text. + * This class extends ExpressionVisitor, walks the expression tree recursively, and + * replaces each SlotReference with the normalized reference name registered in + * SQLRelation.getColumnNames() (e.g. c_5), so planSql column references never leak + * internal ExprIds and never become ambiguous across subqueries. + * + * Design principles (design doc 6.2): + * + * - Explicitly print the common expression types (arithmetic, comparison, logic, + * functions, IN, BETWEEN, CASE WHEN, ...) recursively so the column mapping reaches + * the leaf SlotReferences. + * - For types not explicitly covered, fall back to expr.toSql() (no column mapping; + * a known simplification that later milestones can complete). + * + * Note: in Doris the negative forms such as IS NOT NULL, NOT IN and NOT BETWEEN are + * represented as Not(...) wrappers (not as boolean flags on the node), so each visit + * method only outputs the positive form and visitNot handles negation uniformly. + */ +public class SPMExprSqlBuilder extends ExpressionVisitor<String, SQLRelation> { + + /** + * Depth of the AGGREGATE argument list currently being rendered (see + * visitBoundFunction / visitAggregateExpression / visitWindow). + * + * Only INSIDE an aggregate's argument does an analyzer-inserted integral -> LARGEINT + * widening carry the LOWERING's accumulator semantics + * (see preservesLoweredAccumulator). Anywhere else - an operand of ordinary arithmetic - + * the same shape is an ordinary coercion, and freezing it pins the CAPTURED operand type: + * `k + 9223372036854775808` (BIGINT k, LARGEINT literal) freezes + * `CAST(k AS LARGEINT) + _spm_const_var(1)`, so the `+ 0.25` variant re-analyzes against a + * LARGEINT operand, derives DECIMAL(38,0) and truncates the substituted value to 1 instead + * of the original expression's DECIMAL(23,2) 1.25. + */ + private int aggregateArgumentDepth; + + /** + * Prints a single expression. + * + * @param expr the expression to print + * @param relation the SQLRelation carrying the columnNames mapping + * @return the SQL text + */ + public String print(Expression expr, SQLRelation relation) { + return expr.accept(this, relation); + } + + /** + * Renders one function's argument list, with the ACCUMULATOR frame active for an + * aggregate (see aggregateArgumentDepth). + */ + private String renderArguments(boolean aggregate, List<Expression> arguments, + SQLRelation context) { + if (!aggregate) { + return arguments.stream().map(argument -> argument.accept(this, context)) + .collect(Collectors.joining(", ")); + } + aggregateArgumentDepth++; + try { + return arguments.stream().map(argument -> argument.accept(this, context)) + .collect(Collectors.joining(", ")); + } finally { + aggregateArgumentDepth--; + } + } + + @Override + public String visit(Expression expr, SQLRelation context) { + // SPM placeholder: render as the id-only function call "_spm_const_var(id)" / + // "_spm_const_list(id)" (the value does not participate in the digest / planSql + // serialization). The id-only form keeps the digest value-independent AND makes + // the frozen planSql re-parseable at rewrite time (aligned with SR: the frozen + // text carries the placeholder id, the actual values are substituted by id when + // the baseline is replayed). + if (expr instanceof org.apache.doris.nereids.spm.placeholder.SpmConstVar) { + org.apache.doris.nereids.spm.placeholder.SpmConstVar ph = + (org.apache.doris.nereids.spm.placeholder.SpmConstVar) expr; + return "_spm_const_var(" + ph.getId() + ")"; + } + if (expr instanceof org.apache.doris.nereids.spm.placeholder.SpmConstList) { + org.apache.doris.nereids.spm.placeholder.SpmConstList list = + (org.apache.doris.nereids.spm.placeholder.SpmConstList) expr; + // IN-list placeholder: render size-independently (_spm_const_list(id)) so + // the digest / planSql stays value AND list-length independent: a baseline + // "IN (1, 2, 3)" matches a user "IN (10, 20)" regardless of list length. + return "_spm_const_list(" + list.getId() + ")"; + } + // Default fallback: use toSql() of the expression itself (no column mapping). + // toSql() bypasses the ExprId -> column mapping, so a composite shape whose + // children need remapping (e.g. ~left.a above a join that renamed the colliding + // a to c_N / qualified it) would freeze a stale, unresolvable column and - with + // the default enable_spm_fallback = false - fail the rewritten query. Reject + // such shapes so CREATE keeps the user planSql and the rewrite degrades to the + // parameterized-tree path. + ensureNoRemappedSlots(expr, context); + return expr.toSql(); + } + + /** + * Safety net of the generic toSql() fallback: rejects the expression when any slot + * below it is registered under a reference that toSql() would NOT reproduce (the + * collision-renaming / quoting / qualification case), because toSql() would then emit + * a stale or differently-shaped name. Only a byte-identical registered reference is + * renderable through the fallback. Public for tests. + * + * @param expr the expression about to fall back to toSql() + * @param context the relation carrying the ExprId -> column-reference mapping + * @throws UnsupportedOperationException when a child slot needs remapping + */ + public static void ensureNoRemappedSlots(Expression expr, SQLRelation context) { + if (expr instanceof SlotReference) { + SlotReference slot = (SlotReference) expr; + String mapped = context.getColumnNames().get(slot.getExprId()); + // toSql() prints the BARE slot name, so a registered reference is only safe + // when it is byte-identical to that rendering. A special-character column + // maps to a QUOTED reference (`a-b`) and a disambiguated one to a QUALIFIED + // reference (l.x): both still "name" the column, but freezing their unquoted / + // unqualified toSql() text produces SQL that no longer resolves after reload + // ("a-b MATCH 'foo'" parses as a subtraction; a bare x may be ambiguous). + // Reject the generic fallback for those shapes - CREATE keeps the user planSql + // and the rewrite degrades to the parameterized-tree path. + if (mapped != null && !mapped.equals(slot.getName())) { + throw new UnsupportedOperationException( + "SPM decompile: expression " + expr.toSql() + " needs column remapping (" + + mapped + " -> " + slot.getName() + ") which toSql() cannot apply"); + } + } + for (Expression child : expr.children()) { + ensureNoRemappedSlots(child, context); + } + } + + // ==================== column references ==================== + + @Override + public String visitSlotReference(SlotReference slotReference, SQLRelation context) { + String mapped = context.getColumnNames().get(slotReference.getExprId()); + if (mapped != null) { + return mapped; + } + // Doris rollup materializes "GROUPING(col)" as an internal slot named + // GROUPING_PREFIX_<col>; when such a slot was not registered by the repeat + // decompiler (defensive fallback), render it back to GROUPING(col). + String name = slotReference.getName(); + if (name != null && name.startsWith("GROUPING_PREFIX_")) { + return "GROUPING(" + name.substring("GROUPING_PREFIX_".length()) + ")"; + } + // An unregistered slot whose name is Doris's internal mark-slot naming + // ("$c$N", e.g. the EXISTS marker of an uncorrelated subquery folded into a + // constant CROSS JOIN) can never be resolved in the decompiled SQL - emitting + // the raw name would freeze a planSql that fails on replay ("Unknown column + // '$c$1'"). Such a shape has no faithful SQL rendering here, so fail the + // decompile: CREATE BASELINE then falls back to the user-supplied planSql + // (which is exactly the original query for these constant boolean outputs). + if (name != null && name.startsWith("$c$")) { + throw new UnsupportedOperationException( + "SPM decompile: unresolvable internal mark slot " + name); + } + // Unregistered column reference (e.g. a CTE output column) degrades to the raw + // column name + return slotReference.toSql(); + } + + // ==================== binary operations (arithmetic / comparison / logic) ==================== + + @Override + public String visitComparisonPredicate(ComparisonPredicate comparisonPredicate, SQLRelation context) { + return "(" + comparisonPredicate.left().accept(this, context) + + " " + operatorSymbol(comparisonPredicate) + + " " + comparisonPredicate.right().accept(this, context) + ")"; + } + + @Override + public String visitBinaryArithmetic(BinaryArithmetic binaryArithmetic, SQLRelation context) { + return "(" + binaryArithmetic.left().accept(this, context) + + " " + operatorSymbol(binaryArithmetic) + + " " + binaryArithmetic.right().accept(this, context) + ")"; + } + + /** + * BitNot (~x): the generic fallback would print the expression through toSql() and + * bypass the column mapping; above a join whose colliding column was renamed, that + * freezes the stale slot text. Render the child recursively instead so the mapped + * column reference stays mapped. + */ + @Override + public String visitBitNot(BitNot bitNot, SQLRelation context) { + return "~(" + bitNot.child().accept(this, context) + ")"; + } + + @Override + public String visitCompoundPredicate(CompoundPredicate compoundPredicate, SQLRelation context) { + // And / Or are n-ary in Nereids: a single node can hold more than two children, + // so child(0)/child(1) alone would drop the rest. Flatten the same-type operators + // into a list of operands with ExpressionUtils.extract (see extractConjunction / + // extractDisjunction) and join them with the operator symbol. extract stops at a + // different operator type, so e.g. (a or b) and c keeps the inner Or intact. + // + // CompoundPredicate is not a BinaryOperator (And/Or extend CompoundPredicate + // directly), so resolve the symbol here instead of using operatorSymbol. + String symbol = compoundPredicate instanceof And ? "AND" : "OR"; + String joined = ExpressionUtils.extract(compoundPredicate).stream() + .map(operand -> operand.accept(this, context)) + .collect(Collectors.joining(" " + symbol + " ")); + // AND / OR need parentheses to keep the precedence + return "(" + joined + ")"; + } + + @Override + public String visitNot(Not not, SQLRelation context) { + return "NOT (" + not.child().accept(this, context) + ")"; + } + + @Override + public String visitIsNull(IsNull isNull, SQLRelation context) { + return isNull.child().accept(this, context) + " IS NULL"; + } + + @Override + public String visitIsTrue(IsTrue isTrue, SQLRelation context) { + return isTrue.child().accept(this, context) + " IS TRUE"; + } + + @Override + public String visitIsFalse(IsFalse isFalse, SQLRelation context) { + return isFalse.child().accept(this, context) + " IS FALSE"; + } + + // ==================== predicates ==================== + + @Override + public String visitInPredicate(InPredicate inPredicate, SQLRelation context) { + String compareExpr = inPredicate.getCompareExpr().accept(this, context); + List<String> options = inPredicate.getOptions().stream() + .map(o -> o.accept(this, context)) + .collect(Collectors.toList()); + // A literal-only IN option list may be stored in a set-backed collection whose + // iteration order is NOT stable across runs / FE restarts (observed: TPCDS q75's + // "d_year IN (2001, 2002)" decompiled as (2001, 2002) one run and (2002, 2001) + // the next, flipping the frozen planSql and failing the .out comparison). IN + // semantics are order-independent, so a literal-only list is emitted in sorted + // order to keep the decompiled planSql deterministic. + boolean allLiterals = inPredicate.getOptions().stream() + .allMatch(o -> o instanceof Literal); + if (allLiterals && options.size() > 1) { + Collections.sort(options); + } + return compareExpr + " IN (" + String.join(", ", options) + ")"; + } + + @Override + public String visitBetween(Between between, SQLRelation context) { + String compareExpr = between.getCompareExpr().accept(this, context); + String lower = between.getLowerBound().accept(this, context); + String upper = between.getUpperBound().accept(this, context); + return compareExpr + " BETWEEN " + lower + " AND " + upper; + } + + @Override + public String visitLike(Like like, SQLRelation context) { + String rendered = like.left().accept(this, context) + + " LIKE " + like.right().accept(this, context); + if (like.children().size() > 2) { + // The third child is the ESCAPE character. Dropping it froze a three-argument + // LIKE as a two-child predicate: parameterizing the escape literal fails + // LIKE's literal-only legality check, so creation fell back to the raw plan and + // persisted the ALTERED predicate - a column-pattern LIKE replayed without + // ESCAPE can match different rows. Render the child recursively. + rendered += " ESCAPE " + like.child(2).accept(this, context); + } + return rendered; + } + + // ==================== function calls ==================== + + @Override + public String visitBoundFunction(BoundFunction boundFunction, SQLRelation context) { + // The Nullable marker (nullable(x) wraps a column to make it nullable, e.g. on a + // recursive CTE union branch) has no SQL equivalent: decompile the inner + // expression only. + if (boundFunction instanceof org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable) { + return boundFunction.child(0).accept(this, context); + } + if (boundFunction instanceof GroupConcat || boundFunction instanceof MultiDistinctGroupConcat) { + String rendered = renderGroupConcat((AggregateFunction) boundFunction, context); + if (rendered == null) { + throw new UnsupportedOperationException( + "SPM decompile: group_concat shape is not supported yet"); + } + return rendered; + } + String args = renderArguments(boundFunction instanceof AggregateFunction, + boundFunction.children(), context); + return functionName(boundFunction) + "(" + args + ")"; + } + + /** + * Name of a bound function call for the frozen SQL. A user-defined function keeps its + * database qualifier (Java / Python UDF, UDAF and UDTF nodes retain dbName): a frozen + * db1.f(k) must not resolve to db2.f(k) when the replayed SQL runs under another + * default database. A GLOBAL UDF has no qualifier, and its registered name is not + * necessarily a plain identifier (my-fn): emitting it bare produced frozen text the + * parser reads as a subtraction, and a persisted frozen row has no parameterized-tree + * fallback - the baseline could never replay. + * + * A user function NAMED like the SPM placeholder marker is refused here + * #15): its rendered call would be indistinguishable from a genuine + * _spm_const_var(id) / _spm_const_list(id) marker, so the replay could either + * substitute the real call with a user literal or reject the frozen baseline as an + * unresolved marker. Refusing the decompile keeps the ORIGINAL planSql (provenance + * planFrozen=false) as the baseline's replayable fallback. + * + * Package-visible for tests. + */ + public static String functionName(Function function) { + if (function instanceof Udf) { + String name = function.getName(); + if (SPMFrozenTreeReplacer.CONST_VAR_FUNC.equals(name) + || SPMFrozenTreeReplacer.CONST_LIST_FUNC.equals(name)) { + throw new UnsupportedOperationException("SPM decompile: the user function " + + name + " collides with the SPM placeholder marker; keeping the" + + " original planSql"); + } + String dbName = ((Udf) function).getDbName(); + if (dbName != null && !dbName.isEmpty()) { + // Quote each component: a legal database name like my-db is NOT a + // parser identifier, and `my-db`.f(k) is what re-parses after a reload + // - the unquoted my-db.f(k) made the persisted frozen SQL unparseable. + return SPMPlan2SQLBuilder.quoteIdentifier(dbName) + "." + + SPMPlan2SQLBuilder.quoteIdentifier(name); + } + // a GLOBAL UDF without a database: quote the (possibly non-plain) name + return SPMPlan2SQLBuilder.quoteIdentifier(name); + } + return function.getName(); + } + + @Override + public String visitUnboundFunction(UnboundFunction unboundFunction, SQLRelation context) { + // UnboundFunction (a raw parsed function call, e.g. substr(ca_zip, 1, 5) in a + // WHERE clause before analysis). Render the children recursively so placeholder + // values inside the arguments stay normalized (_spm_const_var(id)); the + // default toSql() fallback would render them with their embedded values and make + // the digest value-dependent. + String args = unboundFunction.children().stream() + .map(a -> a.accept(this, context)) + .collect(Collectors.joining(", ")); + // Keep the database qualifier: db1.f(k + 1) must not freeze as unqualified + // f(...) and resolve to db2.f when replayed under USE db2. + String dbName = unboundFunction.getDbName(); + String qualifiedName = (dbName == null || dbName.isEmpty()) + ? unboundFunction.getName() + : SPMPlan2SQLBuilder.quoteIdentifier(dbName) + "." + + SPMPlan2SQLBuilder.quoteIdentifier(unboundFunction.getName()); + return qualifiedName + "(" + args + ")"; + } + + @Override + public String visitAggregateExpression(AggregateExpression aggregateExpression, SQLRelation context) { + AggregateFunction fn = aggregateExpression.getFunction(); + if (fn instanceof GroupConcat || fn instanceof MultiDistinctGroupConcat) { + String rendered = renderGroupConcat(fn, context); + if (rendered == null) { + throw new UnsupportedOperationException( + "SPM decompile: group_concat shape is not supported yet"); + } + return rendered; + } + // Explicitly handle DISTINCT aggregates: count(distinct col) + String distinct = fn.isDistinct() ? "distinct " : ""; + String args = renderArguments(true, fn.children(), context); + return functionName(fn) + "(" + distinct + args + ")"; + } + + @Override + public String visitWindow(WindowExpression windowExpression, SQLRelation context) { + // Hand-render the OVER clause so every slot reference (partition keys, order + // keys, function arguments) goes through the column mapping - Doris's own + // toSql() prints raw slot names such as "GROUPING_PREFIX_s_county" or a local + // buffer name that is not referenceable in the decompiled SQL. + Expression fn = windowExpression.getFunction(); + String fnSql; + if (fn instanceof AggregateFunction) { + AggregateFunction aggFn = (AggregateFunction) fn; + if (aggFn instanceof GroupConcat || aggFn instanceof MultiDistinctGroupConcat) { + String rendered = renderGroupConcat(aggFn, context); + if (rendered == null) { + throw new UnsupportedOperationException( + "SPM decompile: group_concat shape is not supported yet"); + } + fnSql = rendered; + } else { + String distinct = aggFn.isDistinct() ? "DISTINCT " : ""; + fnSql = functionName(aggFn) + "(" + distinct + + renderArguments(true, fn.children(), context) + ")"; + } + } else if (fn instanceof BoundFunction) { + fnSql = functionName((BoundFunction) fn) + "(" + + renderArguments(false, fn.children(), context) + ")"; + } else { + fnSql = print(fn, context); + } + StringBuilder sb = new StringBuilder(fnSql).append(" OVER ("); + List<String> parts = new ArrayList<>(); + if (!windowExpression.getPartitionKeys().isEmpty()) { + parts.add("PARTITION BY " + windowExpression.getPartitionKeys().stream() + .map(p -> print(p, context)).collect(Collectors.joining(", "))); + } + if (!windowExpression.getOrderKeys().isEmpty()) { + parts.add("ORDER BY " + windowExpression.getOrderKeys().stream() + .map(o -> print(o.child(), context) + (o.isAsc() ? " ASC" : " DESC") + + (o.isNullFirst() ? " NULLS FIRST" : " NULLS LAST")) + .collect(Collectors.joining(", "))); + } + // an explicit frame is only valid together with an ORDER BY clause; without one + // the SQL default frame (RANGE UNBOUNDED PRECEDING .. CURRENT ROW) is identical. + // + // LAG() / LEAD() are the exception: their SQL text must NOT carry a frame + // (WindowFunctionChecker#visitLag / #visitLead REJECT an explicit one and then + // install an INTERNAL rows frame on the window expression), so every captured + // plan carries a frame that the user never wrote. Emitting it froze SQL whose + // own re-analysis throws "WindowFrame for LAG() must be null" - the baseline + // could never be replayed. The user-level absence of a frame is the faithful + // rendering. + if (!isFrameForbiddenWindowFunction(fn) + && !windowExpression.getOrderKeys().isEmpty() + && windowExpression.getWindowFrame().isPresent()) { + parts.add(windowExpression.getWindowFrame().get().computeToSql()); + } + return sb.append(String.join(" ", parts)).append(")").toString(); + } + + /** + * Whether the window function's SQL text must not carry an explicit frame (see + * the frame rendering in visitWindow): LAG / LEAD only. + */ + private static boolean isFrameForbiddenWindowFunction(Expression fn) { + if (fn instanceof Lag || fn instanceof Lead) { + return true; + } + String name = fn instanceof UnboundFunction ? ((UnboundFunction) fn).getName() : null; + return "lag".equalsIgnoreCase(name) || "lead".equalsIgnoreCase(name); + } + + // ==================== other common types ==================== + + /** + * Renders a GROUP_CONCAT / MULTI_DISTINCT_GROUP_CONCAT into the dedicated + * GROUP_CONCAT grammar. + * + * Doris's analyzer admits ONE value expression next to an optional CONSTANT separator + * (the second non-order argument must be constant - a non-constant one is rejected by + * checkLegalityBeforeTypeCoercion), with an optional trailing ORDER BY. That covers + * every SQL-reachable shape: + * + * - GROUP_CONCAT(DISTINCT v ORDER BY k): the physical plan carries the execution + * shape MultiDistinctGroupConcat(v, ORDER BY k) (GroupConcat.mustUseMultiDistinctAgg + * converts it). The DISTINCT form re-analyzes to exactly that node, so it is + * rendered - previously it returned null and forced the whole decompile to fall + * back to the user-supplied plan text; + * - MULTI_DISTINCT_GROUP_CONCAT(v[, sep]): the function NAME carries the dedup + * contract while its isDistinct() flag is false, so the DISTINCT keyword must be + * re-emitted - GROUP_CONCAT(v) would keep the duplicates ("1,1,2" instead of + * "1,2"). + * + * @param fn the aggregate function (GroupConcat or MultiDistinctGroupConcat) + * @param context the relation carrying the column mapping + * @return the SQL text, or null when the shape has no faithful rendering + */ + public String renderGroupConcat(AggregateFunction fn, SQLRelation context) { + List<Expression> children = fn.children(); + int firstOrder = children.size(); + for (int i = 0; i < children.size(); i++) { + if (children.get(i) instanceof OrderExpression) { + firstOrder = i; + break; + } + } + List<Expression> values = children.subList(0, firstOrder); + List<Expression> orders = children.subList(firstOrder, children.size()); + // the dedicated grammar carries one value expression plus an optional CONSTANT + // separator; two NON-constant value arguments cannot be written down at all (the + // analyzer rejects them: "requires separator must be a constant"), so such a + // programmatically-built shape keeps failing the decompile instead of freezing + // SQL that cannot be re-parsed + boolean separatorArgument = values.size() == 2 && values.get(1).isConstant(); + if (values.isEmpty() || (values.size() > 1 && !separatorArgument)) { + return null; + } + // MultiDistinctGroupConcat is the execution shape of GROUP_CONCAT(DISTINCT ...): + // its constructor keeps isDistinct() = false, but the dedup contract must survive + // the freeze + boolean distinct = fn.isDistinct() || fn instanceof MultiDistinctGroupConcat; + StringBuilder sb = new StringBuilder("GROUP_CONCAT("); + if (distinct) { + sb.append("DISTINCT "); + } + sb.append(print(values.get(0), context)); + if (!orders.isEmpty()) { + // GROUP_CONCAT(DISTINCT v ORDER BY k) re-analyzes to the very + // MultiDistinctGroupConcat(v, k) execution shape it was rendered from, so the + // ORDER BY form is faithful; a verifier may also compare it against the + // alternative spelling MULTI_DISTINCT_GROUP_CONCAT(v ORDER BY k) + sb.append(" ORDER BY "); + sb.append(orders.stream().map(order -> { + OrderExpression orderExpression = (OrderExpression) order; + return print(orderExpression.child(), context) + + (orderExpression.isAsc() ? " ASC" : " DESC") + + (orderExpression.isNullFirst() ? " NULLS FIRST" : " NULLS LAST"); + }).collect(Collectors.joining(", "))); + } + if (separatorArgument) { + sb.append(" SEPARATOR ").append(print(values.get(1), context)); + } + return sb.append(")").toString(); + } + + @Override + public String visitCast(Cast cast, SQLRelation context) { + if (!cast.isExplicitType() && !preservesLoweredAccumulator(cast)) { Review Comment: [P1] Keep the semantic conversion of typed literal constructors With default enable_strict_cast=false, capture and identically replay SELECT DATE 'not-a-date' AS d FROM t. visitTypeConstructor catches the invalid date literal and creates Cast(STRING 'not-a-date' AS DATEV2) with isExplicitType=false: ```text Capture: Project(Cast(ph1[STRING] AS DATEV2) AS d) -> Scan(t) Frozen: Project(ph1 AS d) -> Scan(t) ``` The nonfoldable marker preserves the cast during capture, but this branch strips it. Ordinary execution yields typed DATE NULL; frozen replay returns the string not-a-date instead. CREATE's probe only replans and accepts that string projection. Preserve parser-origin semantic conversions or decline freezing them, and cover DATE/TIMESTAMP constructor fallback. This differs from the repaired explicit-CAST simplification report: this conversion has non-explicit parser provenance and is removed by the decompiler itself. ########## fe/fe-core/src/main/java/org/apache/doris/nereids/spm/SPMOptimizer.java: ########## @@ -0,0 +1,819 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.spm; + +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.UserException; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.cost.Cost; +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.properties.SelectHint; +import org.apache.doris.nereids.properties.SelectHintSetVar; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.Command; +import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalSelectHint; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.OriginStatement; +import org.apache.doris.qe.SessionVariable; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.BitSet; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * SPMOptimizer - baseline-dedicated optimizer (M3). + * + * Corresponds to design doc section 6.7. When a baseline is created, a dedicated + * optimizer is used that deliberately disables many "state-sensitive" optimization + * rules (MV rewrite, table pruning, UKFK JOIN pruning, equivalence derivation, + * structural rewrites, ...), so the baseline plan only depends on SQL semantics and the + * general cost model - because a baseline is a cross-time promise that must remain + * reproducible and semantically correct even after data, statistics or MVs change. + * + * Rule exclusion mechanism (WHITELIST mode): the session variable enable_nereids_rules + * carries a comma-separated rule WHITELIST; when non-empty, the engine only applies the + * listed rules (the statement-level rule mask forbids every rule outside the list, see + * StatementContext#getOrCacheDisableRules). SPMOptimizer temporarily replaces that + * variable with the SPM whitelist - every RuleType except the excluded set below, see + * buildSpmEnabledRules - while a baseline is created, and restores the original value + * after optimization completes (the original value is cached before calling and + * restored in a finally block). disable_nereids_rules is NOT touched: the user's own + * disable list keeps applying on top of the whitelist. + * + * The excluded set is: + * - an explicit list of state-sensitive RuleType names (categories 1, 3, 4, 5, 6), and + * - the whole RuleTypeClass.MATERIALIZE_VIEW rule family (category 2) enumerated + * programmatically via RuleType.isMaterializedViewRule(), so new MV rules are covered + * automatically. + */ +public class SPMOptimizer { + + /** + * RuleType names excluded from the SPM whitelist (aligned with design doc 5.4). + * All names must exist in RuleType (the whitelist is parsed with + * RuleType.valueOf, which throws on unknown names). + */ + public static final List<String> SPM_EXCLUDED_RULE_NAMES = List.of( + // ===== category 1: data-state sensitive ===== + // simple aggregate to constant (wrong result on an empty table) + "REWRITE_SIMPLE_AGG_TO_CONSTANT", + // skewed JOIN salt splitting (skew pattern changes over time) + "SALT_JOIN", + // grouping sets decomposition (topology change, cardinality dependent) + "DECOMPOSE_REPEAT", + + // ===== category 1b: TYPE-changing lowerings ===== + // avg(DISTINCT x) -> sum(DISTINCT CAST(x AS LARGEINT)) / count(...): the + // LARGEINT accumulator cast is part of the lowered expression's SEMANTICS, but + // the frozen text is re-planned for a caller whose literal may have ANOTHER type + // (avg(DISTINCT k + 1) captured, avg(DISTINCT k + 0.25) matching): the replay + // then casts the substituted DECIMAL to an integer first and returns a different + // value. Keeping AVG itself leaves the coercion to the replay's own analysis. + "AVG_DISTINCT_TO_SUM_DIV_COUNT", + + // ===== category 2: MV rewrite (all) ===== + // Excluded via the whole RuleTypeClass.MATERIALIZE_VIEW family (see + // getMaterializedViewRuleNames()), not listed here. + + // ===== category 3: table pruning + UKFK ===== + "ELIMINATE_JOIN_BY_UK", // UK constraint JOIN elimination + "ELIMINATE_JOIN_BY_FK", // FK constraint JOIN elimination + "ELIMINATE_GROUP_BY_KEY", // UKFK GROUP BY key elimination + "ELIMINATE_GROUP_BY_KEY_BY_UNIFORM", // uniform-distribution GROUP BY key elimination + // GROUP BY elimination to a row-wise projection on a DECLARED UNIQUE key: + // SELECT k, SUM(v) FROM t GROUP BY k collapses to SELECT k, v FROM t once + // DataTrait proves {k} UNIQUE+NOT NULL - a proof the DECLARED UNIQUE(k) + // constraint feeds, and which schemaFingerprint does NOT capture (it hashes + // only the table id + base columns). After DROP CONSTRAINT and a second row + // with the same k the original query returns one summed row while the frozen + // replay returns two unsummed rows. Same uniqueness family as the _KEY / + // unlock rules above - exclude it, and the replay re-plans the aggregate + // against the CURRENT constraint state. + "ELIMINATE_GROUP_BY", + // ORDER BY key elimination by a declared UNIQUE key: ORDER BY a, b LIMIT 1 can + // collapse to ORDER BY a LIMIT 1, and the frozen SQL keeps the reduced order. + // Dropping the UNIQUE declaration (or its backing constraint state) changes no + // value the fingerprint hashes, so the replay may pick another row with the + // same smallest a - exclude the rule like the other UKFK ones. + "ELIMINATE_ORDER_BY_KEY", + // PK/FK-derived aggregate push down below the (FK) join: the rewritten + // topology stops being correct once the constraint state changes. The rule + // derives its rewrite from canEliminateByFk, so dropping the constraints and + // adding duplicate keys on the former primary side makes the original + // aggregate above the multiplying join return one doubled group while the + // frozen pre-aggregate replay returns duplicate undoubled rows - a wrong + // result, not just a missed rewrite. Audit (mutable-constraint consumers in + // the whitelist): the remaining canEliminateByFk / canEliminateByUk consumers + // are EliminateJoinByFK / EliminateJoinByUK (both excluded above) and the MV + // comparator family (whole RuleTypeClass excluded); this rule was the only + // unexcluded one. + "PUSH_DOWN_AGG_THROUGH_JOIN_ON_PKFK", + // uniqueness-dependent rewrites: each fires only when DataTrait proves the + // relevant slots UNIQUE (and NOT NULL), and that proof may come from a + // DECLARED UNIQUE constraint - which schemaFingerprint does not capture (it + // hashes only the table id + base columns), so dropping the declaration + // after freezing would leave the rewrite in place: + // - SIMPLIFY_WINDOW_EXPRESSION / AGG_SCALAR_SUBQUERY_TO_WINDOW_FUNCTION: + // Window(SUM(v) PARTITION BY k) -> Project(v) -> Scan(t) once t declares + // UNIQUE(k); after dropping the declaration and adding (k,10),(k,20) the + // original window returns 30,30 while the frozen replay returns 10,20; + // - PUSH_DOWN_TOP_N_DISTINCT_THROUGH_JOIN / _PROJECT_JOIN: a hard TopN + // pushed through the join / project retains rows justified by the + // declared uniqueness. + "SIMPLIFY_WINDOW_EXPRESSION", + "AGG_SCALAR_SUBQUERY_TO_WINDOW_FUNCTION", + "PUSH_DOWN_TOP_N_DISTINCT_THROUGH_JOIN", + "PUSH_DOWN_TOP_N_DISTINCT_THROUGH_PROJECT_JOIN", + + // ===== category 4: equivalence derivation ===== + "INFER_PREDICATES", // predicate derivation + "INFER_FILTER_NOT_NULL", // Filter NOT NULL derivation + "INFER_JOIN_NOT_NULL", // JOIN NOT NULL derivation + "CONSTANT_PROPAGATION", // constant propagation + + // ===== category 5: structural rewrite ===== + "EXTRACT_SINGLE_TABLE_EXPRESSION_FROM_DISJUNCTION", // split OR into single table + "OR_EXPANSION", // OR expansion into UNION + "PUSH_DOWN_FILTER_THROUGH_WINDOW", // window predicate push down + "ELIMINATE_AGG_CASE_WHEN", // aggregate CASE WHEN elimination + "ELIMINATE_OUTER_JOIN", // outer join elimination + "ELIMINATE_LIMIT", // LIMIT elimination + // ELIMINATE_LIMIT is registered in the same rule class (EliminateLimit) as + // ELIMINATE_LIMIT_ON_ONE_ROW_RELATION; excluding only the former let + // "SELECT 1 LIMIT 1" freeze the one-row child WITHOUT its LIMIT, while the + // sibling digest of "SELECT 1 LIMIT 0" still matched it (top-level LIMIT + // values are deliberately ignored during matching) - the replay then + // returned one row instead of none. + "ELIMINATE_LIMIT_ON_ONE_ROW_RELATION", + "ELIMINATE_AGGREGATE", // aggregate elimination + // two-phase LIMIT split GLOBAL(l, o) -> LOCAL(l + o, 0): the freeze keeps + // only the UPPER limit topology, the decompiled SQL then carries both + // phases as two query blocks, and the rewrite-time LIMIT merge (which only + // reaches the block that has a user-tree counterpart) can never grow the + // inner one - a captured order-free LIMIT 10 kept returning 10 rows when a + // matching user query asked for LIMIT 20. The split is an execution + // detail; freezing the single-phase limit keeps LIMIT ... OFFSET + // semantics (see SPMPlan2SQLBuilder.visitPhysicalLimit for the + // collapse of already-frozen pairs). + "SPLIT_LIMIT", + + // ===== category 6: external sources / empty relations (data dependent) ===== + "PUSH_FILTER_INTO_SCHEMA_SCAN", // schema table predicate push down + "ELIMINATE_JOIN_ON_EMPTYRELATION", // empty-relation operator elimination + "ELIMINATE_FILTER_ON_EMPTYRELATION", + "ELIMINATE_AGG_ON_EMPTYRELATION", + "ELIMINATE_PROJECT_ON_EMPTYRELATION", + "ELIMINATE_UNION_ON_EMPTYRELATION", + "ELIMINATE_TOPN_ON_EMPTYRELATION", + "ELIMINATE_SORT_ON_EMPTYRELATION", + "ELIMINATE_INTERSECTION_ON_EMPTYRELATION", + "ELIMINATE_EXCEPT_ON_EMPTYRELATION", + "ELIMINATE_LIMIT_ON_EMPTY_RELATION", + "PRUNE_EMPTY_PARTITION", + // Partition pruning freezes the CURRENT partition set: `SELECT p FROM t + // WHERE p = 99` with only p1 VALUES IN (1) prunes every partition and the + // rule emits a LogicalEmptyRelation, which the decompiler stores as WHERE + // FALSE. ADD PARTITION p99 + a 99 row changes no value schemaFingerprint + // hashes (the table id and base columns are unchanged), so the same query + // kept hitting the frozen baseline and returned zero rows instead of the + // new row. Excluding the rule keeps the live scan + + // filter in the captured plan; the frozen SQL carries no partition pin + // (only a MANUAL PARTITION(...) is ever rendered), so the replay re-derives + // the selection against the partition set of ITS time - exactly like a + // normal query. + "OLAP_SCAN_PARTITION_PRUNE" + ); + + /** + * SET_VAR keys that carry the SPM safety overrides (see optimize()): a plan-SQL hint + * must never clear or weaken them for the nested statement, otherwise the frozen plan + * could be produced with state-sensitive rewrites re-enabled (e.g. + * SET_VAR(enable_nereids_rules='') re-enables FK join elimination, so a join could be + * frozen away and return different rows once constraint state changes) or with TopN + * lazy materialization / CTE inlining undoing the plan-serialization guards. + */ + private static final Set<String> SPM_LOCKED_SET_VAR_KEYS = ImmutableSet.of( + SessionVariable.ENABLE_NEREIDS_RULES, + "topn_lazy_materialization_threshold", + SessionVariable.ENABLE_CTE_MATERIALIZE, + SessionVariable.INLINE_CTE_REFERENCED_THRESHOLD, + SessionVariable.CTE_INLINE_MODE); + + /** + * Depth of the SPM CAPTURE optimizations running on this thread (see + * {@link #isCaptureInProgress}): the parameterized plan of a baseline is optimized by + * this class, and a rule that must not lose a piece of USER-VISIBLE intent while that + * plan is turned into the frozen text (an explicit CAST over a substitutable literal) + * consults it. + */ + private static final ThreadLocal<Integer> CAPTURE_DEPTH = ThreadLocal.withInitial(() -> 0); + + /** + * The nested-context factory (see createNestedStatementContext). A seam, not a + * configuration: the production value always builds a {@link CaptureStatementContext}, + * and a unit test substitutes a recording subclass to observe that the close() in + * optimize()'s finally releases the context the plan actually used. A construction mock + * of StatementContext cannot see it - the allocation names the SUBCLASS. + */ + private static volatile java.util.function.BiFunction<ConnectContext, String, + StatementContext> nestedContextFactory = CaptureStatementContext::new; + + private SPMOptimizer() { + } + + /** + * Installs the REPLAY-side rule mask on a replaying statement's context: every + * MATERIALIZED_VIEW rewrite is forbidden while the frozen plan is re-planned. + * + * The frozen plan was produced under the SPM whitelist (see optimize) - + * every MV rewrite excluded - so its fingerprint pins the SOURCE tables. The replay + * is normally planned with the session's full rule set, and an MV that became + * eligible AFTER the baseline was created (the reported case: an async MTMV over the + * very query, built and refreshed later) then substitutes the MV's storage table for + * the source table. The post-plan fingerprint guard + * (SPMPlanner#verifyReplayMetadata, which pins the frozen plan's tables) + * rejects its own replay and, with the default enable_spm_fallback=false, the + * SELECT fails although the source table has not changed. + * + * The mask is deliberately NARROWER than the CREATE-side whitelist: it forbids + * exactly the rules that SUBSTITUTE one table for another. The remaining + * SPM-excluded rules are optimization opportunities that a re-plan may legally take + * (they either keep the frozen tables - salt join, structural rewrites, predicate + * inference - or their absence would break semantics: ELIMINATE_LIMIT turns the + * CALLER's LIMIT 0 into an empty relation, and a full CREATE mask forbids it, which + * replayed LIMIT 0 as one row). Rules that REMOVE a table (UK / FK join + * elimination, aggregate-to-constant) keep their established fail-closed behavior: + * the fingerprint mismatch surfaces through the caller's fallback policy. + * + * @param statementContext the REPLAYING statement's context; its rule cache must not + * have been computed yet (the callers install the mask before + * the planner runs) + */ + public static void installSpmReplayRuleMask(StatementContext statementContext) { + statementContext.setSpmExcludedRules(materializedViewRuleMask()); + } + + /** + * The replay mask: one bit per materialized-view rewrite rule + * (RuleType#isMaterializedViewRule(), so newly added MV rules are covered + * automatically). + * + * @return the forbidden-rule mask of the whole MV family + */ + private static BitSet materializedViewRuleMask() { + BitSet mask = new BitSet(); + for (RuleType ruleType : RuleType.values()) { + if (ruleType.isMaterializedViewRule()) { + mask.set(ruleType.ordinal()); + } + } + return mask; + } + + /** + * All materialized view rewrite rule names (RuleTypeClass.MATERIALIZE_VIEW), + * enumerated programmatically so newly added MV rules are excluded from the SPM + * whitelist automatically. + * + * @return the immutable list of MV rewrite RuleType names + */ + public static List<String> getMaterializedViewRuleNames() { + return ImmutableList.copyOf(Arrays.stream(RuleType.values()) + .filter(RuleType::isMaterializedViewRule) + .map(Enum::name) + .collect(Collectors.toList())); + } + + /** + * The full set of RuleType names excluded from the SPM whitelist: the explicit + * state-sensitive list plus the whole MV rewrite family (deduplicated). + * + * @return the immutable list of all SPM-excluded rule names + */ + public static List<String> getSpmExcludedRuleNames() { + return ImmutableList.copyOf(Stream.concat( + SPM_EXCLUDED_RULE_NAMES.stream(), + getMaterializedViewRuleNames().stream()) + .distinct() + .collect(Collectors.toList())); + } + + /** + * Builds the SPM rule whitelist written into enable_nereids_rules while a baseline is + * created: every RuleType name except the SPM-excluded set, intersected with the + * caller's existing whitelist when one is configured (a session whitelist is + * preserved, never widened). The engine turns a non-empty enable_nereids_rules into + * the statement-level forbidden-rule mask (StatementContext#getOrCacheDisableRules), + * so the excluded rules cannot apply during baseline creation. + * + * @param originalEnabled the previous raw enable_nereids_rules value (may be null / + * empty) + * @return the comma-separated whitelist (RuleType declaration order) + * @throws AnalysisException when the original value names an unknown rule, or when + * its intersection with the SPM whitelist is empty (the + * session whitelists only rules that SPM excludes) + */ + public static String buildSpmEnabledRules(String originalEnabled) throws AnalysisException { + Set<String> excluded = new LinkedHashSet<>(getSpmExcludedRuleNames()); + Set<String> original = new LinkedHashSet<>(); + if (originalEnabled != null && !originalEnabled.isEmpty()) { + for (String ruleName : originalEnabled.split(",")) { + String normalized = ruleName.trim().toUpperCase(Locale.ROOT); + if (normalized.isEmpty()) { + continue; + } + try { + RuleType.valueOf(normalized); + } catch (IllegalArgumentException e) { + throw new AnalysisException( + "Unknown rule in enable_nereids_rules: " + normalized); + } + original.add(normalized); + } + } + List<String> enabled = new ArrayList<>(); + for (RuleType ruleType : RuleType.values()) { + String name = ruleType.name(); + if (excluded.contains(name)) { + continue; + } + if (!original.isEmpty() && !original.contains(name)) { + continue; + } + enabled.add(name); + } + if (enabled.isEmpty()) { + throw new AnalysisException("enable_nereids_rules only whitelists rules that SPM excludes: " + + originalEnabled); + } + return String.join(",", enabled); + } + + /** + * Optimizes a planSql in SPM mode: parse -> analyze -> rewrite -> CBO optimize with + * the SPM rule whitelist installed in enable_nereids_rules, then return the best + * physical plan and its estimated cost. + * + * The original enable_nereids_rules value is cached before the call and restored in + * a finally block (disable_nereids_rules is not touched). A fresh StatementContext is + * used so the per-statement forbidden-rule cache + * (CascadesContext.getAndCacheDisableRules) reflects the whitelist. + * + * @param ctx the connect context (provides session variables and catalog) + * @param planSql the plan SQL to optimize (a SELECT statement, may contain SET_VAR + * hints) + * @return the optimization result (best physical plan + estimated cost) + * @throws UserException when the SQL cannot be parsed or planned + */ + public static OptimizeResult optimize(ConnectContext ctx, String planSql) throws UserException { + Plan parsed = new NereidsParser().parseSingle(planSql); + // A SELECT statement is parsed as a logical plan; DDL/DML parse to a Command. + if (!(parsed instanceof LogicalPlan) || parsed instanceof Command) { + throw new AnalysisException("SPM only supports SELECT statements: " + planSql); + } + return optimize(ctx, (LogicalPlan) parsed, planSql); + } + + /** + * Optimizes an already-parsed (possibly parameterized) plan tree in SPM mode: + * analyze -> rewrite -> CBO optimize with the SPM rule whitelist installed in + * enable_nereids_rules, then return the best physical plan and its estimated cost. + * + * This is the entry used by CREATE BASELINE when the plan tree is first + * parameterized (literals replaced by SpmConstVar / SpmConstList placeholders) and + * then optimized: the placeholders travel through the optimizer and survive into + * the physical plan, so the decompiled frozen planSql keeps the placeholder ids + * (matching the SR model where the frozen SQL is re-parsed and user values are + * substituted by id at rewrite time). + * + * @param ctx the connect context (provides session variables and catalog) + * @param logicalPlan the (possibly parameterized) SELECT plan tree + * @param originSql the original SQL text (used for the statement context / + * error messages) + * @return the optimization result (best physical plan + estimated cost) + * @throws UserException when the plan cannot be analyzed / planned + */ + public static OptimizeResult optimize(ConnectContext ctx, LogicalPlan logicalPlan, String originSql) + throws UserException { + // Plan-SQL hints are applied DURING analysis (after the SPM overrides below are + // installed): SelectHintSetVar writes the SAME SessionVariable object, so a hint + // such as SET_VAR(enable_nereids_rules='') would silently clear the whitelist and + // the TopN / CTE guards for the nested statement. Reject such hints up front - + // CREATE then keeps the user's planSql text instead of freezing an unguarded plan. + checkProtectedSetVarHints(logicalPlan); + StatementContext statementContext = createNestedStatementContext(ctx, originSql); + NereidsPlanner planner = new NereidsPlanner(statementContext); + + SessionVariable sessionVar = ctx.getSessionVariable(); + String originalEnabled = sessionVar.getEnableNereidsRulesStr(); + StatementContext originalCtx = ctx.getStatementContext(); + int originalTopnLazyThreshold = SessionVariable.getTopNLazyMaterializationThreshold(); + boolean originalCteMaterialize = sessionVar.enableCTEMaterialize; + int originalInlineCteThreshold = sessionVar.inlineCTEReferencedThreshold; + int originalCteInlineMode = sessionVar.cteInlineMode; + try { + // WHITELIST mode: only the rules SPM allows may apply while the baseline plan + // is produced. The mask is installed on THIS nested statement's context - NOT + // by redefining the public enable_nereids_rules variable, which would change + // its established behavior for every ordinary statement in the session (a + // session value like enable_nereids_rules='ELIMINATE_GROUP_BY_KEY_BY_UNIFORM' + // would forbid every binding / implementation rule not named there). The + // user's own disable_nereids_rules keeps applying on top of the mask. + statementContext.setSpmExcludedRules(buildSpmExcludedRuleMask(originalEnabled)); + // TopN lazy materialization is an execution detail (post-process): it prunes + // base-table columns from the physical plan and re-reads them later by rowid + // (PhysicalLazyMaterialize). Such pruned columns would be missing from the + // decompiled frozen planSql ("Unknown column in table list" on replay), so it + // is disabled while the baseline plan is produced - the frozen SQL must carry + // the full column set; a re-plan at rewrite time may still apply lazy + // materialization itself. + sessionVar.setTopNLazyMaterializationThreshold(-1); + // The WITH structure of the user query must survive into the frozen planSql: + // the decompiler turns PhysicalCTEAnchor / PhysicalCTEProducer / + // PhysicalCTEConsumer into a real WITH clause (one shared definition, + // referenced by alias), like StarRocks. Three settings are overridden while + // the baseline plan is produced: + // - enable_cte_materialize / inline_cte_referenced_threshold: by default the + // engine inlines a CTE with a single consumer (a common TPCDS shape), which + // deletes the anchor from the optimized plan before the decompiler can see it; + // - cte_inline_mode = -1: mode 0 (the default) builds an alternative fully + // inlined plan and uses it when consumer filters can eliminate union branches + // of the CTE body, which silently drops WITH for such queries (TPCDS q04/ + // q11/q74); SPM needs the anchored plan, and a re-plan at rewrite time still + // applies the user's own cte_inline_mode. + // CTEInline still inlines the CTEs a recursive CTE requires inlined + // (StatementContext mustInlineCTEs), so WITH RECURSIVE planning is unaffected. + sessionVar.enableCTEMaterialize = true; + sessionVar.inlineCTEReferencedThreshold = 0; + sessionVar.cteInlineMode = -1; + // Install the FRESH statement context for the whole nested plan: the table + // collector caches the SELECT tables into the ConnectContext's CURRENT + // statement context, and collectAndLockTable() then locks THAT object. When + // the caller (StmtExecutor for a CREATE BASELINE statement) already installed + // the outer command's context, leaving it active made collectRelation cache + // into the outer object while lock() ran on the fresh (empty) one: the nested + // plan was optimized / decompiled WITHOUT its metadata-stability locks and + // could race concurrent ALTER / DROP. The exact original (possibly null) is + // restored in the finally below. + ctx.setStatementContext(statementContext); + // planWithLock runs preprocess (SET_VAR hint) -> analyze -> rewrite -> + // optimize -> postProcess; distribution planning is not needed for the + // decompiler (the physical plan already carries distribution specs). + // The root physical plan is the RETURN value of planWithLock: the planner's + // physicalPlan field is only assigned through the lockCallback used by + // plan(), so planner.getPhysicalPlan() would be null here. + // + // The result becomes the FROZEN text, so the thread is marked as a CAPTURE + // while it is produced: a rule that would erase user-visible intent the frozen + // text must keep (an explicit CAST over a substitutable literal, see + // SimplifyCastRule) holds back. + CAPTURE_DEPTH.set(CAPTURE_DEPTH.get() + 1); + Plan resultPlan; + try { + resultPlan = planner.planWithLock(logicalPlan, + PhysicalProperties.ANY, ExplainLevel.NONE); + } finally { + CAPTURE_DEPTH.set(CAPTURE_DEPTH.get() - 1); + } + if (!(resultPlan instanceof PhysicalPlan)) { + throw new AnalysisException("SPM failed to plan SQL: " + originSql); + } + // Belt-and-braces for any hint path the pre-scan does not model: the protected + // variables must still carry the SPM values after planning, otherwise the plan + // may have been produced without a guard - fail the freeze instead of + // publishing it. + verifySpmOverridesIntact(sessionVar); + // The bind-side tables of this plan were resolved UNDER the locks taken above + // and are read here, while the nested statement context that owns them is still + // open: a fingerprint computed later can only re-resolve the LIVE catalog (see + // OptimizeResult#tableSnapshot). + return new OptimizeResult((PhysicalPlan) resultPlan, + extractCost((PhysicalPlan) resultPlan), + snapshotTables(statementContext.getTables()), + statementContext instanceof CaptureStatementContext + ? ((CaptureStatementContext) statementContext).getTableFingerprints() + : java.util.Map.of()); + } finally { + sessionVar.setTopNLazyMaterializationThreshold(originalTopnLazyThreshold); + sessionVar.enableCTEMaterialize = originalCteMaterialize; + sessionVar.inlineCTEReferencedThreshold = originalInlineCteThreshold; + sessionVar.cteInlineMode = originalCteInlineMode; + ctx.setStatementContext(originalCtx); + // The nested statement context OWNS everything the nested plan + // registered on it - the connector scope an external (e.g. Iceberg) bind + // installed, whose table-cache lease / tracked tables are released only by + // close() (connectorStatementScope.closeAll), plus its planner resources + // (releasePlannerResources). Restoring the outer context above is NOT a + // substitute: the statement's own teardown (ConnectProcessor / the + // forwarded-request finally) later closes only the context the session + // carries at that point, so without this close every CREATE / capture refresh + // left the nested scope pinned for the lifetime of the FE. + statementContext.close(); + } + } + + /** + * Creates the nested statement context of one baseline-plan optimization. The context + * is closed in the caller's finally. + */ + private static StatementContext createNestedStatementContext(ConnectContext ctx, + String originSql) { + return nestedContextFactory.apply(ctx, originSql); + } + + /** + * Installs a nested-context factory for one test (see nestedContextFactory). + * + * @param factory the factory, or null to restore the production one + */ + static void setNestedContextFactoryForTest(java.util.function.BiFunction<ConnectContext, + String, StatementContext> factory) { + nestedContextFactory = factory == null ? CaptureStatementContext::new : factory; + } + + /** + * The locked bind-side tables of one nested plan, keyed like + * SPMPlanTreeSupport#lockedTablesByQualifiedName (bare name and database-qualified + * name, lower case). MUST be read before the nested statement context is closed. + * + * @param tables the nested statement's resolved tables (qualifier to TableIf) + * @return lowercase bare / db-qualified table name to the under-lock TableIf + */ + private static java.util.Map<String, org.apache.doris.catalog.TableIf> snapshotTables( + java.util.Map<List<String>, org.apache.doris.catalog.TableIf> tables) { + if (tables == null || tables.isEmpty()) { + return java.util.Map.of(); + } + java.util.Map<String, org.apache.doris.catalog.TableIf> snapshot = new java.util.HashMap<>(); + for (org.apache.doris.catalog.TableIf table : tables.values()) { + if (table == null || table.getName() == null) { + continue; + } + snapshot.putIfAbsent(table.getName().toLowerCase(Locale.ROOT), table); + if (table.getDatabase() != null && table.getDatabase().getFullName() != null) { + snapshot.putIfAbsent((table.getDatabase().getFullName() + "." + table.getName()) + .toLowerCase(Locale.ROOT), table); + } + } + return snapshot; + } + + /** + * Rejects SET_VAR hints that target an SPM-protected session variable. Public for + * tests (the hint objects are constructed directly there). + */ + public static void rejectProtectedSetVarHints(List<SelectHint> hints) throws AnalysisException { + if (hints == null) { + return; + } + for (SelectHint hint : hints) { + if (!(hint instanceof SelectHintSetVar)) { + continue; + } + for (String key : ((SelectHintSetVar) hint).getParameters().keySet()) { + if (key != null && SPM_LOCKED_SET_VAR_KEYS.contains(key.toLowerCase(Locale.ROOT))) { + throw new AnalysisException("SPM cannot freeze a plan whose SET_VAR hint targets the" + + " SPM-protected session variable '" + key + "'"); + } + } + } + } + + /** + * Walks the WHOLE statement and rejects any SELECT hint targeting a protected + * variable. The walk must follow the plans held outside children() (CTE bodies via + * extraPlans(), IN / EXISTS / scalar subquery plans via SubqueryExpr.queryPlan and + * through expression coercions): a hint inside a CTE definition or a subquery would + * otherwise escape the pre-scan and EliminateLogicalSelectHint would apply it AFTER + * the SPM CTE / TopN serialization overrides were installed, letting CTE inlining or + * TopN materialization change the frozen plan. + * + * Package-visible for tests. + */ + static void checkProtectedSetVarHints(Plan node) throws AnalysisException { + SPMPlanTreeSupport.<AnalysisException>walkPlans(node, (Plan current) -> { + if (current instanceof LogicalSelectHint) { + rejectProtectedSetVarHints(((LogicalSelectHint<?>) current).getHints()); + } + }); + } + + /** + * Fails when the SPM overrides were altered while the nested statement was planned. + */ + private static void verifySpmOverridesIntact(SessionVariable sessionVar) throws AnalysisException { + if (SessionVariable.getTopNLazyMaterializationThreshold() != -1 + || !sessionVar.enableCTEMaterialize + || sessionVar.inlineCTEReferencedThreshold != 0 + || sessionVar.cteInlineMode != -1) { + throw new AnalysisException("SPM safety overrides were altered during planning;" + + " the plan cannot be frozen"); + } + } + + /** + * The per-statement rule mask for a baseline-creation plan: every RuleType outside the + * SPM whitelist (built from the session's own enable_nereids_rules value, if any) is + * forbidden, except the engine-essential privilege / row-policy checks which are never + * gated. + */ + public static BitSet buildSpmExcludedRuleMask(String originalEnabled) throws AnalysisException { + Set<String> enabled = new HashSet<>(Arrays.asList(buildSpmEnabledRules(originalEnabled).split(","))); + BitSet mask = new BitSet(); + for (RuleType ruleType : RuleType.values()) { + if (ruleType == RuleType.CHECK_PRIVILEGES || ruleType == RuleType.CHECK_ROW_POLICY) { + continue; + } + if (!enabled.contains(ruleType.name())) { + mask.set(ruleType.type()); + } + } + return mask; + } + + /** + * Extracts the estimated cost of the root physical plan. + * + * The cost is estimated over the PARAMETERIZED plan tree, whose literals are still + * SpmConstVar / SpmConstList placeholder calls (not concrete values), so predicates + * such as c > SpmConstVar(...) cannot be folded and their selectivity falls back to + * the optimizer's default (e.g. 1/3). The absolute value is therefore not the cost + * of any one concrete query; it is only used as a RELATIVE ranking key among the + * baselines that share one bind digest (when several stored plans exist, the + * cheapest frozen plan wins - see BaselineManager), exactly like StarRocks, which + * also takes the CBO cost of the placeholder-carrying optimized plan + * (optimizedPlan.getCost()) with no placeholder-specific adjustment. + * + * @param physicalPlan the root physical plan + * @return the cost value (0 when the group cost is not available) + */ + private static double extractCost(PhysicalPlan physicalPlan) { + try { + if (physicalPlan.getGroupExpression().isPresent()) { + GroupExpression groupExpression = physicalPlan.getGroupExpression().get(); + Cost cost = groupExpression.getCostValueByProperties(PhysicalProperties.ANY); + if (cost != null) { + return cost.getValue(); + } + } + } catch (RuntimeException e) { + // cost is best-effort; fall through to 0 + } + return 0; + } + + /** + * Result of an SPM-mode optimization: the best physical plan and its estimated cost. + */ + public static class OptimizeResult { + private final PhysicalPlan physicalPlan; + private final double cost; + /** + * The under-lock metadata snapshot of the statement's catalog relations, keyed by + * bare / database-qualified lowercase table name (see + * SPMPlanTreeSupport#schemaFingerprintForCreate). The nested statement context + * that owns these TableIf objects is CLOSED when optimize() returns, so a + * fingerprint computed afterwards could only re-resolve the tables from the LIVE + * catalog - hashing a schema a concurrent ALTER may already have changed while + * the frozen output slots still describe the older one (the CREATE-time + * fingerprint would then accept a replay of the stale frozen plan). + */ + private final java.util.Map<String, org.apache.doris.catalog.TableIf> tableSnapshot; + + /** + * Table fingerprint entries hashed WHILE the planning locks were held (see + * {@link CaptureStatementContext}): lower-case bare / db-qualified / + * catalog-qualified table name to the entry. The TableIf objects are MUTABLE in + * place, so a fingerprint computed after optimize() returned could pair the frozen + * output slots with a NEWER schema. + */ + private final java.util.Map<String, String> tableFingerprints; + + public OptimizeResult(PhysicalPlan physicalPlan, double cost) { + this(physicalPlan, cost, java.util.Map.of(), java.util.Map.of()); + } + + public OptimizeResult(PhysicalPlan physicalPlan, double cost, + java.util.Map<String, org.apache.doris.catalog.TableIf> tableSnapshot) { + this(physicalPlan, cost, tableSnapshot, java.util.Map.of()); + } + + public OptimizeResult(PhysicalPlan physicalPlan, double cost, + java.util.Map<String, org.apache.doris.catalog.TableIf> tableSnapshot, + java.util.Map<String, String> tableFingerprints) { + this.physicalPlan = physicalPlan; + this.cost = cost; + this.tableSnapshot = tableSnapshot == null ? java.util.Map.of() : tableSnapshot; + this.tableFingerprints = tableFingerprints == null + ? java.util.Map.of() : tableFingerprints; + } + + public PhysicalPlan getPhysicalPlan() { + return physicalPlan; + } + + public double getCost() { + return cost; + } + + /** + * The bind-side tables as they were resolved UNDER the planning locks (see the + * field javadoc); empty when unknown. + * + * @return lowercase bare / db-qualified table name to the locked TableIf + */ + public java.util.Map<String, org.apache.doris.catalog.TableIf> getTableSnapshot() { + return tableSnapshot; + } + + /** + * The table fingerprint entries hashed under the planning locks (see the field + * javadoc). + * + * @return lowercase table name to the fingerprint entry + */ + public java.util.Map<String, String> getTableFingerprints() { + return tableFingerprints; + } + } + + /** + * The nested statement context of one baseline-plan optimization: it records the + * fingerprint of EVERY table the capture analysis resolves, at the moment the table is + * cached - i.e. while the planning locks are held. + * + * A TableIf is mutable in place: `ALTER TABLE ... ADD COLUMN` updates the SAME object, so + * describing it after `planWithLock` released the locks hashed a schema the frozen output + * slots (and the decompiled planSql) never saw. The replay guard then accepted the stale + * baseline and silently returned the old column set. + */ + private static final class CaptureStatementContext extends StatementContext { + private final java.util.Map<String, String> tableFingerprints = + new java.util.concurrent.ConcurrentHashMap<>(); + + CaptureStatementContext(ConnectContext ctx, String originSql) { + super(ctx, new OriginStatement(originSql, 0)); + } + + @Override + public org.apache.doris.catalog.TableIf getAndCacheTable(List<String> tableQualifier, + TableFrom tableFrom, + java.util.Optional<org.apache.doris.nereids.analyzer.UnboundRelation> unboundRelation) { + org.apache.doris.catalog.TableIf table = + super.getAndCacheTable(tableQualifier, tableFrom, unboundRelation); + SPMPlanTreeSupport.recordTableFingerprint(tableFingerprints, table, tableQualifier); Review Comment: [P1] Record capture fingerprints only after taking metadata locks CollectRelation calls this override before NereidsPlanner.collectAndLockTable acquires the table locks; recordTableFingerprint then keeps that first schema via putIfAbsent. For t(k,v), capture SELECT t.*, k + 1 AS z FROM t: the collector records [k,v], a concurrent DROP COLUMN v finishes, and the locked capture freezes Project(k,k+ph AS z) over Scan(t[k]). Binding cannot replace the old fingerprint, and the replan probe discards its own snapshot. Re-adding v with the same type/nullability restores the name/type fingerprint. SELECT t.*, k + 1 AS z then passes both fingerprint checks but replays only [k,z], omitting v. Populate the map under the acquired planning locks. The earlier post-unlock fingerprint report concerns another location; this replacement hook records its first value before those locks. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
