github-actions[bot] commented on code in PR #69322: URL: https://github.com/apache/doris/pull/69322#discussion_r4358044306
########## fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboJoinConditions.java: ########## @@ -0,0 +1,222 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NullSafeEqual; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.Join; + +import com.google.common.hash.Hashing; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; + +/** + * Canonical description and fingerprint of the equality conditions of a join, used as the key of + * manually injected join expansion entries ({@code HBO SET EXPANSION}). + * + * <p>The join type and the residual (non equality) conditions take part in the key next to the hash + * (equality) conjuncts, because the injected factor replaces the output rows of the whole join: an + * inner join which matches nothing and a left outer join of the same columns produce very different + * row counts, and a selective residual predicate changes the output by orders of magnitude. The + * shape of both sub trees is still excluded, so one injected entry stays reusable by every query + * which joins the same columns with the same join type and the same residual conditions. + * + * <p>The canonical form is {@code JE{<join type>;EqualTo(...);...}} with the pairs and the residual + * conditions sorted; a pair is rendered by {@link GroupStructInfo#normalizeExpression(Expression)} + * which already sorts the two operands of an equality (equality is commutative), so {@code a = b} + * and {@code b = a} collapse. A residual condition is rendered with its literal in both modes, + * because it decides how many partner rows a row finds ({@code b > c + 1} is not {@code b > c + 2}). + */ +public class HboJoinConditions { + private static final String PREFIX = "JE{"; + private static final String SEP = ";"; + + private HboJoinConditions() { + } + + /** True when the injected expansion of this join type is applicable at all. */ + public static boolean isExpansionApplicable(JoinType joinType) { + // semi / anti / mark style joins can never produce more rows than their left input, and + // asof joins match at most one row per left row, so expansion can not happen there + return joinType == JoinType.INNER_JOIN + || joinType == JoinType.LEFT_OUTER_JOIN + || joinType == JoinType.RIGHT_OUTER_JOIN + || joinType == JoinType.FULL_OUTER_JOIN; + } + + /** + * Canonical string of the conditions of a join, empty when the join has no expansion key at all. + * + * <p>The key describes the <b>equality</b> conditions only: the factor is the fan out of the equi + * join, and the non equality conditions are applied by the estimator on top of it (see + * {@code JoinEstimation}), so a join with a residual predicate uses the factor of its equality pair + * and the residual keeps its own selectivity. The operands have to name the columns of the tables + * themselves: an operand which is not a direct slot of a catalog table (a computed column, an + * aggregate output, a cast) cannot name its table, so such a join has no key. A null safe equality + * ({@code <=>}) is a hash conjunct as well and takes part like a plain equality. + */ + public static Optional<String> canonicalOf(Join join) { + return keyParts(join).map(parts -> PREFIX + String.join(SEP, parts) + "}"); + } + + /** + * The key of the expansion entry of a join: the sha256 of its structural identity, empty when the + * join has no expansion key (see {@link #canonicalOf}). The canonical string is the readable form of + * the same parts: it is printed by EXPLAIN and stored as the struct of the entry, and the key is not + * derived from the text. + */ + public static Optional<String> fingerprintOf(Join join) { + return keyParts(join).map(parts -> { + ShapeKeyWriter key = new ShapeKeyWriter(); + key.tag(0); + for (String part : parts) { + key.text(part); + } + return Hashing.sha256().hashBytes(key.toByteArray()).toString(); + }); + } + + /** + * The identity of a join expansion: its type, the equality pairs, the residual conditions and the + * columns of the tables the join reads. The identity is the same while a group is estimated (its + * children are group references there) and while the chosen plan is printed, because the operands are + * read from the slots themselves. + */ + private static Optional<List<String>> keyParts(Join join) { + Set<String> operands = new TreeSet<>(); + List<String> forward = new ArrayList<>(); + List<String> backward = new ArrayList<>(); + for (Expression conjunct : join.getHashJoinConjuncts()) { + if (!(conjunct instanceof EqualTo) && !(conjunct instanceof NullSafeEqual)) { + return Optional.empty(); + } + Optional<String> left = operandIdentity(conjunct.child(0)); + Optional<String> right = operandIdentity(conjunct.child(1)); + if (!left.isPresent() || !right.isPresent()) { + return Optional.empty(); + } + operands.add(left.get()); + operands.add(right.get()); + String name = conjunct instanceof NullSafeEqual ? "NullSafeEqual" : "EqualTo"; + Integer leftSide = sideOf(join, conjunct.child(0)); + Integer rightSide = sideOf(join, conjunct.child(1)); + if (leftSide == null || rightSide == null || leftSide.equals(rightSide)) { + String first = left.get().compareTo(right.get()) <= 0 ? left.get() : right.get(); + String second = first.equals(left.get()) ? right.get() : left.get(); + String pair = name + "(col(" + first + "),col(" + second + "))"; + forward.add(pair); + backward.add(pair); + } else { + forward.add(leftSide == 0 Review Comment: [P2] Encode expansion operands before assembling pair text Raw quoted column names can forge the operand boundary before `fingerprintOf` length-prefixes the complete pair. For tables `internal.db.l` and `internal.db.r`, name left column A `a),col(internal.db.r.b` and right column B `b),col(internal.db.r.z` (supported with the default Unicode-name setting). Compare these reduced INNER joins: ``` common: l.a=r.c1, A=r.c2, l.c3=B, l.c4=r.z extra: A=r.z -- versus l.a=B ``` Both extra pairs become exactly `EqualTo(col(internal.db.l.a),col(internal.db.r.b),col(internal.db.r.z))`. The four common pairs make the operand sets identical too, so all key fields and actual fingerprints collide. In positional columns `(a,A,c3,c4)` / `(c1,c2,B,z)`, give both tables rows `(0,0,0,0),(0,1,1,1),(1,0,1,0),(1,1,0,1)`: the joins return four versus two rows despite identical marginal column statistics. A pinned expansion therefore crosses different equality populations without another guard. The supported setting `disable_nereids_rules='INFER_PREDICATES,CONSTANT_PROPAGATION,REORDER_JOIN,REWRITE_JOIN_EXPRESSION'` preserves these five direct hash predicates; duplicate elimination removes none. Encode comparison kind and each operand separately, then render the readable text. This differs from the existing table-binding/self-join threads: both sides are correctly bound to different named tables here. ########## fe/fe-core/src/main/java/org/apache/doris/nereids/stats/GroupStructInfo.java: ########## @@ -0,0 +1,3300 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.nereids.memo.Group; +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.OrderKey; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.And; +import org.apache.doris.nereids.trees.expressions.ArrayItemReference; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Match; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.NullSafeEqual; +import org.apache.doris.nereids.trees.expressions.NullToNonNullFunction; +import org.apache.doris.nereids.trees.expressions.Or; +import org.apache.doris.nereids.trees.expressions.OrderExpression; +import org.apache.doris.nereids.trees.expressions.SearchExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.WindowExpression; +import org.apache.doris.nereids.trees.expressions.functions.Function; +import org.apache.doris.nereids.trees.expressions.functions.Udf; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda; +import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.trees.plans.AbstractPlan; +import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.Aggregate; +import org.apache.doris.nereids.trees.plans.algebra.Except; +import org.apache.doris.nereids.trees.plans.algebra.Filter; +import org.apache.doris.nereids.trees.plans.algebra.Generate; +import org.apache.doris.nereids.trees.plans.algebra.Intersect; +import org.apache.doris.nereids.trees.plans.algebra.Join; +import org.apache.doris.nereids.trees.plans.algebra.Limit; +import org.apache.doris.nereids.trees.plans.algebra.OlapScan; +import org.apache.doris.nereids.trees.plans.algebra.Project; +import org.apache.doris.nereids.trees.plans.algebra.Repeat; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation; +import org.apache.doris.nereids.trees.plans.algebra.Sort; +import org.apache.doris.nereids.trees.plans.algebra.TopN; +import org.apache.doris.nereids.trees.plans.algebra.Union; +import org.apache.doris.nereids.trees.plans.algebra.Window; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEAnchor; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEProducer; +import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; +import org.apache.doris.nereids.trees.plans.physical.PhysicalFileScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalGenerate; +import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; +import org.apache.doris.nereids.trees.plans.physical.PhysicalStorageLayerAggregate; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.util.MutableState; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.hash.Hashing; +import com.google.common.primitives.UnsignedBytes; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Simplified struct info of a memo {@link Group}, used by History Based Optimization (HBO). + * + * <p>The struct info is a property of the <b>group</b>, not of one of its group expressions: all + * expressions of a group are logically equivalent, so they describe the same set of relations with + * the same predicates, and the canonical form below is deliberately insensitive to which expression + * is picked. + * <ul> + * <li>join commutativity: children of a join are canonicalized by sorting;</li> + * <li>join associativity: a whole inner / cross join chain is emitted as one node + * ({@code J{inner,c:[...]}(...;...)}) whose children are the chain leaves sorted by their + * canonical form and whose conditions are the sorted union of every condition of the chain, + * so {@code (A join B) join C} and {@code A join (B join C)} produce the same descriptor;</li> + * <li>aggregation: only the group by keys and the child participate, because the output row count + * of an aggregation is the number of groups and cannot be changed by the aggregate functions + * or by the output expressions;</li> + * <li>project is transparent for structure matching.</li> + * </ul> + * This mirrors what the MV oriented {@code rules.exploration.mv.StructInfo} does (it identifies a + * sub tree by its relation set and by level independent - shuttled - predicates), with every MV + * specific field dropped: no hyper graph, no shuttle map, no equivalence class, no lineage. + * + * <p>What the canonical form keeps: + * <ul> + * <li>for each scan: the table qualifier and - as an annotation only - the data state of the scan + * (its visible version, the rows it reads and how many of the table partitions it selects, + * see {@link HboScanDescriptor});</li> + * <li>for filter / join / aggregate: the normalized predicates / join conditions / grouping keys, + * i.e. the structural pattern only (slot and expression ids are stripped by + * {@link #normalizeExpression}).</li> + * </ul> + * The string contains no per-query identifier, so it is reproducible across runs and queries for + * structurally identical sub trees, and its sha256 is used as the HBO cache key (fingerprint). + * + * <p>The sha256 is taken over the canonical string <b>without</b> the scan baselines + * ({@link #stripScanBaseline}): a hbo key identifies a plan pattern and the relation it reads, not a + * data state, so an entry whose table simply keeps growing stays matchable instead of silently never + * matching again. Whether an entry may still be applied is decided on the read side, by comparing + * the baseline of the entry with the data state of the current query (see + * {@link HboStructFreshness}). + * + * <p>The descriptor is derived from the group's logical expression and its child groups (memo level + * traversal, with a visited set to guard shared sub graphs such as CTE). Groups whose content is not + * supported (e.g. contains TVF / CTE consumer, or the head operator is not one of the supported + * kinds) are {@link #isValid() invalid} and callers must fall back to the legacy behavior. + * + * <p>This description is always built by the FE from the plan it is looking at, and every string it can + * produce is one of its renderings: {@link #getCanonical(LiteralMode)} is the text EXPLAIN prints and a + * user may paste into {@code HBO SET STATISTICS ... STRUCT='...'}, {@link #getShape(LiteralMode)} is the + * same text without the data state of the scans, and {@link #getFingerprint(LiteralMode)} is the sha256 + * of that shape. Nothing is ever parsed to produce a fingerprint - in particular the text a user pasted + * is only compared with the fingerprint next to it, never taken as its source. + */ +public class GroupStructInfo { + /** + * The default of {@code hbo_max_scans_per_group}, used when no session is available. 20 keeps + * every TPC-DS shape observed in this repository (the largest single sub tree is query64's + * 19 scan join group) while refusing trees which are pathologically wide. + */ + public static final int DEFAULT_MAX_SCANS_PER_GROUP = 20; + /** + * Snapshot value which marks a plan node that is deliberately excluded from the hbo + * fingerprint (its group is unsupported, e.g. a sort / topn). It is not a fingerprint and is + * never hashed: the publish path uses it to tell an expected gap from a lost snapshot. A + * concurrent map can not store a null marker, so an empty string is used. + */ + public static final String UNSUPPORTED_FINGERPRINT = ""; + + /** + * The prefix of the reason a node whose kind the description does not support is reported with, and + * the two halves of the reason a node with an unsupported number of children is reported with. They + * are constants so that the corpus can bind the frozen skip texts to the formula which produces them + * instead of restating the prefix. + */ + static final String UNSUPPORTED_REASON_PREFIX = "unsupported="; + static final String ARITY_REASON_PREFIX = "arity="; + static final String ARITY_REASON_MIDDLE = " of "; + + private static final List<HboScanDescriptor> INVALID_SCANS = Collections.emptyList(); + + /** + * Characters which structure the canonical form and therefore can not appear raw inside a + * literal value (see {@link #encodeLiteralValue}). + */ + private static final String RESERVED_LITERAL_CHARS = "\u0000\\%(){}[],;:|"; + + private static final char[] HEX_DIGITS = "0123456789ABCDEF".toCharArray(); + + /** + * How many groups this query described. A group is described once: the description is memoized on + * the group and composed by its parents, so this number only grows when a new group is reached. + */ + private static final AtomicInteger describedGroupCount = new AtomicInteger(); + + /** + * How many descriptions were walked while materializing a string. A description whose string is + * already materialized is copied instead of being walked again, so this number does not grow for + * the sub tree of an already materialized child. + */ + private static final AtomicInteger materializedGroupCount = new AtomicInteger(); + + /** Shared invalid instance: the group content is not supported by the simplified struct info. */ + public static final GroupStructInfo INVALID = new GroupStructInfo(false, "", null, INVALID_SCANS); + + private static final String SEP = ";"; + + /** + * The header letters whose arguments are part of the identity of their node (see + * {@link Kind#literalsAreIdentity()}): the fold of a pasted struct info copies those headers verbatim, + * because folding one of their literals would make the sha256 of the text differ from the fingerprint + * EXPLAIN printed. The set is derived from the kinds which render the text, so a kind whose literals + * stop being identity cannot be forgotten here. + */ + private static final String PROTECTED_HEADERS = protectedHeaders(); + + /** + * The markers of the headers whose condition collection is sorted per literal mode, i.e. the ones the + * fold of a pasted struct info has to re-sort after it folded their literals (see + * {@code recanonicalizeFoldedConditions}): a filter, an aggregation, a join (which also de-duplicates) + * and the grouping sets of a repeat. + */ + private static final String[] CONDITION_HEADER_MARKERS = {"F{", "A{gb:", "J{", "R{gs:["}; + + /** + * The functions whose value depends on the session time zone although neither their arguments nor + * their result are a date: an epoch value is read in that zone ({@code hour_from_unixtime(t.ts) = 0} + * keeps 1000 rows in one session and 1 row in another although its argument is the same integer). + * A function whose argument or result is a date or a datetime is covered by its type instead (see + * {@link #usesSessionTimeZone}). + */ + private static final List<String> EPOCH_TIME_ZONE_FUNCTIONS = Arrays.asList( + "hour_from_unixtime", "minute_from_unixtime", "second_from_unixtime", + "microsecond_from_unixtime", "unix_timestamp"); + + private static final String NO_LOOKUP_PATH_REASON = + "storage layer aggregate: the read side applies the row count of the aggregation above it"; + + private static final String PARTIAL_STAGE_REASON = + "local (partial) aggregation stage: the read side applies the row count of the final" + + " aggregation above it"; + + private static final Logger LOG = LogManager.getLogger(GroupStructInfo.class); + + /** + * Whether literal values participate in the canonical string / fingerprint. + * <ul> + * <li>{@link #WITH_LITERAL} keeps the literal value ({@code lit(10:int)}): the fingerprint is + * bound to the exact constants, used for filter roots where the constant changes the + * output row count;</li> + * <li>{@link #NO_LITERAL} replaces every literal value with {@code *} ({@code lit(*)}): the + * fingerprint is constant agnostic. It is the primary key of a scan, of a constant-free + * filter shape and of the key the publish path generates for a join / aggregate root; every + * root which is annotated also carries a literal carrying form.</li> + * </ul> + * The literal data type is intentionally not kept: nereids inserts an explicit {@code Cast} + * for mismatching operand types, so two semantically different literal forms already differ by + * the cast node, while equivalent forms (e.g. {@code d > 1} and {@code d > 1.0} on a decimal + * column) are allowed to collapse. + */ + public enum LiteralMode { + WITH_LITERAL, + NO_LITERAL + } + + /** + * Whether the description could be built. It is invalid - and the memo caches that answer, so every + * node above the sub tree gets no fingerprint for the whole query - when one of the following holds + * for the sub tree it covers: + * <ul> + * <li>it contains an operator the canonical form does not describe, which the annotation prints as + * {@code unsupported=<Operator>}: a table valued function relation ({@code numbers}, + * {@code s3}), an apply, a recursive union, a partition top-n, ...;</li> + * <li>it reads more scans than {@code hbo_max_scans_per_group}: + * {@code subtree scans=N > hbo_max_scans_per_group=M};</li> + * <li>a table name carries a struct info delimiter ({@code ,} or {@code }}), so the printed token + * could not be read back: {@code delimiter in table name <table>};</li> + * <li>the memo does not look like a canonical form can express it: a unary operator whose group + * expression has another arity, a CTE consumer which has no memo edge to its producer, a + * reused sub tree whose description is not available;</li> + * <li>the data state of a scan cannot be read (an external table whose metadata is not available, + * a cloud rpc failure): {@code no data state for <table> (<cause>)}.</li> + * </ul> + * A node whose group cannot be resolved at all is not described either, which the annotation reports + * as {@code no group for <Operator>}. + */ + private final boolean valid; + /** + * The tree of this description: the information of its own group and the descriptions of the groups + * below it, all as objects. Nothing is rendered while it is built - {@link #render} is the one place + * which knows the grammar - and every node remembers the forms it has rendered, so a parent which is + * rendered later composes the text of a child instead of walking its sub tree again. + */ + private final Node root; + /** The scans of the sub tree, in the order their tokens appear in the canonical string. */ + private final List<HboScanDescriptor> scans; + /** Why no struct info was built, when it was refused by a limit instead of being unsupported. */ + private final String skipReason; + // the forms materialized on first request: the read side only needs a fingerprint, while the + // explain annotation needs the canonical string, so building all of them up front would be wasted + private String canonicalWithLiteral; + private String canonicalNoLiteral; + private String shapeWithLiteral; + private String shapeNoLiteral; + private String fingerprintWithLiteral; + private String fingerprintNoLiteral; + + private GroupStructInfo(boolean valid, String skipReason, Node root, + List<HboScanDescriptor> scans) { + this.valid = valid; + this.skipReason = skipReason; + this.root = root; + this.scans = scans; + } + + private static GroupStructInfo ofNode(Node root) { + List<HboScanDescriptor> scans = new ArrayList<>(); + collectScans(root, scans); + return new GroupStructInfo(true, "", root, Collections.unmodifiableList(scans)); + } + + private static GroupStructInfo invalidStructInfo(String skipReason) { + return skipReason.isEmpty() + ? INVALID + : new GroupStructInfo(false, skipReason, null, INVALID_SCANS); + } + + private static String protectedHeaders() { + StringBuilder letters = new StringBuilder(); + for (Kind kind : Kind.values()) { + if (kind.literalsAreIdentity()) { + letters.append(kind.headerLetters()); + } + } + return letters.toString(); + } + + /** The reason the struct info was refused by a limit, or empty. Only for diagnostics. */ + public String getSkipReason() { + return skipReason; + } + + public boolean isValid() { + return valid; + } + + /** The node this description is made of; null when the description is invalid. */ + private Node root() { + return root; + } + + /** + * The row count and partition selection of every scan of the sub tree, in the order the scan tokens + * appear in the canonical string (i.e. the order a user sees when pasting the struct info). A sub + * tree which the canonical string writes twice - a child which is referenced more than once - keeps + * its token in the text twice while its tables are read once, so its descriptors appear here once; + * a caller which compares this list with the tokens of a text has to expect that difference (the + * learned freshness comparison folds the tokens of a text per table for the same reason). Empty when + * the struct info is invalid. + */ + public List<HboScanDescriptor> getScans() { + return scans; + } + + /** The canonical string of one literal mode, with the data state of every scan. */ + public String getCanonical(LiteralMode mode) { + boolean noLiteral = mode == LiteralMode.NO_LITERAL; + String materialized = noLiteral ? canonicalNoLiteral : canonicalWithLiteral; + if (materialized != null) { + return materialized; + } + materialized = root == null ? "" : render(root, mode, false); + if (noLiteral) { + canonicalNoLiteral = materialized; + } else { + canonicalWithLiteral = materialized; + } + return materialized; + } + + /** + * The fingerprint input of one literal mode: the canonical string with the data state of every scan + * removed ({@code S{db.t}}). It is rendered from the same tree as the canonical string, with the scan + * tokens reduced to their identity, and never parsed out of the text, so no text inside a table name + * or a literal can be mistaken for a scan token. + */ + public String getShape(LiteralMode mode) { + boolean noLiteral = mode == LiteralMode.NO_LITERAL; + String materialized = noLiteral ? shapeNoLiteral : shapeWithLiteral; + if (materialized != null) { + return materialized; + } + materialized = root == null ? "" : render(root, mode, true); + if (noLiteral) { + shapeNoLiteral = materialized; + } else { + shapeWithLiteral = materialized; + } + return materialized; + } + + /** + * The sha256 of {@link #shapeKey(LiteralMode)}: the key an hbo entry is registered under. + * + * <p>The preimage is the structural form of this description, not the canonical string: every field + * is written with its length, so a table name, an alias or a literal which contains a character of + * the printed grammar cannot change which field it belongs to. The canonical string is the readable + * projection of the same tree ({@link #getCanonical}), and a pasted struct info is never hashed. + */ + public String getFingerprint(LiteralMode mode) { + boolean noLiteral = mode == LiteralMode.NO_LITERAL; + String materialized = noLiteral ? fingerprintNoLiteral : fingerprintWithLiteral; + if (materialized != null) { + return materialized; + } + materialized = sha256(shapeKey(mode)); + if (noLiteral) { + fingerprintNoLiteral = materialized; + } else { + fingerprintWithLiteral = materialized; + } + return materialized; + } + + private static String sha256(byte[] value) { + return Hashing.sha256().hashBytes(value).toString(); + } + + /** The structural identity of this description in one literal mode (see {@link #getFingerprint}). */ + private byte[] shapeKey(LiteralMode mode) { + ShapeKeyWriter out = new ShapeKeyWriter(); + if (root == null) { + // an invalid description has no tree: its key is a marker of its own + out.tag(-1); + return out.toByteArray(); + } + out.tag(0); + appendShapeKey(out, root, mode); + return out.toByteArray(); + } + + /** + * Write one node of the structural identity: the kind, the fields the canonical string prints for it + * (each as one length prefixed text) and the children. A field is never split by the text of another + * one, which the printed grammar cannot promise for a value which carries its characters. + */ + private static void appendShapeKey(ShapeKeyWriter out, Node node, LiteralMode mode) { + if (node.kind == Kind.PASSTHROUGH && node.children.size() == 1) { + // a node which only composes its child renders no text of its own, so a plan which keeps such + // a node and the same plan with it inlined are one description: the identity must not name + // the node, otherwise the two forms of one plan would be keyed apart + appendShapeKey(out, node.children.get(0).node, mode); + return; + } + out.tag(node.kind.ordinal()); + switch (node.kind) { + case SCAN: + case FILE_SCAN: + appendScanKey(out, node.scan); + return; + case STORAGE_AGGREGATE: + out.text(String.valueOf(node.storageAggregate.getAggOp())); + out.flag(node.scan != null); + if (node.scan != null) { + appendScanKey(out, node.scan); + } + return; + case FILTER: + appendConditionsKey(out, node.conditions, mode, false); + break; + case AGGREGATE: + appendConditionsKey(out, node.conditions, LiteralMode.WITH_LITERAL, false); + break; + case JOIN: + out.text(node.flattenable ? "inner" : String.valueOf(node.joinType)); + out.flag(node.markJoin); + appendConditionsKey(out, node.conditions, LiteralMode.WITH_LITERAL, true); + break; + case TOPN: { + StringBuilder header = new StringBuilder(); + appendTopNHeader(header, node, mode); + out.text(header.toString()); + break; + } + case LIMIT: + out.text(bound("L", node.limit, node.offset)); + break; + case WINDOW: { + StringBuilder header = new StringBuilder(); + appendWindowHeader(header, node); + out.text(header.toString()); + // the header is the display form; the expressions it names are written structurally as + // well, so a value inside them (an array literal, for instance) keeps its own boundaries + out.count(node.windowExpressions.size()); + for (NamedExpression namedExpression : node.windowExpressions) { + appendExpressionKey(out, namedExpression, LiteralMode.WITH_LITERAL); + } + break; + } + case GENERATE: { + StringBuilder header = new StringBuilder(); + appendGenerateHeader(header, node); + out.text(header.toString()); + out.count(node.generators.size()); + for (Expression generator : node.generators) { + appendExpressionKey(out, generator, LiteralMode.WITH_LITERAL); + } + out.count(node.generatorConjuncts.size()); + for (Expression conjunct : node.generatorConjuncts) { + appendExpressionKey(out, conjunct, LiteralMode.WITH_LITERAL); + } + break; + } + case REPEAT: { + StringBuilder header = new StringBuilder(); + appendRepeatHeader(header, node, mode); + out.text(header.toString()); + out.count(node.groupingSets.size()); + for (List<Expression> set : node.groupingSets) { + out.count(set.size()); + for (Expression key : set) { + appendExpressionKey(out, key, LiteralMode.WITH_LITERAL); + } + } + break; + } + case SET_OPERATION: + appendSetOperationKey(out, node); + break; + case PASSTHROUGH: + break; + default: + throw new IllegalStateException("unknown node kind " + node.kind); + } + out.count(node.children.size()); + for (Child child : node.children) { + // a child which is referenced more than once renders the text of the first reference, so the + // identity names the sub tree only, not how often the plan references it + out.count(child.mapping.size()); + for (Expression mapped : child.mapping) { + appendExpressionKey(out, mapped, mode); + } + appendShapeKey(out, child.node, mode); + } + } + + /** + * The identity of a scan, one field at a time: a table name which carries the characters of the + * printed token ({@code #ti:}, a comma) cannot stand for another field because each field has its own + * length here. + */ + private static void appendScanKey(ShapeKeyWriter out, HboScanDescriptor descriptor) { + out.text(descriptor.getTable()); + out.number(descriptor.getTableId()); + out.text(descriptor.getRelationIdentity()); + out.text(descriptor.getPartitionIdentity()); + out.text(descriptor.getTabletIdentity()); + out.text(descriptor.getIndexIdentity()); + } + + /** + * The key of one expression: its kind, the fields which define its value (a literal's type and + * value, a slot's qualifier) and its children, every one of them length prefixed. The display form + * of an expression is not used here: a value which carries the characters of that form cannot stand + * for a field of the identity. + */ + private static void appendExpressionKey(ShapeKeyWriter out, Expression expression, LiteralMode mode) { + // the identity suffix of the expression (its udf scope and id, the analyzer of a match, the dsl + // of a search, the arguments of a lambda, the distinct flag of an aggregate...) is part of the + // kind field, so it is not lost by the structural form + out.text(expressionName(expression)); + List<Expression> children = expression.children(); + out.count(children.size()); + if (isOrderInsensitive(expression)) { + // the renderer sorts the operands of a commutative expression, so the structural form has to + // be order independent as well: `t.a = t.b` and `t.b = t.a` keep their written order upstream + // (both operands are slots) and would otherwise hash differently while rendering one struct + List<byte[]> childKeys = new ArrayList<>(children.size()); + for (Expression child : children) { + ShapeKeyWriter childKey = new ShapeKeyWriter(); + appendExpressionKey(childKey, child, mode); + childKeys.add(childKey.toByteArray()); + } + childKeys.sort(UnsignedBytes.lexicographicalComparator()); + for (byte[] childKey : childKeys) { + out.bytes(childKey); + } + } else { + for (Expression child : children) { + appendExpressionKey(out, child, mode); + } + } + if (expression instanceof SlotReference) { + SlotReference slot = (SlotReference) expression; + out.text(String.join(".", slot.getQualifier())); + out.text(slot.getName()); + // a variant column read through a sub path (`v['age']`) is a different column + out.text(String.join("\u0000", slot.getSubPath())); + } else if (expression instanceof Literal) { + Literal literal = (Literal) expression; + out.text(String.valueOf(literal.getDataType())); Review Comment: [P2] Remove value-derived string lengths from NO_LITERAL keys Compare `Filter(Like(t.s,'%x_%')) -> Scan(t)` with the same tree using `'%long_%'`. `visitStringLiteral` produces `VARCHAR(4)` and `VARCHAR(7)` literals, and compatible string coercion preserves those lengths. Both printed NO_LITERAL structs contain `Like(col(...s),lit(*))`, but this unconditional type field makes their actual structural fingerprints differ. Consequently a NO_LITERAL pin for the first pattern silently misses the second despite identical scans; the advertised constant-agnostic string parameterization only works for patterns of the same length. Give literals a mode-aware key that removes value-derived length/type distinctions when folding them, and add actual-fingerprint assertions for different-length string parameters. ########## fe/fe-core/src/main/java/org/apache/doris/nereids/stats/GroupStructInfo.java: ########## @@ -0,0 +1,3300 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.nereids.memo.Group; +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.OrderKey; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.And; +import org.apache.doris.nereids.trees.expressions.ArrayItemReference; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Match; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.NullSafeEqual; +import org.apache.doris.nereids.trees.expressions.NullToNonNullFunction; +import org.apache.doris.nereids.trees.expressions.Or; +import org.apache.doris.nereids.trees.expressions.OrderExpression; +import org.apache.doris.nereids.trees.expressions.SearchExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.WindowExpression; +import org.apache.doris.nereids.trees.expressions.functions.Function; +import org.apache.doris.nereids.trees.expressions.functions.Udf; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda; +import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.trees.plans.AbstractPlan; +import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.Aggregate; +import org.apache.doris.nereids.trees.plans.algebra.Except; +import org.apache.doris.nereids.trees.plans.algebra.Filter; +import org.apache.doris.nereids.trees.plans.algebra.Generate; +import org.apache.doris.nereids.trees.plans.algebra.Intersect; +import org.apache.doris.nereids.trees.plans.algebra.Join; +import org.apache.doris.nereids.trees.plans.algebra.Limit; +import org.apache.doris.nereids.trees.plans.algebra.OlapScan; +import org.apache.doris.nereids.trees.plans.algebra.Project; +import org.apache.doris.nereids.trees.plans.algebra.Repeat; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation; +import org.apache.doris.nereids.trees.plans.algebra.Sort; +import org.apache.doris.nereids.trees.plans.algebra.TopN; +import org.apache.doris.nereids.trees.plans.algebra.Union; +import org.apache.doris.nereids.trees.plans.algebra.Window; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEAnchor; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEProducer; +import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; +import org.apache.doris.nereids.trees.plans.physical.PhysicalFileScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalGenerate; +import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; +import org.apache.doris.nereids.trees.plans.physical.PhysicalStorageLayerAggregate; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.util.MutableState; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.hash.Hashing; +import com.google.common.primitives.UnsignedBytes; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Simplified struct info of a memo {@link Group}, used by History Based Optimization (HBO). + * + * <p>The struct info is a property of the <b>group</b>, not of one of its group expressions: all + * expressions of a group are logically equivalent, so they describe the same set of relations with + * the same predicates, and the canonical form below is deliberately insensitive to which expression + * is picked. + * <ul> + * <li>join commutativity: children of a join are canonicalized by sorting;</li> + * <li>join associativity: a whole inner / cross join chain is emitted as one node + * ({@code J{inner,c:[...]}(...;...)}) whose children are the chain leaves sorted by their + * canonical form and whose conditions are the sorted union of every condition of the chain, + * so {@code (A join B) join C} and {@code A join (B join C)} produce the same descriptor;</li> + * <li>aggregation: only the group by keys and the child participate, because the output row count + * of an aggregation is the number of groups and cannot be changed by the aggregate functions + * or by the output expressions;</li> + * <li>project is transparent for structure matching.</li> + * </ul> + * This mirrors what the MV oriented {@code rules.exploration.mv.StructInfo} does (it identifies a + * sub tree by its relation set and by level independent - shuttled - predicates), with every MV + * specific field dropped: no hyper graph, no shuttle map, no equivalence class, no lineage. + * + * <p>What the canonical form keeps: + * <ul> + * <li>for each scan: the table qualifier and - as an annotation only - the data state of the scan + * (its visible version, the rows it reads and how many of the table partitions it selects, + * see {@link HboScanDescriptor});</li> + * <li>for filter / join / aggregate: the normalized predicates / join conditions / grouping keys, + * i.e. the structural pattern only (slot and expression ids are stripped by + * {@link #normalizeExpression}).</li> + * </ul> + * The string contains no per-query identifier, so it is reproducible across runs and queries for + * structurally identical sub trees, and its sha256 is used as the HBO cache key (fingerprint). + * + * <p>The sha256 is taken over the canonical string <b>without</b> the scan baselines + * ({@link #stripScanBaseline}): a hbo key identifies a plan pattern and the relation it reads, not a + * data state, so an entry whose table simply keeps growing stays matchable instead of silently never + * matching again. Whether an entry may still be applied is decided on the read side, by comparing + * the baseline of the entry with the data state of the current query (see + * {@link HboStructFreshness}). + * + * <p>The descriptor is derived from the group's logical expression and its child groups (memo level + * traversal, with a visited set to guard shared sub graphs such as CTE). Groups whose content is not + * supported (e.g. contains TVF / CTE consumer, or the head operator is not one of the supported + * kinds) are {@link #isValid() invalid} and callers must fall back to the legacy behavior. + * + * <p>This description is always built by the FE from the plan it is looking at, and every string it can + * produce is one of its renderings: {@link #getCanonical(LiteralMode)} is the text EXPLAIN prints and a + * user may paste into {@code HBO SET STATISTICS ... STRUCT='...'}, {@link #getShape(LiteralMode)} is the + * same text without the data state of the scans, and {@link #getFingerprint(LiteralMode)} is the sha256 + * of that shape. Nothing is ever parsed to produce a fingerprint - in particular the text a user pasted + * is only compared with the fingerprint next to it, never taken as its source. + */ +public class GroupStructInfo { + /** + * The default of {@code hbo_max_scans_per_group}, used when no session is available. 20 keeps + * every TPC-DS shape observed in this repository (the largest single sub tree is query64's + * 19 scan join group) while refusing trees which are pathologically wide. + */ + public static final int DEFAULT_MAX_SCANS_PER_GROUP = 20; + /** + * Snapshot value which marks a plan node that is deliberately excluded from the hbo + * fingerprint (its group is unsupported, e.g. a sort / topn). It is not a fingerprint and is + * never hashed: the publish path uses it to tell an expected gap from a lost snapshot. A + * concurrent map can not store a null marker, so an empty string is used. + */ + public static final String UNSUPPORTED_FINGERPRINT = ""; + + /** + * The prefix of the reason a node whose kind the description does not support is reported with, and + * the two halves of the reason a node with an unsupported number of children is reported with. They + * are constants so that the corpus can bind the frozen skip texts to the formula which produces them + * instead of restating the prefix. + */ + static final String UNSUPPORTED_REASON_PREFIX = "unsupported="; + static final String ARITY_REASON_PREFIX = "arity="; + static final String ARITY_REASON_MIDDLE = " of "; + + private static final List<HboScanDescriptor> INVALID_SCANS = Collections.emptyList(); + + /** + * Characters which structure the canonical form and therefore can not appear raw inside a + * literal value (see {@link #encodeLiteralValue}). + */ + private static final String RESERVED_LITERAL_CHARS = "\u0000\\%(){}[],;:|"; + + private static final char[] HEX_DIGITS = "0123456789ABCDEF".toCharArray(); + + /** + * How many groups this query described. A group is described once: the description is memoized on + * the group and composed by its parents, so this number only grows when a new group is reached. + */ + private static final AtomicInteger describedGroupCount = new AtomicInteger(); + + /** + * How many descriptions were walked while materializing a string. A description whose string is + * already materialized is copied instead of being walked again, so this number does not grow for + * the sub tree of an already materialized child. + */ + private static final AtomicInteger materializedGroupCount = new AtomicInteger(); + + /** Shared invalid instance: the group content is not supported by the simplified struct info. */ + public static final GroupStructInfo INVALID = new GroupStructInfo(false, "", null, INVALID_SCANS); + + private static final String SEP = ";"; + + /** + * The header letters whose arguments are part of the identity of their node (see + * {@link Kind#literalsAreIdentity()}): the fold of a pasted struct info copies those headers verbatim, + * because folding one of their literals would make the sha256 of the text differ from the fingerprint + * EXPLAIN printed. The set is derived from the kinds which render the text, so a kind whose literals + * stop being identity cannot be forgotten here. + */ + private static final String PROTECTED_HEADERS = protectedHeaders(); + + /** + * The markers of the headers whose condition collection is sorted per literal mode, i.e. the ones the + * fold of a pasted struct info has to re-sort after it folded their literals (see + * {@code recanonicalizeFoldedConditions}): a filter, an aggregation, a join (which also de-duplicates) + * and the grouping sets of a repeat. + */ + private static final String[] CONDITION_HEADER_MARKERS = {"F{", "A{gb:", "J{", "R{gs:["}; + + /** + * The functions whose value depends on the session time zone although neither their arguments nor + * their result are a date: an epoch value is read in that zone ({@code hour_from_unixtime(t.ts) = 0} + * keeps 1000 rows in one session and 1 row in another although its argument is the same integer). + * A function whose argument or result is a date or a datetime is covered by its type instead (see + * {@link #usesSessionTimeZone}). + */ + private static final List<String> EPOCH_TIME_ZONE_FUNCTIONS = Arrays.asList( + "hour_from_unixtime", "minute_from_unixtime", "second_from_unixtime", + "microsecond_from_unixtime", "unix_timestamp"); + + private static final String NO_LOOKUP_PATH_REASON = + "storage layer aggregate: the read side applies the row count of the aggregation above it"; + + private static final String PARTIAL_STAGE_REASON = + "local (partial) aggregation stage: the read side applies the row count of the final" + + " aggregation above it"; + + private static final Logger LOG = LogManager.getLogger(GroupStructInfo.class); + + /** + * Whether literal values participate in the canonical string / fingerprint. + * <ul> + * <li>{@link #WITH_LITERAL} keeps the literal value ({@code lit(10:int)}): the fingerprint is + * bound to the exact constants, used for filter roots where the constant changes the + * output row count;</li> + * <li>{@link #NO_LITERAL} replaces every literal value with {@code *} ({@code lit(*)}): the + * fingerprint is constant agnostic. It is the primary key of a scan, of a constant-free + * filter shape and of the key the publish path generates for a join / aggregate root; every + * root which is annotated also carries a literal carrying form.</li> + * </ul> + * The literal data type is intentionally not kept: nereids inserts an explicit {@code Cast} + * for mismatching operand types, so two semantically different literal forms already differ by + * the cast node, while equivalent forms (e.g. {@code d > 1} and {@code d > 1.0} on a decimal + * column) are allowed to collapse. + */ + public enum LiteralMode { + WITH_LITERAL, + NO_LITERAL + } + + /** + * Whether the description could be built. It is invalid - and the memo caches that answer, so every + * node above the sub tree gets no fingerprint for the whole query - when one of the following holds + * for the sub tree it covers: + * <ul> + * <li>it contains an operator the canonical form does not describe, which the annotation prints as + * {@code unsupported=<Operator>}: a table valued function relation ({@code numbers}, + * {@code s3}), an apply, a recursive union, a partition top-n, ...;</li> + * <li>it reads more scans than {@code hbo_max_scans_per_group}: + * {@code subtree scans=N > hbo_max_scans_per_group=M};</li> + * <li>a table name carries a struct info delimiter ({@code ,} or {@code }}), so the printed token + * could not be read back: {@code delimiter in table name <table>};</li> + * <li>the memo does not look like a canonical form can express it: a unary operator whose group + * expression has another arity, a CTE consumer which has no memo edge to its producer, a + * reused sub tree whose description is not available;</li> + * <li>the data state of a scan cannot be read (an external table whose metadata is not available, + * a cloud rpc failure): {@code no data state for <table> (<cause>)}.</li> + * </ul> + * A node whose group cannot be resolved at all is not described either, which the annotation reports + * as {@code no group for <Operator>}. + */ + private final boolean valid; + /** + * The tree of this description: the information of its own group and the descriptions of the groups + * below it, all as objects. Nothing is rendered while it is built - {@link #render} is the one place + * which knows the grammar - and every node remembers the forms it has rendered, so a parent which is + * rendered later composes the text of a child instead of walking its sub tree again. + */ + private final Node root; + /** The scans of the sub tree, in the order their tokens appear in the canonical string. */ + private final List<HboScanDescriptor> scans; + /** Why no struct info was built, when it was refused by a limit instead of being unsupported. */ + private final String skipReason; + // the forms materialized on first request: the read side only needs a fingerprint, while the + // explain annotation needs the canonical string, so building all of them up front would be wasted + private String canonicalWithLiteral; + private String canonicalNoLiteral; + private String shapeWithLiteral; + private String shapeNoLiteral; + private String fingerprintWithLiteral; + private String fingerprintNoLiteral; + + private GroupStructInfo(boolean valid, String skipReason, Node root, + List<HboScanDescriptor> scans) { + this.valid = valid; + this.skipReason = skipReason; + this.root = root; + this.scans = scans; + } + + private static GroupStructInfo ofNode(Node root) { + List<HboScanDescriptor> scans = new ArrayList<>(); + collectScans(root, scans); + return new GroupStructInfo(true, "", root, Collections.unmodifiableList(scans)); + } + + private static GroupStructInfo invalidStructInfo(String skipReason) { + return skipReason.isEmpty() + ? INVALID + : new GroupStructInfo(false, skipReason, null, INVALID_SCANS); + } + + private static String protectedHeaders() { + StringBuilder letters = new StringBuilder(); + for (Kind kind : Kind.values()) { + if (kind.literalsAreIdentity()) { + letters.append(kind.headerLetters()); + } + } + return letters.toString(); + } + + /** The reason the struct info was refused by a limit, or empty. Only for diagnostics. */ + public String getSkipReason() { + return skipReason; + } + + public boolean isValid() { + return valid; + } + + /** The node this description is made of; null when the description is invalid. */ + private Node root() { + return root; + } + + /** + * The row count and partition selection of every scan of the sub tree, in the order the scan tokens + * appear in the canonical string (i.e. the order a user sees when pasting the struct info). A sub + * tree which the canonical string writes twice - a child which is referenced more than once - keeps + * its token in the text twice while its tables are read once, so its descriptors appear here once; + * a caller which compares this list with the tokens of a text has to expect that difference (the + * learned freshness comparison folds the tokens of a text per table for the same reason). Empty when + * the struct info is invalid. + */ + public List<HboScanDescriptor> getScans() { + return scans; + } + + /** The canonical string of one literal mode, with the data state of every scan. */ + public String getCanonical(LiteralMode mode) { + boolean noLiteral = mode == LiteralMode.NO_LITERAL; + String materialized = noLiteral ? canonicalNoLiteral : canonicalWithLiteral; + if (materialized != null) { + return materialized; + } + materialized = root == null ? "" : render(root, mode, false); + if (noLiteral) { + canonicalNoLiteral = materialized; + } else { + canonicalWithLiteral = materialized; + } + return materialized; + } + + /** + * The fingerprint input of one literal mode: the canonical string with the data state of every scan + * removed ({@code S{db.t}}). It is rendered from the same tree as the canonical string, with the scan + * tokens reduced to their identity, and never parsed out of the text, so no text inside a table name + * or a literal can be mistaken for a scan token. + */ + public String getShape(LiteralMode mode) { + boolean noLiteral = mode == LiteralMode.NO_LITERAL; + String materialized = noLiteral ? shapeNoLiteral : shapeWithLiteral; + if (materialized != null) { + return materialized; + } + materialized = root == null ? "" : render(root, mode, true); + if (noLiteral) { + shapeNoLiteral = materialized; + } else { + shapeWithLiteral = materialized; + } + return materialized; + } + + /** + * The sha256 of {@link #shapeKey(LiteralMode)}: the key an hbo entry is registered under. + * + * <p>The preimage is the structural form of this description, not the canonical string: every field + * is written with its length, so a table name, an alias or a literal which contains a character of + * the printed grammar cannot change which field it belongs to. The canonical string is the readable + * projection of the same tree ({@link #getCanonical}), and a pasted struct info is never hashed. + */ + public String getFingerprint(LiteralMode mode) { + boolean noLiteral = mode == LiteralMode.NO_LITERAL; + String materialized = noLiteral ? fingerprintNoLiteral : fingerprintWithLiteral; + if (materialized != null) { + return materialized; + } + materialized = sha256(shapeKey(mode)); + if (noLiteral) { + fingerprintNoLiteral = materialized; + } else { + fingerprintWithLiteral = materialized; + } + return materialized; + } + + private static String sha256(byte[] value) { + return Hashing.sha256().hashBytes(value).toString(); + } + + /** The structural identity of this description in one literal mode (see {@link #getFingerprint}). */ + private byte[] shapeKey(LiteralMode mode) { + ShapeKeyWriter out = new ShapeKeyWriter(); + if (root == null) { + // an invalid description has no tree: its key is a marker of its own + out.tag(-1); + return out.toByteArray(); + } + out.tag(0); + appendShapeKey(out, root, mode); + return out.toByteArray(); + } + + /** + * Write one node of the structural identity: the kind, the fields the canonical string prints for it + * (each as one length prefixed text) and the children. A field is never split by the text of another + * one, which the printed grammar cannot promise for a value which carries its characters. + */ + private static void appendShapeKey(ShapeKeyWriter out, Node node, LiteralMode mode) { + if (node.kind == Kind.PASSTHROUGH && node.children.size() == 1) { + // a node which only composes its child renders no text of its own, so a plan which keeps such + // a node and the same plan with it inlined are one description: the identity must not name + // the node, otherwise the two forms of one plan would be keyed apart + appendShapeKey(out, node.children.get(0).node, mode); + return; + } + out.tag(node.kind.ordinal()); + switch (node.kind) { + case SCAN: + case FILE_SCAN: + appendScanKey(out, node.scan); + return; + case STORAGE_AGGREGATE: + out.text(String.valueOf(node.storageAggregate.getAggOp())); + out.flag(node.scan != null); + if (node.scan != null) { + appendScanKey(out, node.scan); + } + return; + case FILTER: + appendConditionsKey(out, node.conditions, mode, false); + break; + case AGGREGATE: + appendConditionsKey(out, node.conditions, LiteralMode.WITH_LITERAL, false); + break; + case JOIN: + out.text(node.flattenable ? "inner" : String.valueOf(node.joinType)); + out.flag(node.markJoin); + appendConditionsKey(out, node.conditions, LiteralMode.WITH_LITERAL, true); + break; + case TOPN: { + StringBuilder header = new StringBuilder(); + appendTopNHeader(header, node, mode); + out.text(header.toString()); + break; + } + case LIMIT: + out.text(bound("L", node.limit, node.offset)); + break; + case WINDOW: { + StringBuilder header = new StringBuilder(); + appendWindowHeader(header, node); + out.text(header.toString()); + // the header is the display form; the expressions it names are written structurally as + // well, so a value inside them (an array literal, for instance) keeps its own boundaries + out.count(node.windowExpressions.size()); + for (NamedExpression namedExpression : node.windowExpressions) { + appendExpressionKey(out, namedExpression, LiteralMode.WITH_LITERAL); + } + break; + } + case GENERATE: { + StringBuilder header = new StringBuilder(); + appendGenerateHeader(header, node); + out.text(header.toString()); + out.count(node.generators.size()); + for (Expression generator : node.generators) { + appendExpressionKey(out, generator, LiteralMode.WITH_LITERAL); + } + out.count(node.generatorConjuncts.size()); + for (Expression conjunct : node.generatorConjuncts) { + appendExpressionKey(out, conjunct, LiteralMode.WITH_LITERAL); + } + break; + } + case REPEAT: { + StringBuilder header = new StringBuilder(); + appendRepeatHeader(header, node, mode); + out.text(header.toString()); + out.count(node.groupingSets.size()); + for (List<Expression> set : node.groupingSets) { + out.count(set.size()); + for (Expression key : set) { + appendExpressionKey(out, key, LiteralMode.WITH_LITERAL); + } + } + break; + } + case SET_OPERATION: + appendSetOperationKey(out, node); + break; + case PASSTHROUGH: + break; + default: + throw new IllegalStateException("unknown node kind " + node.kind); + } + out.count(node.children.size()); + for (Child child : node.children) { + // a child which is referenced more than once renders the text of the first reference, so the + // identity names the sub tree only, not how often the plan references it + out.count(child.mapping.size()); + for (Expression mapped : child.mapping) { + appendExpressionKey(out, mapped, mode); + } + appendShapeKey(out, child.node, mode); + } + } + + /** + * The identity of a scan, one field at a time: a table name which carries the characters of the + * printed token ({@code #ti:}, a comma) cannot stand for another field because each field has its own + * length here. + */ + private static void appendScanKey(ShapeKeyWriter out, HboScanDescriptor descriptor) { + out.text(descriptor.getTable()); + out.number(descriptor.getTableId()); + out.text(descriptor.getRelationIdentity()); + out.text(descriptor.getPartitionIdentity()); + out.text(descriptor.getTabletIdentity()); + out.text(descriptor.getIndexIdentity()); + } + + /** + * The key of one expression: its kind, the fields which define its value (a literal's type and + * value, a slot's qualifier) and its children, every one of them length prefixed. The display form + * of an expression is not used here: a value which carries the characters of that form cannot stand + * for a field of the identity. + */ + private static void appendExpressionKey(ShapeKeyWriter out, Expression expression, LiteralMode mode) { + // the identity suffix of the expression (its udf scope and id, the analyzer of a match, the dsl + // of a search, the arguments of a lambda, the distinct flag of an aggregate...) is part of the + // kind field, so it is not lost by the structural form + out.text(expressionName(expression)); + List<Expression> children = expression.children(); + out.count(children.size()); + if (isOrderInsensitive(expression)) { + // the renderer sorts the operands of a commutative expression, so the structural form has to + // be order independent as well: `t.a = t.b` and `t.b = t.a` keep their written order upstream + // (both operands are slots) and would otherwise hash differently while rendering one struct + List<byte[]> childKeys = new ArrayList<>(children.size()); + for (Expression child : children) { + ShapeKeyWriter childKey = new ShapeKeyWriter(); + appendExpressionKey(childKey, child, mode); + childKeys.add(childKey.toByteArray()); + } + childKeys.sort(UnsignedBytes.lexicographicalComparator()); + for (byte[] childKey : childKeys) { + out.bytes(childKey); + } + } else { + for (Expression child : children) { + appendExpressionKey(out, child, mode); + } + } + if (expression instanceof SlotReference) { + SlotReference slot = (SlotReference) expression; + out.text(String.join(".", slot.getQualifier())); Review Comment: [P2] Encode qualifier components separately in the slot key With `enable_nested_namespace=true`, consider the supported reduced trees `Aggregate(gb=z.v) -> CROSS(FileScan(cat.[x.y].t AS z), FileScan(cat.x.u AS [y.z]))` and the same tree grouping by `[y.z].v` (brackets denote quoted identifiers). For `t.v={0,1}` and `u.v={0,0}`, they have two versus one groups. `LogicalSubQueryAlias.computeOutputInternal` gives those slots qualifiers `[cat,x.y,z]` and `[cat,x,y.z]`; alias-to-project preserves them. This join writes both as `cat.x.y.z`, with the same slot name/type, so their actual fingerprints coincide in both modes even though the displayed canonicals differ. The common child and relation summaries are identical, and unversioned file scans permit pins with unknown freshness, allowing the other aggregate's count to apply. Encode the qualifier count and each component separately. This is independent of the existing VARIANT-subpath thread: neither slot has a subpath. ########## fe/fe-core/src/main/java/org/apache/doris/statistics/hbo/InputTableStatisticsInfo.java: ########## @@ -17,25 +17,91 @@ package org.apache.doris.statistics.hbo; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; /** * Describes plan statistics which are derived from history based optimizer. */ public class InputTableStatisticsInfo { - private final Optional<String> hash; private final Optional<List<PlanStatistics>> inputTableStatistics; - public InputTableStatisticsInfo(Optional<String> hash, Optional<List<PlanStatistics>> inputTableStatistics) { - this.hash = Objects.requireNonNull(hash, "hash is null"); + public InputTableStatisticsInfo(Optional<List<PlanStatistics>> inputTableStatistics) { this.inputTableStatistics = Objects.requireNonNull(inputTableStatistics, "inputTableStatistics is null"); } - public Optional<String> getHash() { - return hash; + /** + * Fold the input table statistics of another node which shares the fingerprint of this one: two + * runtime nodes can carry one fingerprint (the two scans of a self join, for instance), and the + * provider stores one entry per fingerprint. + * + * <p>The two nodes read the same table population, so an entry of the same scan (the same table, the + * same index and the same partitions) is one population however many nodes read it: folding two of + * them keeps the larger measurement instead of adding the two up. Summing them would describe a table + * which holds the rows of both scans at once, and the drift comparison - which sums the rows of every + * recorded scan of a table - would then judge the next query of a single scan stale by construction. + * Two different scans are both kept, because they describe two populations. + */ + public InputTableStatisticsInfo merge(InputTableStatisticsInfo other) { + if (inputTableStatistics.isEmpty()) { + return other; + } + if (other.inputTableStatistics.isEmpty()) { + return this; + } + Map<String, PlanStatistics> merged = new LinkedHashMap<>(); + for (PlanStatistics statistics : inputTableStatistics.get()) { + merged.merge(identityOf(statistics), statistics, InputTableStatisticsInfo::largerRows); Review Comment: [P2] Preserve within-node scan multiplicity when merging feedback Use a 1000-row DUP table and `UNION ALL(Join(Scan(t a),Scan(t b),a.k=b.k), Join(Scan(t a),Scan(t b),a.k=b.k))`, with identical local aliases and runtime filters disabled. The two hash joins have different runtime IDs but the same fingerprint, so `genPlanStatisticsMap` merges their inputs here. Each input list correctly contains two 1000-row occurrences; `identityOf` omits the occurrence, and this loop collapses even the first list to one before adding the second. The published entry records 1,000 rows, while the live join descriptor contains both aliases and freshness sums 2,000, rejecting feedback from the next unchanged query as 100% growth. Ordinary matching still passes because it clones the historical one-input list. Preserve the multiplicity within each node when aligning repeated measurements, or retain one complete matching measurement. This differs from the existing output/input-pairing thread: both branches have identical predicates, equal outputs and accurate full-scan measurements; the loss is the two occurrences within either one of them. ########## fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboStructFreshness.java: ########## @@ -0,0 +1,682 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.Config; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; +import org.apache.doris.rpc.RpcException; +import org.apache.doris.statistics.hbo.PlanStatistics; +import org.apache.doris.statistics.hbo.ScanPlanStatistics; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * Whether a hbo entry may still be applied: the recorded data state of an entry (the baseline of + * every scan of its struct info, see {@link HboScanDescriptor}) compared with the data state of the + * query which wants to use it. + * + * <p>A hbo fingerprint does not contain any data state any more, so an entry keeps matching while a + * table grows. The read side therefore has to decide whether the injected row count still describes + * the data it was measured on: + * <ul> + * <li>{@link #STATE_LIVE}: the data the entry was measured on is still the current one, the entry + * is applied;</li> + * <li>{@link #STATE_DRIFTED}: the data moved, but by at most + * {@code Config.hbo_row_count_change_ratio} of the recorded row count (10% by default): the + * entry is applied, which is the point of the tolerance - a table which merely keeps growing + * (e.g. a few new rows in a partition) does not invalidate a measured row count;</li> + * <li>{@link #STATE_STALE}: the data moved further than that: the entry is <b>not</b> applied, and + * the query falls back to the optimizer estimation;</li> + * <li>{@link #STATE_UNKNOWN}: nothing can be compared (the entry records no row count, or the + * current one cannot be read / was obtained a different way), so there is nothing to judge: the + * entry is applied as it always was.</li> + * </ul> + * The recorded row count is never scaled: an entry either applies with the value the user measured + * or does not apply at all ("soft strategy" - a user who needs the exact recorded state in any case + * should not inject hbo statistics for that query). Setting + * {@code hbo_row_count_change_ratio} to 0 or less switches the verdict back to the strict semantics + * hbo had before the baseline existed: the recorded visible version has to still be the current one. + * + * <p>The rows are compared <b>before</b> the version, and only against a number which was obtained + * the same way ({@link HboScanDescriptor.RowsKind}): the visible version is a property of the table, + * so two different partition selections of one table share it while reading completely different + * amounts of data, and a measured row count must not be compared with one that was derived from the + * table average. Whenever nothing can be compared - a hand written struct literal, an entry whose + * rows were never measured, a current row count which cannot be read, or two numbers of different + * kinds - the state is {@link #STATE_UNKNOWN} and the entry is applied: a transient failure to read + * the catalog must not drop an entry the user injected. The version is only decisive in strict mode + * ({@code hbo_row_count_change_ratio <= 0}). + * + * <p>A scan is judged by the rows of its <b>selected</b> partitions, so pruning decides which data + * the entry depends on: an unrelated partition which grows does not invalidate a pruned entry. + * {@code HBO SHOW STATISTICS} and {@code HBO DELETE STALE STATISTICS} cannot run the query of an + * entry, so there the recorded state is compared with the table as a whole; an entry whose struct + * prunes partitions can therefore only be judged by rows when it was recorded as a whole table scan + * (its state is reported as unknown otherwise, and such an entry is kept unless the user asks for + * {@code OLDER_THAN}). + * + * <p>Learned (profile collected) entries carry the input table statistics of the run they were + * measured in, so they are judged by the same tolerance over those recorded rows; an entry injected + * by {@code HBO SET LEARNED STATISTICS} carries no such statistics and is applied as it always was. + */ +public class HboStructFreshness { + /** The recorded data state is still the current one. */ + public static final String STATE_LIVE = "live"; + /** The data moved, but within the tolerance of {@code Config.hbo_row_count_change_ratio}. */ + public static final String STATE_DRIFTED = "drifted"; + /** The data moved further than the tolerance: the entry must not be applied. */ + public static final String STATE_STALE = "stale"; + /** The entry records no data state (or it cannot be read any more): nothing to judge. */ + public static final String STATE_UNKNOWN = "unknown"; + + private static final Logger LOG = LogManager.getLogger(HboStructFreshness.class); + + private final String state; + // the recorded baseline and the current state, both rendered for display + private final String recorded; + private final String live; + // the row counts of the scan which decided the state, {@link HboScanDescriptor#UNKNOWN} when the + // state was decided by something else (e.g. a version which is not readable) + private final long recordedRows; + private final long liveRows; + + private HboStructFreshness(String state, String recorded, String live, long recordedRows, long liveRows) { + this.state = state; + this.recorded = recorded; + this.live = live; + this.recordedRows = recordedRows; + this.liveRows = liveRows; + } + + public String getState() { + return state; + } + + public boolean isStale() { + return STATE_STALE.equals(state); + } + + public boolean isUnknown() { + return STATE_UNKNOWN.equals(state); + } + + public boolean isDrifted() { + return STATE_DRIFTED.equals(state); + } + + /** The data state the entry was recorded in, e.g. {@code hbo_test.t1:v3,r1000,p1/5}. */ + public String getRecorded() { + return recorded; + } + + /** The data state the entry is compared with (empty when it cannot be read). */ + public String getLive() { + return live; + } + + /** + * The verdict as one token, printed after {@code used=} / {@code skipped=} in the explain + * annotation, e.g. {@code drifted(rows=1030,rec=1000,+3.0%)}. + */ + public String getSummary() { + if (!STATE_DRIFTED.equals(state) && !STATE_STALE.equals(state)) { + return state; + } + if (recordedRows == HboScanDescriptor.UNKNOWN || liveRows == HboScanDescriptor.UNKNOWN) { + return state; + } + return state + "(rows=" + liveRows + ",rec=" + recordedRows + "," + + changeText(recordedRows, liveRows) + ")"; + } + + /** The {@code now} part of {@code HBO SHOW STATISTICS}: the state read from the catalog. */ + public String getLiveDetail() { + if (live.isEmpty()) { + return "-"; + } + if (recordedRows == HboScanDescriptor.UNKNOWN || liveRows == HboScanDescriptor.UNKNOWN) { + return live; + } + return live + "," + changeText(recordedRows, liveRows); + } + + /** + * Whether the entry records any data state at all. An entry which does not (a struct literal + * typed by hand, or one copied from a plan whose tables were never reported) is always applied, + * so a caller does not have to compute the state of the current query to find that out. + */ + public static boolean hasRecordedDataState(String recordedCanonical) { + for (HboScanDescriptor scan : HboScanDescriptor.parseAll(recordedCanonical)) { + if (scan.hasVisibleVersion() || scan.hasScanRows()) { + return true; + } + } + return false; + } + + /** + * Judge an entry against the current query: the data state recorded in {@code recordedCanonical} + * (the struct info of the entry) against the state of {@code liveCanonical} (the struct info the + * read side just computed for the group). This is the decision of the read side. + */ + public static HboStructFreshness between(String recordedCanonical, GroupStructInfo liveStructInfo) { + return between(recordedCanonical, liveStructInfo.getScans()); + } + + /** The same comparison for a caller which has the scans of the current query only. */ + static HboStructFreshness between(String recordedCanonical, List<HboScanDescriptor> liveScans) { + List<HboScanDescriptor> recordedScans = HboScanDescriptor.parseAll(recordedCanonical); + String liveText = renderScans(liveScans); + if (recordedScans.isEmpty()) { + // an entry without a baseline (a hand written struct literal): nothing to judge + return new HboStructFreshness(STATE_UNKNOWN, "", liveText, + HboScanDescriptor.UNKNOWN, HboScanDescriptor.UNKNOWN); + } + if (recordedScans.size() != liveScans.size()) { + // the entry describes a different number of scans than the group: cannot be compared + return new HboStructFreshness(STATE_UNKNOWN, renderScans(recordedScans), liveText, + HboScanDescriptor.UNKNOWN, HboScanDescriptor.UNKNOWN); + } + String state = STATE_LIVE; + String recordedText = renderScans(recordedScans); + + long recordedRows = HboScanDescriptor.UNKNOWN; + long liveRows = HboScanDescriptor.UNKNOWN; + for (int i = 0; i < recordedScans.size(); i++) { + HboScanDescriptor recorded = recordedScans.get(i); + HboScanDescriptor live = liveScans.get(i); + String scanState; + if (!recorded.getTable().equals(live.getTable())) { + // the same fingerprint describes the same scans in the same order, so this cannot + // happen for an entry of this group: refuse to judge rather than guess + scanState = STATE_UNKNOWN; + } else { + scanState = stateOf(recorded, live); + } + if (worse(scanState, state).equals(scanState)) { + state = scanState; + recordedRows = recorded.getScanRows(); + liveRows = live.getScanRows(); + } + } + return new HboStructFreshness(state, recordedText, liveText, recordedRows, liveRows); + } + + /** + * Judge an entry against the catalog as a whole (used by {@code HBO SHOW STATISTICS} and + * {@code HBO DELETE STALE STATISTICS}, which cannot run the query of an entry): the recorded + * visible version and row count of every table against the current ones. + * + * <p>Row counts are only comparable when the scan read the whole table (no partition was pruned + * away), because only then the recorded rows are the rows of the table. + */ + public static HboStructFreshness of(String canonicalStructInfo) { + List<HboScanDescriptor> recordedScans = HboScanDescriptor.parseAll(canonicalStructInfo); + if (recordedScans.isEmpty()) { + return new HboStructFreshness(STATE_UNKNOWN, "", "", HboScanDescriptor.UNKNOWN, + HboScanDescriptor.UNKNOWN); + } + String state = STATE_LIVE; + // the current state of every table, de duplicated like the recorded side (a self join reads + // the same table twice, which would only repeat the same numbers) + Set<String> liveTexts = new LinkedHashSet<>(); + long recordedRows = HboScanDescriptor.UNKNOWN; + long liveRows = HboScanDescriptor.UNKNOWN; + for (HboScanDescriptor recorded : recordedScans) { + OlapTable table = resolveTable(recorded.getTable()); + if (table != null && tableReplaced(recorded, table)) { + // the name now resolves to another table (the recorded one was dropped and created again): + // the entry describes data which is gone, and no plan of the new table can produce its + // key, so judging that key by the replacement's state would keep a dead row as LIVE + liveTexts.add(recorded.getTable() + ":replaced#" + table.getId()); + if (worse(STATE_STALE, state).equals(STATE_STALE)) { + state = STATE_STALE; + recordedRows = HboScanDescriptor.UNKNOWN; + liveRows = HboScanDescriptor.UNKNOWN; + } + continue; + } + if (table == null) { + if (worse(STATE_UNKNOWN, state).equals(STATE_UNKNOWN)) { + state = STATE_UNKNOWN; + } + liveTexts.add(recorded.getTable() + ":-"); + continue; + } + if (table.getIndexIdList().size() > 1 && !recorded.isPartitionSelectionComplete()) { + // the table has more than one index (a rollup) and the entry records only a partition + // count, so this statement cannot tell which index its row count came from: judging it + // against the base index could be wrong in both directions, so it is left to a query + liveTexts.add(recorded.getTable() + ":-"); + if (worse(STATE_UNKNOWN, state).equals(STATE_UNKNOWN)) { + state = STATE_UNKNOWN; + } + continue; + } + long currentVersion = visibleVersionOf(table); + // the rows of the whole table, computed the same way the recorded rows of a whole table + // scan were computed, so the two numbers describe the same thing - and read from the index + // the entry read: a whole table query which the optimizer bound to an aggregate rollup + // recorded that rollup's rows, and comparing them with the base index would report an + // unchanged entry as stale (DELETE STALE would remove an entry a query can still apply) + HboScanDescriptor.Rows liveRowsOfTable = HboScanDescriptor.rowsOf(table, + new ArrayList<>(table.getPartitionIds()), indexIdOf(recorded, table)); + long currentRows = liveRowsOfTable.getRows(); + HboScanDescriptor.RowsKind currentRowsKind = liveRowsOfTable.getKind(); + liveTexts.add(recorded.getTable() + ":v" + currentVersion + + (currentRows == HboScanDescriptor.UNKNOWN ? "" + : (currentRowsKind == HboScanDescriptor.RowsKind.MEASURED ? ",r" : ",e") + + currentRows)); + String scanState; + if (!recorded.hasVisibleVersion() && !recorded.hasScanRows()) { + // nothing was recorded for this scan: it applies as it always was, nothing to judge + scanState = STATE_UNKNOWN; + currentRows = HboScanDescriptor.UNKNOWN; + } else if (threshold() <= 0) { + scanState = strictStateOf(recorded.hasVisibleVersion(), recorded.getVisibleVersion(), + currentVersion); + currentRows = HboScanDescriptor.UNKNOWN; + } else if (recorded.hasVisibleVersion() && currentVersion == recorded.getVisibleVersion() + && (currentRows == HboScanDescriptor.UNKNOWN + || currentRowsKind != recorded.getRowsKind() + || !recorded.hasScanRows() + || !recorded.isPartitionSelectionComplete())) { + // the visible version is a property of the whole table, so an unchanged version means + // that nothing was loaded into any of its partitions: the recorded state is still the + // current one, whatever the entry was measured on. (The read side can not use this + // shortcut: it knows which partitions the entry was measured on and which ones the + // query reads, and those can differ under one unchanged version.) + // Only a recorded scan which covers the whole table is compared below: its rows + // describe the same thing as the live rows of the table, and they can move under one + // unchanged version (an unreported partition uses TableStatsMeta, which ALTER COLUMN + // STATS ... ROW_COUNT updates), which the read side reports as stale as well. The + // rows of a pruned scan are not the rows of the table, so they are never compared. + scanState = STATE_LIVE; + } else if (!recorded.hasScanRows() || !recorded.isPartitionSelectionComplete()) { + // the entry records no row count at all (a struct literal of an older format, for + // example), or it records the rows of pruned partitions only, which are not the rows + // of the table: the state can only be judged by a query of that entry + scanState = STATE_UNKNOWN; + currentRows = HboScanDescriptor.UNKNOWN; + } else if (currentRows == HboScanDescriptor.UNKNOWN + || currentRowsKind != recorded.getRowsKind()) { + // the current row count cannot be read or was obtained differently than the recorded + // one, so the two are not comparable: the read side reports unknown for exactly this + // case and keeps applying the entry, and this scope must not disagree by calling it + // stale (DELETE STALE would then remove an entry which queries still use) + scanState = STATE_UNKNOWN; + currentRows = HboScanDescriptor.UNKNOWN; + } else if (recorded.getScanRows() == currentRows) { + scanState = STATE_LIVE; + } else { + scanState = changeRatio(recorded.getScanRows(), currentRows) <= threshold() + ? STATE_DRIFTED : STATE_STALE; + } + if (worse(scanState, state).equals(scanState) && scanState != STATE_LIVE) { + state = scanState; + recordedRows = scanState == STATE_UNKNOWN + ? HboScanDescriptor.UNKNOWN : recorded.getScanRows(); + liveRows = scanState == STATE_UNKNOWN ? HboScanDescriptor.UNKNOWN : currentRows; + } + } + return new HboStructFreshness(state, renderScans(recordedScans), String.join("; ", liveTexts), + recordedRows, liveRows); + } + + /** + * Whether the table a recorded scan names was dropped and created again under the same name: the token + * carries the id of the table it read, so a different id means the entry describes data which no + * longer exists. An entry of the older format carries none and cannot be judged. + */ + static boolean tableReplaced(HboScanDescriptor recorded, OlapTable table) { + return recorded.getTableId() != HboScanDescriptor.UNKNOWN + && recorded.getTableId() != table.getId(); + } + + /** + * The index a recorded scan of a whole table read: the one its token names, or the base index of the + * table when the token names none. A scan which reads a rollup records that index in its token, so + * the rows of the entry are compared with the rows of the index it was measured on. An index id + * which cannot be read (a struct info a user pasted with a hand written marker) leaves the catalog + * verdict unknown instead of failing the statement. + */ + static long indexIdOf(HboScanDescriptor recorded, OlapTable table) { + if (recorded.getIndexIdentity().isEmpty()) { + return table.getBaseIndexId(); + } + try { + return Long.parseLong(recorded.getIndexIdentity()); + } catch (NumberFormatException e) { + return HboScanDescriptor.UNKNOWN; + } + } + + /** + * Judge a learned (profile collected) entry: the rows its input tables had when it was measured + * against the rows of the scans of the current query. The publish path records one scan statistics + * per scan which feeds the published node, so this holds for an entry of any kind of node (see + * {@code HboStatsCalculator.applyLearnedStats}). The input tables are aggregated by table name, because + * a learned entry describes the scans of the node it was published for in plan order while the + * struct info orders them canonically. + * + * <p>This is a one way guard: it only rejects an entry when a comparable pair of row counts + * proves that the data moved beyond the tolerance. Everything which cannot be compared (an + * injected entry without input statistics, a different number of tables, a row count of a + * different kind) leaves the entry applicable, so a learned lookup which worked before this + * check keeps working. + */ + public static HboStructFreshness ofLearnedEntry(List<PlanStatistics> recordedInputStatistics, + GroupStructInfo liveStructInfo) { + return ofLearnedEntry(recordedInputStatistics, liveStructInfo.getScans()); + } + + /** The same guard for a caller which has the scans of the current query only. */ + static HboStructFreshness ofLearnedEntry(List<PlanStatistics> recordedInputStatistics, + List<HboScanDescriptor> liveScans) { + Map<String, Long> recordedRowsByTable = new LinkedHashMap<>(); + if (recordedInputStatistics != null) { + for (PlanStatistics input : recordedInputStatistics) { + if (!(input instanceof ScanPlanStatistics)) { + continue; + } + PhysicalOlapScan scan = ((ScanPlanStatistics) input).getScan(); + if (scan == null || scan.getTable() == null) { + continue; + } + String table = HboScanDescriptor.qualifiedNameOf(scan.getTable()); + // the input rows of a scan node are the rows it read, i.e. the number a catalog row + // count (which also ignores the predicates) can be compared with + recordedRowsByTable.merge(table, input.getInputRows(), Long::sum); Review Comment: [P2] Check distinct selected scan populations before summing For `INNER(a.k=b.k, Scan(t PARTITION(p1) AS a), Scan(t PARTITION(p2) AS b))`, use a DUP table with every k=0 and runtime filters disabled. Learn the join with p1=10 rows and p2=10000 rows, then grow only p1 to 1,000 and wait for measured partition reports. The same aliases/index/partition IDs keep the fingerprint and FULL_MATCH unchanged. This merge and `ofLearnedRows` compare only table totals, 10010 versus11000 (+9.89%), so the default 10% tolerance accepts the old 100,000-row output although the join now has 10,000,000 rows and one selected input grew 100x. Each runtime count accurately measures its full selected population; no missing/zero or scan-counter discrepancy is needed. This loses the per-selected-scan tolerance described by this class and used by pinned freshness. Align by table/index/partition selection and check each distinct population before aggregating; unrelated stable scans should not dilute its drift. This requires no repeated-key merge and is distinct from th e existing population-counter and output/input-pairing threads. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
