englefly commented on code in PR #69322: URL: https://github.com/apache/doris/pull/69322#discussion_r4357915481
########## fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboScanDescriptor.java: ########## @@ -0,0 +1,625 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.stats; + +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.nereids.trees.plans.algebra.OlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; +import org.apache.doris.rpc.RpcException; +import org.apache.doris.statistics.analysis.TableStatsMeta; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * The data state of one scan of a hbo struct info: the table it reads, the table visible version, + * the number of rows the scan reads (the sum of the row counts of its selected partitions) and which + * part of the table partitions it selects. + * + * <p>These numbers are the <b>baseline</b> a hbo entry is judged against. The scan token of a + * canonical struct info carries them as an annotation + * ({@code S{db.t,v3,r1000,p1/5}}), so a user copies the state the entry was measured in together + * with the struct info, and the read side compares them with the state of the current query to + * decide whether the injected row count may still be applied (see {@link HboStructFreshness}). + * + * <p>None of these numbers takes part in the fingerprint: + * {@link GroupStructInfo#stripScanBaseline} removes the whole baseline of every scan token before + * the sha256 is taken, so a table which simply keeps growing neither changes the key of an entry nor + * makes it unmatchable; whether the entry may still be applied is a read side decision. + * + * <p>A number which could not be determined is omitted from the token (and parsed back as + * {@link #UNKNOWN}), e.g. the row count of a table whose row count was never reported by any + * backend: such an entry cannot be compared by rows any more and falls back to the version check. + */ +public class HboScanDescriptor { + /** A number which could not be determined; a row count of 0 is a known, meaningful value. */ + public static final long UNKNOWN = -1; + + /** + * Separates the table from the stable ids of the partitions a scan selects inside a scan token. It + * must not contain a comma: the text based reader of a canonical form takes everything before the + * first comma inside {@code S{...}} as the identity of the scan (see + * {@link GroupStructInfo#stripScanBaseline}), so a comma here would move the ids into the baseline + * and the fingerprint of a pasted struct info would no longer match the one of the plan. + */ + static final String PARTITION_IDENTITY_SEPARATOR = ":"; + private static final String PARTITION_IDENTITY_MARK = "#pt"; + private static final String TABLET_IDENTITY_MARK = "#tb"; + private static final String INDEX_IDENTITY_MARK = "#ix"; + private static final String TABLE_IDENTITY_MARK = "#ti"; + private static final String RELATION_IDENTITY_MARK = "#ra"; + + /** + * The qualified name of the table, which is not its identity: {@code DROP TABLE db.t} and the + * creation of another {@code db.t} with the same row count leaves the name, the partitions and the + * version unchanged while the data is unrelated, and an entry pinned for the first table would still + * match. The id of the table is stable for the lifetime of the table (a rename keeps it) and differs + * for every other table, so it is part of the identity. It is {@link #UNKNOWN} for a relation whose + * id is not stable (an external table) or not known. + */ + private final long tableId; + private final String table; + /** + * The alias the query gave this relation ({@code FROM t AS x}), empty when it has none. Two scans of + * one table read the same data, but they are two relation instances: a join predicate which names + * their aliases ({@code x.b < y.b}) is bound to the instances, and the leaves are sorted by their + * text - without the alias the two plans which swap the aliases render the same key although the + * ordered comparison selects a different number of rows. + */ + private final String relationIdentity; + private final long visibleVersion; + private final long scanRows; + private final RowsKind rowsKind; + private final long selectedPartitions; + private final long totalPartitions; + /** + * The partitions this scan selects, as their stable ids in ascending order (empty when it selects + * every partition of the table). The partition *count* alone is not an identity: {@code PARTITION(p1)} + * and {@code PARTITION(p2)} of a table with equally sized partitions produce the same count, the same + * row count and therefore the same key, while they read different data - the ids are what tells them + * apart, and they are stable across renames (a rename keeps the id, so a pinned entry keeps applying). + */ + private final String partitionIdentity; + /** + * The tablets this scan selects explicitly ({@code TABLET(id1, id2)}), ascending, empty when the + * scan reads whole partitions. Like the partition ids they are part of the identity: two tablet + * selections of one partition read different data while they report the same partition count and + * the same row count. + */ + private final String tabletIdentity; + /** + * The index this scan reads when it is not the base index of the table (a rollup), empty otherwise. + * It belongs to the identity as well: an aggregate rollup holds a different number of rows than the + * base index, so an entry recorded for one of them must never be compared with the other - a whole + * table query and a rollup query would otherwise share an entry, and the drift guard would judge the + * rollup's row count against the base index and report the entry as stale. + */ + private final String indexIdentity; + + HboScanDescriptor(String table, long tableId, String relationIdentity, long visibleVersion, + long scanRows, RowsKind rowsKind, long selectedPartitions, long totalPartitions, + String partitionIdentity, String tabletIdentity, String indexIdentity) { + this.table = table; + this.tableId = tableId; + this.relationIdentity = relationIdentity; + this.visibleVersion = visibleVersion; + this.scanRows = scanRows; + this.rowsKind = rowsKind; + this.selectedPartitions = selectedPartitions; + this.totalPartitions = totalPartitions; + this.partitionIdentity = partitionIdentity; + this.tabletIdentity = tabletIdentity; + this.indexIdentity = indexIdentity; + } + + /** + * The name of a table as it is recorded in a scan token. {@code getNameWithFullQualifiers} joins the + * qualifiers and the name with dots, so the table {@code a.b} of the database {@code db} and the + * table {@code b} of the database {@code db.a} serialize identically - and a resolver which takes + * the last dot separated part as the table name would judge the sibling of the recorded table. The + * dots *inside the name* are therefore escaped while the dots which separate the qualifiers are not; + * an ordinary name is unchanged, so no key of an ordinary table moves. + */ + public static String qualifiedNameOf(TableIf table) { + DatabaseIf<?> database = table.getDatabase(); + if (database == null) { + // some kinds of table (a function generated one, for instance) belong to no database + return "null.null." + escapeNamePart(table.getName()); + } + return database.getCatalog().getName() + "." + database.getFullName() + "." + + escapeNamePart(table.getName()); + } + + /** The dots of a table name (and the escape character itself) are escaped. */ + static String escapeNamePart(String name) { + // '#' introduces the partition / tablet / index selector markers of a scan token, so a table name + // which carries one would be truncated to the part before it while the planner summary keeps the + // whole name - the relation key of the entry would then never match the query + return name.replace("%", "%25").replace(".", "%2E").replace("#", "%23") + .replace("{", "%7B").replace("}", "%7D"); + } + + /** The inverse of {@link #escapeNamePart}, for a name read back from a scan token. */ + public static String unescapeNamePart(String name) { + return name.replace("%23", "#").replace("%2E", ".").replace("%7B", "{").replace("%7D", "}") + .replace("%25", "%"); + } + + /** The id of the index a scan reads, empty when it reads the base index of its table. */ + static String indexIdentityOf(OlapScan scan) { + long selectedIndexId = scan.getSelectedIndexId(); + return selectedIndexId == scan.getTable().getBaseIndexId() ? "" : String.valueOf(selectedIndexId); + } + + /** The identity of a relation from the alias a query gave it (empty when it has none). */ + static String relationIdentityOf(String tableAlias) { + return escapeNamePart(tableAlias == null ? "" : tableAlias); + } + + /** + * The alias the query gave the relation this scan reads, empty when it has none. The alias is what a + * join predicate names when it is bound to one relation instance ({@code t AS x}), and the leaves of a + * join chain are sorted by their own identity: two plans which swap the aliases of two tables render + * the same predicates, so the binding has to be part of the leaf identity. + */ + static String relationIdentityOf(OlapScan scan) { + // both forms of a scan carry the alias: a group which a rule filled with physical expressions + // only is described through its physical scan, and its key must name the same relation as the + // logical description of the same scan + if (scan instanceof LogicalOlapScan) { + return escapeNamePart(((LogicalOlapScan) scan).getTableAlias()); + } + if (scan instanceof PhysicalOlapScan) { + return escapeNamePart(((PhysicalOlapScan) scan).getTableAlias()); + } + return ""; + } + + /** The stable ids of the explicitly selected tablets, ascending, empty when none were selected. */ + static String tabletIdentityOf(List<Long> selectedTabletIds) { + if (selectedTabletIds.isEmpty()) { + return ""; + } + List<Long> sorted = new ArrayList<>(selectedTabletIds); + sorted.sort(null); + StringBuilder sb = new StringBuilder(); + for (Long id : sorted) { + if (sb.length() > 0) { + sb.append(PARTITION_IDENTITY_SEPARATOR); + } + sb.append(id); + } + return sb.toString(); + } + + /** + * The stable ids of the selected partitions, ascending, or empty when every partition is read (then + * the scan is the whole table and its identity does not have to name a selection). + */ + static String partitionIdentityOf(List<Long> selectedPartitionIds, long totalPartitions) { + if (selectedPartitionIds.size() == totalPartitions) { Review Comment: temporary partition 上查询是非常少见的操作. 不收录hbo信息 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
