This is an automated email from the ASF dual-hosted git repository.
englefly pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 203d25d4352 [feat](nereids) disable join reorder if any table row
count is not available #40349 (2.0) (#41425)
203d25d4352 is described below
commit 203d25d435245b70f746999419a6892f2bf758ec
Author: minghong <[email protected]>
AuthorDate: Sun Sep 29 10:45:50 2024 +0800
[feat](nereids) disable join reorder if any table row count is not
available #40349 (2.0) (#41425)
## Proposed changes
pick #40349
Issue Number: close #xxx
<!--Describe your changes.-->
---
.../java/org/apache/doris/catalog/OlapTable.java | 3 ++
.../org/apache/doris/nereids/NereidsPlanner.java | 13 ++++++
.../doris/nereids/stats/StatsCalculator.java | 48 ++++++++++++++++++++++
3 files changed, 64 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index ffff643a861..c3f8600b214 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -1277,6 +1277,9 @@ public class OlapTable extends Table {
return getRowCountForIndex(baseIndexId, false);
}
+ /**
+ * @return -1 if there are some tablets whose row count is not reported to
FE
+ */
public long getRowCountForIndex(long indexId, boolean strict) {
long rowCount = 0;
for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index c17d3fa545b..150780d95e1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -23,6 +23,7 @@ import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
+import org.apache.doris.common.FeConstants;
import org.apache.doris.common.NereidsException;
import org.apache.doris.common.Pair;
import org.apache.doris.common.util.DebugUtil;
@@ -43,12 +44,14 @@ import org.apache.doris.nereids.minidump.NereidsTracer;
import org.apache.doris.nereids.processor.post.PlanPostProcessors;
import org.apache.doris.nereids.processor.pre.PlanPreprocessors;
import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.stats.StatsCalculator;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.plans.AbstractPlan;
import org.apache.doris.nereids.trees.plans.Plan;
import
org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
@@ -258,6 +261,16 @@ public class NereidsPlanner extends Planner {
}
}
+ // if we cannot get table row count, skip join reorder
+ // except:
+ // 1. user set leading hint
+ // 2. ut test. In ut test, FeConstants.enableInternalSchemaDb is
false or FeConstants.runningUnitTest is true
+ if (FeConstants.enableInternalSchemaDb && !FeConstants.runningUnitTest
&& cascadesContext.isLeadingJoin()) {
+ List<LogicalOlapScan> scans = cascadesContext.getRewritePlan()
+ .collectToList(LogicalOlapScan.class::isInstance);
+
StatsCalculator.disableJoinReorderIfTableRowCountNotAvailable(scans,
cascadesContext);
+ }
+
optimize();
if (statementContext.getConnectContext().getExecutor() != null) {
statementContext.getConnectContext().getExecutor().getSummaryProfile().setNereidsOptimizeTime();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index 1b67b80a62d..9458407295d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.stats;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.Pair;
@@ -179,6 +180,11 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
this.cascadesContext = context;
}
+ private StatsCalculator(CascadesContext context) {
+ this.groupExpression = null;
+ this.cascadesContext = context;
+ }
+
public Map<String, Histogram> getTotalHistogramMap() {
return totalHistogramMap;
}
@@ -1132,4 +1138,46 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
return groupExpression.childStatistics(1);
}
+ /**
+ * if the table is not analyzed and BE does not report row count, return -1
+ */
+ private double getOlapTableRowCount(OlapScan olapScan) {
+ OlapTable olapTable = olapScan.getTable();
+ AnalysisManager analysisManager =
Env.getCurrentEnv().getAnalysisManager();
+ TableStatsMeta tableMeta =
analysisManager.findTableStatsStatus(olapScan.getTable().getId());
+ double rowCount = -1;
+ if (tableMeta != null && tableMeta.userInjected) {
+ rowCount = tableMeta.getRowCount(olapScan.getSelectedIndexId());
+ } else {
+ rowCount =
olapTable.getRowCountForIndex(olapScan.getSelectedIndexId(), true);
+ if (rowCount == -1) {
+ if (tableMeta != null) {
+ rowCount =
tableMeta.getRowCount(olapScan.getSelectedIndexId());
+ }
+ }
+ }
+ return rowCount;
+ }
+
+ /**
+ * disable join reorder if any table row count is not available.
+ */
+ public static void disableJoinReorderIfTableRowCountNotAvailable(
+ List<LogicalOlapScan> scans,
+ CascadesContext context) {
+ StatsCalculator calculator = new StatsCalculator(context);
+ for (LogicalOlapScan scan : scans) {
+ double rowCount = calculator.getOlapTableRowCount(scan);
+ if (rowCount == -1 && ConnectContext.get() != null) {
+ try {
+
ConnectContext.get().getSessionVariable().disableNereidsJoinReorderOnce();
+ LOG.info("disable join reorder since row count not
available: "
+ + scan.getTable().getNameWithFullQualifiers());
+ } catch (Exception e) {
+ LOG.info("disableNereidsJoinReorderOnce failed");
+ }
+ return;
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
