This is an automated email from the ASF dual-hosted git repository.
englefly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 800f5c600be [feat](nereids)disable join reorder if any table row count
is not available (#43000)
800f5c600be is described below
commit 800f5c600be31c0cce8aa1eef236a53528e1f8fd
Author: minghong <[email protected]>
AuthorDate: Mon Nov 4 19:22:32 2024 +0800
[feat](nereids)disable join reorder if any table row count is not available
(#43000)
## Proposed changes
in previous PR #41790, if any Olap table row count is not available,
planner set disable_join_reorder true.
this pr make this rule apply to external table.
Issue Number: close #xxx
<!--Describe your changes.-->
---
.../org/apache/doris/nereids/NereidsPlanner.java | 6 ++--
.../doris/nereids/stats/StatsCalculator.java | 41 ++++++++++++++--------
2 files changed, 29 insertions(+), 18 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index 7ea92ee73b3..86886336d85 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -54,11 +54,11 @@ import
org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.ComputeResultSet;
import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
import
org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
import org.apache.doris.nereids.trees.plans.distribute.DistributePlanner;
import org.apache.doris.nereids.trees.plans.distribute.DistributedPlan;
import org.apache.doris.nereids.trees.plans.distribute.FragmentIdMapping;
-import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSqlCache;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
@@ -272,8 +272,8 @@ public class NereidsPlanner extends Planner {
// 2. ut test. In ut test, FeConstants.enableInternalSchemaDb is
false or FeConstants.runningUnitTest is true
if (FeConstants.enableInternalSchemaDb && !FeConstants.runningUnitTest
&& !cascadesContext.isLeadingDisableJoinReorder()) {
- List<LogicalOlapScan> scans = cascadesContext.getRewritePlan()
- .collectToList(LogicalOlapScan.class::isInstance);
+ List<CatalogRelation> scans = cascadesContext.getRewritePlan()
+ .collectToList(CatalogRelation.class::isInstance);
Optional<String> disableJoinReorderReason = StatsCalculator
.disableJoinReorderIfStatsInvalid(scans, cascadesContext);
disableJoinReorderReason.ifPresent(statementContext::setDisableJoinReorderReason);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index d4a9a81fc07..a3c117bb46f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -226,32 +226,34 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
* 2. col stats ndv=0 but minExpr or maxExpr is not null
* 3. ndv > 10 * rowCount
*/
- public static Optional<String>
disableJoinReorderIfStatsInvalid(List<LogicalOlapScan> scans,
+ public static Optional<String>
disableJoinReorderIfStatsInvalid(List<CatalogRelation> scans,
CascadesContext context) {
StatsCalculator calculator = new StatsCalculator(context);
if (ConnectContext.get() == null) {
// ut case
return Optional.empty();
}
- for (LogicalOlapScan scan : scans) {
- double rowCount = calculator.getOlapTableRowCount(scan);
+ for (CatalogRelation scan : scans) {
+ double rowCount = calculator.getTableRowCount(scan);
// row count not available
if (rowCount == -1) {
LOG.info("disable join reorder since row count not available: "
+ scan.getTable().getNameWithFullQualifiers());
return Optional.of("table[" + scan.getTable().getName() + "]
row count is invalid");
}
- // ndv abnormal
- Optional<String> reason = calculator.checkNdvValidation(scan,
rowCount);
- if (reason.isPresent()) {
- try {
-
ConnectContext.get().getSessionVariable().disableNereidsJoinReorderOnce();
- LOG.info("disable join reorder since col stats invalid: "
- + reason.get());
- } catch (Exception e) {
- LOG.info("disableNereidsJoinReorderOnce failed");
+ if (scan instanceof OlapScan) {
+ // ndv abnormal
+ Optional<String> reason =
calculator.checkNdvValidation((OlapScan) scan, rowCount);
+ if (reason.isPresent()) {
+ try {
+
ConnectContext.get().getSessionVariable().disableNereidsJoinReorderOnce();
+ LOG.info("disable join reorder since col stats
invalid: "
+ + reason.get());
+ } catch (Exception e) {
+ LOG.info("disableNereidsJoinReorderOnce failed");
+ }
+ return reason;
}
- return reason;
}
}
return Optional.empty();
@@ -407,6 +409,14 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
}
}
+ private double getTableRowCount(CatalogRelation scan) {
+ if (scan instanceof OlapScan) {
+ return getOlapTableRowCount((OlapScan) scan);
+ } else {
+ return scan.getTable().getRowCount();
+ }
+ }
+
/**
* if the table is not analyzed and BE does not report row count, return -1
*/
@@ -1212,11 +1222,12 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
*/
private Statistics computeCatalogRelation(CatalogRelation catalogRelation)
{
StatisticsBuilder builder = new StatisticsBuilder();
+ double tableRowCount = catalogRelation.getTable().getRowCount();
// for FeUt, use ColumnStatistic.UNKNOWN
if (!FeConstants.enableInternalSchemaDb
|| ConnectContext.get() == null
|| ConnectContext.get().getSessionVariable().internalSession) {
-
builder.setRowCount(catalogRelation.getTable().getRowCountForNereids());
+ builder.setRowCount(Math.max(1, tableRowCount));
for (Slot slot : catalogRelation.getOutput()) {
builder.putColumnStatistics(slot, ColumnStatistic.UNKNOWN);
}
@@ -1232,8 +1243,8 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
}
}
Set<SlotReference> slotSet = slotSetBuilder.build();
- double tableRowCount = catalogRelation.getTable().getRowCount();
if (tableRowCount <= 0) {
+ tableRowCount = 1;
// try to get row count from col stats
for (SlotReference slot : slotSet) {
ColumnStatistic cache =
getColumnStatsFromTableCache(catalogRelation, slot);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
