This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4c751202e3 [fix](Es Catalog) Pushdown predicate of Es catalog query in
nereids (#23345)
4c751202e3 is described below
commit 4c751202e3b19725e2ff8c7a41aa83e0f8f109b2
Author: Tiewei Fang <[email protected]>
AuthorDate: Wed Aug 23 18:03:39 2023 +0800
[fix](Es Catalog) Pushdown predicate of Es catalog query in nereids (#23345)
---
.../glue/translator/PhysicalPlanTranslator.java | 1 +
.../doris/nereids/jobs/executor/Rewriter.java | 4 ++-
.../org/apache/doris/nereids/rules/RuleType.java | 1 +
.../LogicalEsScanToPhysicalEsScan.java | 3 ++-
.../PushConjunctsIntoEsScan.java} | 28 +++++++++------------
.../nereids/trees/plans/logical/LogicalEsScan.java | 29 +++++++++++++++++++---
.../trees/plans/physical/PhysicalEsScan.java | 23 +++++++++++++----
7 files changed, 62 insertions(+), 27 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 5a83a7acf4..6613fd71a9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -481,6 +481,7 @@ public class PhysicalPlanTranslator extends
DefaultPlanVisitor<PlanFragment, Pla
ExternalTable table = esScan.getTable();
TupleDescriptor tupleDescriptor = generateTupleDesc(slots, table,
context);
EsScanNode esScanNode = new EsScanNode(context.nextPlanNodeId(),
tupleDescriptor, true);
+
esScanNode.addConjuncts(translateToLegacyConjuncts(esScan.getConjuncts()));
Utils.execWithUncheckedException(esScanNode::init);
context.addScanNode(esScanNode);
context.getRuntimeTranslator().ifPresent(
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index e134f9822d..07963987fc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -79,6 +79,7 @@ import
org.apache.doris.nereids.rules.rewrite.PruneFileScanPartition;
import org.apache.doris.nereids.rules.rewrite.PruneOlapScanPartition;
import org.apache.doris.nereids.rules.rewrite.PruneOlapScanTablet;
import org.apache.doris.nereids.rules.rewrite.PullUpCteAnchor;
+import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoEsScan;
import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoJdbcScan;
import org.apache.doris.nereids.rules.rewrite.PushFilterInsideJoin;
import org.apache.doris.nereids.rules.rewrite.PushProjectIntoOneRowRelation;
@@ -271,7 +272,8 @@ public class Rewriter extends AbstractBatchJobExecutor {
topDown(
new PruneOlapScanPartition(),
new PruneFileScanPartition(),
- new PushConjunctsIntoJdbcScan()
+ new PushConjunctsIntoJdbcScan(),
+ new PushConjunctsIntoEsScan()
)
),
topic("MV optimization",
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 5bc135ae49..baed7b53ea 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -221,6 +221,7 @@ public enum RuleType {
OLAP_SCAN_PARTITION_PRUNE(RuleTypeClass.REWRITE),
FILE_SCAN_PARTITION_PRUNE(RuleTypeClass.REWRITE),
PUSH_CONJUNCTS_INTO_JDBC_SCAN(RuleTypeClass.REWRITE),
+ PUSH_CONJUNCTS_INTO_ES_SCAN(RuleTypeClass.REWRITE),
OLAP_SCAN_TABLET_PRUNE(RuleTypeClass.REWRITE),
PUSH_AGGREGATE_TO_OLAP_SCAN(RuleTypeClass.REWRITE),
EXTRACT_SINGLE_TABLE_EXPRESSION_FROM_DISJUNCTION(RuleTypeClass.REWRITE),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalEsScanToPhysicalEsScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalEsScanToPhysicalEsScan.java
index 1e4db2eba1..9e83fec2a1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalEsScanToPhysicalEsScan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalEsScanToPhysicalEsScan.java
@@ -37,7 +37,8 @@ public class LogicalEsScanToPhysicalEsScan extends
OneImplementationRuleFactory
esScan.getQualifier(),
DistributionSpecAny.INSTANCE,
Optional.empty(),
- esScan.getLogicalProperties())
+ esScan.getLogicalProperties(),
+ esScan.getConjuncts())
).toRule(RuleType.LOGICAL_ES_SCAN_TO_PHYSICAL_ES_SCAN_RULE);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalEsScanToPhysicalEsScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoEsScan.java
similarity index 55%
copy from
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalEsScanToPhysicalEsScan.java
copy to
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoEsScan.java
index 1e4db2eba1..9bd485716d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalEsScanToPhysicalEsScan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoEsScan.java
@@ -15,29 +15,25 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.nereids.rules.implementation;
+package org.apache.doris.nereids.rules.rewrite;
-import org.apache.doris.nereids.properties.DistributionSpecAny;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
-import org.apache.doris.nereids.trees.plans.physical.PhysicalEsScan;
-
-import java.util.Optional;
+import org.apache.doris.nereids.trees.plans.logical.LogicalEsScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
/**
- * Implementation rule that convert logical JdbcScan to physical JdbcScan.
+ * Rewrite es plan to set the conjuncts.
*/
-public class LogicalEsScanToPhysicalEsScan extends
OneImplementationRuleFactory {
+public class PushConjunctsIntoEsScan extends OneRewriteRuleFactory {
+
@Override
public Rule build() {
- return logicalEsScan().then(esScan ->
- new PhysicalEsScan(
- esScan.getRelationId(),
- esScan.getTable(),
- esScan.getQualifier(),
- DistributionSpecAny.INSTANCE,
- Optional.empty(),
- esScan.getLogicalProperties())
- ).toRule(RuleType.LOGICAL_ES_SCAN_TO_PHYSICAL_ES_SCAN_RULE);
+ return logicalFilter(logicalEsScan()).thenApply(ctx -> {
+ LogicalFilter<LogicalEsScan> filter = ctx.root;
+ LogicalEsScan scan = filter.child();
+ LogicalEsScan rewrittenScan =
scan.withConjuncts(filter.getConjuncts());
+ return rewrittenScan;
+ }).toRule(RuleType.PUSH_CONJUNCTS_INTO_ES_SCAN);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java
index d34f47266f..35c722083d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.catalog.external.ExternalTable;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.RelationId;
@@ -27,26 +28,32 @@ import
org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
import java.util.List;
+import java.util.Objects;
import java.util.Optional;
+import java.util.Set;
/**
* Logical scan for external es catalog.
*/
public class LogicalEsScan extends LogicalCatalogRelation {
+ private final Set<Expression> conjuncts;
+
/**
* Constructor for LogicalEsScan.
*/
public LogicalEsScan(RelationId id, ExternalTable table, List<String>
qualifier,
Optional<GroupExpression> groupExpression,
- Optional<LogicalProperties> logicalProperties) {
+ Optional<LogicalProperties> logicalProperties,
Set<Expression> conjuncts) {
super(id, PlanType.LOGICAL_ES_SCAN, table, qualifier, groupExpression,
logicalProperties);
+ this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts,
"conjuncts should not be null"));
}
public LogicalEsScan(RelationId id, ExternalTable table, List<String>
qualifier) {
- this(id, table, qualifier, Optional.empty(), Optional.empty());
+ this(id, table, qualifier, Optional.empty(), Optional.empty(),
ImmutableSet.of());
}
@Override
@@ -66,13 +73,19 @@ public class LogicalEsScan extends LogicalCatalogRelation {
@Override
public LogicalEsScan withGroupExpression(Optional<GroupExpression>
groupExpression) {
return new LogicalEsScan(relationId, (ExternalTable) table, qualifier,
groupExpression,
- Optional.of(getLogicalProperties()));
+ Optional.of(getLogicalProperties()), conjuncts);
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression>
groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan>
children) {
- return new LogicalEsScan(relationId, (ExternalTable) table, qualifier,
groupExpression, logicalProperties);
+ return new LogicalEsScan(relationId, (ExternalTable) table, qualifier,
groupExpression, logicalProperties,
+ conjuncts);
+ }
+
+ public LogicalEsScan withConjuncts(Set<Expression> conjuncts) {
+ return new LogicalEsScan(relationId, (ExternalTable) table, qualifier,
groupExpression,
+ Optional.of(getLogicalProperties()), conjuncts);
}
@Override
@@ -80,4 +93,12 @@ public class LogicalEsScan extends LogicalCatalogRelation {
return visitor.visitLogicalEsScan(this, context);
}
+ @Override
+ public boolean equals(Object o) {
+ return super.equals(o) && Objects.equals(conjuncts, ((LogicalEsScan)
o).conjuncts);
+ }
+
+ public Set<Expression> getConjuncts() {
+ return this.conjuncts;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java
index 4ee49ca198..eaddff7a26 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java
@@ -22,6 +22,7 @@ import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.DistributionSpec;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.RelationId;
@@ -29,8 +30,12 @@ import
org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.statistics.Statistics;
+import com.google.common.collect.ImmutableSet;
+
import java.util.List;
+import java.util.Objects;
import java.util.Optional;
+import java.util.Set;
/**
* Physical es scan for external catalog.
@@ -38,15 +43,17 @@ import java.util.Optional;
public class PhysicalEsScan extends PhysicalCatalogRelation {
private final DistributionSpec distributionSpec;
+ private final Set<Expression> conjuncts;
/**
* Constructor for PhysicalEsScan.
*/
public PhysicalEsScan(RelationId id, ExternalTable table, List<String>
qualifier,
DistributionSpec distributionSpec, Optional<GroupExpression>
groupExpression,
- LogicalProperties logicalProperties) {
+ LogicalProperties logicalProperties, Set<Expression> conjuncts) {
super(id, PlanType.PHYSICAL_ES_SCAN, table, qualifier,
groupExpression, logicalProperties);
this.distributionSpec = distributionSpec;
+ this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts,
"conjuncts should not be null"));
}
/**
@@ -54,10 +61,12 @@ public class PhysicalEsScan extends PhysicalCatalogRelation
{
*/
public PhysicalEsScan(RelationId id, ExternalTable table, List<String>
qualifier,
DistributionSpec distributionSpec, Optional<GroupExpression>
groupExpression,
- LogicalProperties logicalProperties, PhysicalProperties
physicalProperties, Statistics statistics) {
+ LogicalProperties logicalProperties, PhysicalProperties
physicalProperties, Statistics statistics,
+ Set<Expression> conjuncts) {
super(id, PlanType.PHYSICAL_ES_SCAN, table, qualifier,
groupExpression, logicalProperties,
physicalProperties, statistics);
this.distributionSpec = distributionSpec;
+ this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts,
"conjuncts should not be null"));
}
@Override
@@ -77,14 +86,14 @@ public class PhysicalEsScan extends PhysicalCatalogRelation
{
@Override
public PhysicalEsScan withGroupExpression(Optional<GroupExpression>
groupExpression) {
return new PhysicalEsScan(relationId, getTable(), qualifier,
distributionSpec,
- groupExpression, getLogicalProperties());
+ groupExpression, getLogicalProperties(), conjuncts);
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression>
groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan>
children) {
return new PhysicalEsScan(relationId, getTable(), qualifier,
distributionSpec,
- groupExpression, logicalProperties.get());
+ groupExpression, logicalProperties.get(), conjuncts);
}
@Override
@@ -96,6 +105,10 @@ public class PhysicalEsScan extends PhysicalCatalogRelation
{
public PhysicalEsScan withPhysicalPropertiesAndStats(PhysicalProperties
physicalProperties,
Statistics
statsDeriveResult) {
return new PhysicalEsScan(relationId, getTable(), qualifier,
distributionSpec,
- groupExpression, getLogicalProperties(), physicalProperties,
statsDeriveResult);
+ groupExpression, getLogicalProperties(), physicalProperties,
statsDeriveResult, conjuncts);
+ }
+
+ public Set<Expression> getConjuncts() {
+ return this.conjuncts;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
