This is an automated email from the ASF dual-hosted git repository.
jakevin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 007e8158f45 [refactor](Nereids): compute unique and uniform property
respectively (#32908)
007e8158f45 is described below
commit 007e8158f4506e89f870097b4f1e28f94f24dae6
Author: 谢健 <[email protected]>
AuthorDate: Thu Apr 11 15:57:38 2024 +0800
[refactor](Nereids): compute unique and uniform property respectively
(#32908)
---
.../nereids/properties/FunctionalDependencies.java | 4 +
.../doris/nereids/trees/plans/AbstractPlan.java | 2 +-
.../trees/plans/BlockFuncDepsPropagation.java | 18 +-
.../nereids/trees/plans/PropagateFuncDeps.java | 20 +-
.../trees/plans/logical/LogicalAggregate.java | 108 +++---
.../trees/plans/logical/LogicalAssertNumRows.java | 33 +-
.../plans/logical/LogicalCatalogRelation.java | 23 +-
.../plans/logical/LogicalDeferMaterializeTopN.java | 29 +-
.../nereids/trees/plans/logical/LogicalExcept.java | 59 ++--
.../nereids/trees/plans/logical/LogicalFilter.java | 25 +-
.../trees/plans/logical/LogicalGenerate.java | 20 +-
.../nereids/trees/plans/logical/LogicalHaving.java | 17 +-
.../trees/plans/logical/LogicalIntersect.java | 27 +-
.../nereids/trees/plans/logical/LogicalJoin.java | 130 ++++---
.../nereids/trees/plans/logical/LogicalLimit.java | 30 +-
.../trees/plans/logical/LogicalOneRowRelation.java | 21 +-
.../nereids/trees/plans/logical/LogicalPlan.java | 17 +-
.../trees/plans/logical/LogicalProject.java | 65 ++--
.../nereids/trees/plans/logical/LogicalRepeat.java | 19 +-
.../trees/plans/logical/LogicalSubQueryAlias.java | 26 +-
.../nereids/trees/plans/logical/LogicalTopN.java | 30 +-
.../nereids/trees/plans/logical/LogicalUnion.java | 22 +-
.../nereids/trees/plans/logical/LogicalView.java | 20 +-
.../nereids/trees/plans/logical/LogicalWindow.java | 73 ++--
.../apache/doris/nereids/util/ExpressionUtils.java | 4 +-
.../properties/FunctionalDependenciesTest.java | 4 +-
.../doris/nereids/properties/UniformTest.java | 207 ++++++++++++
...tionalDependenciesTest.java => UniqueTest.java} | 376 ++++++++++-----------
28 files changed, 866 insertions(+), 563 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FunctionalDependencies.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FunctionalDependencies.java
index c7e6030e137..2b0c1f5c914 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FunctionalDependencies.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FunctionalDependencies.java
@@ -136,6 +136,10 @@ public class FunctionalDependencies {
uniqueSet.add(slotSet);
}
+ public void addUniqueSlot(FunctionalDependencies
functionalDependencies) {
+ uniqueSet.add(functionalDependencies.uniqueSet);
+ }
+
public void addFdItems(ImmutableSet<FdItem> items) {
fdItems = ImmutableSet.copyOf(items);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java
index 286a92aab76..4747aa84898 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java
@@ -179,7 +179,7 @@ public abstract class AbstractPlan extends
AbstractTreeNode<Plan> implements Pla
} else {
Supplier<List<Slot>> outputSupplier =
Suppliers.memoize(this::computeOutput);
Supplier<FunctionalDependencies> fdSupplier = () -> this
instanceof LogicalPlan
- ? ((LogicalPlan) this).computeFuncDeps(outputSupplier)
+ ? ((LogicalPlan) this).computeFuncDeps()
: FunctionalDependencies.EMPTY_FUNC_DEPS;
return new LogicalProperties(outputSupplier, fdSupplier);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/BlockFuncDepsPropagation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/BlockFuncDepsPropagation.java
index 07804587e3b..a679ad0f7d2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/BlockFuncDepsPropagation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/BlockFuncDepsPropagation.java
@@ -19,25 +19,31 @@ package org.apache.doris.nereids.trees.plans;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
-import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import com.google.common.collect.ImmutableSet;
-import java.util.List;
-import java.util.function.Supplier;
-
/**
* Block fd propagation, it always returns an empty fd
*/
public interface BlockFuncDepsPropagation extends LogicalPlan {
@Override
- default FunctionalDependencies computeFuncDeps(Supplier<List<Slot>>
outputSupplier) {
+ default FunctionalDependencies computeFuncDeps() {
return FunctionalDependencies.EMPTY_FUNC_DEPS;
}
@Override
- default ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>>
outputSupplier) {
+ default ImmutableSet<FdItem> computeFdItems() {
return ImmutableSet.of();
}
+
+ @Override
+ default void computeUnique(FunctionalDependencies.Builder fdBuilder) {
+ // don't generate
+ }
+
+ @Override
+ default void computeUniform(FunctionalDependencies.Builder fdBuilder) {
+ // don't generate
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PropagateFuncDeps.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PropagateFuncDeps.java
index c344f22ca21..f37059049ae 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PropagateFuncDeps.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PropagateFuncDeps.java
@@ -19,20 +19,16 @@ package org.apache.doris.nereids.trees.plans;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
-import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import com.google.common.collect.ImmutableSet;
-import java.util.List;
-import java.util.function.Supplier;
-
/**
* Propagate fd, keep children's fd
*/
public interface PropagateFuncDeps extends LogicalPlan {
@Override
- default FunctionalDependencies computeFuncDeps(Supplier<List<Slot>>
outputSupplier) {
+ default FunctionalDependencies computeFuncDeps() {
if (children().size() == 1) {
// Note when changing function dependencies, we always clone it.
// So it's safe to return a reference
@@ -42,13 +38,13 @@ public interface PropagateFuncDeps extends LogicalPlan {
children().stream()
.map(p -> p.getLogicalProperties().getFunctionalDependencies())
.forEach(builder::addFunctionalDependencies);
- ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
+ ImmutableSet<FdItem> fdItems = computeFdItems();
builder.addFdItems(fdItems);
return builder.build();
}
@Override
- default ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>>
outputSupplier) {
+ default ImmutableSet<FdItem> computeFdItems() {
if (children().size() == 1) {
// Note when changing function dependencies, we always clone it.
// So it's safe to return a reference
@@ -60,4 +56,14 @@ public interface PropagateFuncDeps extends LogicalPlan {
.forEach(builder::addAll);
return builder.build();
}
+
+ @Override
+ default void computeUnique(FunctionalDependencies.Builder fdBuilder) {
+
fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies());
+ }
+
+ @Override
+ default void computeUniform(FunctionalDependencies.Builder fdBuilder) {
+
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java
index fa4f891e7a2..1e5e5a45abe 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java
@@ -21,17 +21,13 @@ import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdFactory;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
-import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.TableFdItem;
-import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
-import org.apache.doris.nereids.trees.expressions.VirtualSlotReference;
import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
-import
org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
import org.apache.doris.nereids.trees.expressions.functions.agg.Ndv;
import org.apache.doris.nereids.trees.plans.Plan;
@@ -45,12 +41,9 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
-import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
-import java.util.Set;
-import java.util.function.Supplier;
import java.util.stream.Collectors;
/**
@@ -300,80 +293,85 @@ public class LogicalAggregate<CHILD_TYPE extends Plan>
hasPushed, sourceRepeat, Optional.empty(), Optional.empty(),
normalizedChild);
}
- private void updateFuncDepsGroupByUnique(NamedExpression namedExpression,
Builder fdBuilder) {
- if (ExpressionUtils.isInjective(namedExpression)) {
-
fdBuilder.addUniqueSlot(ImmutableSet.copyOf(namedExpression.getInputSlots()));
- return;
+ private boolean isUniqueGroupByUnique(NamedExpression namedExpression) {
+ if (namedExpression.children().size() != 1) {
+ return false;
}
+ Expression agg = namedExpression.child(0);
+ return ExpressionUtils.isInjectiveAgg(agg)
+ &&
child().getLogicalProperties().getFunctionalDependencies().isUniqueAndNotNull(agg.getInputSlots());
+ }
- if (!(namedExpression instanceof Alias && namedExpression.child(0)
instanceof AggregateFunction)) {
- return;
+ private boolean isUniformGroupByUnique(NamedExpression namedExpression) {
+ if (namedExpression.children().size() != 1) {
+ return false;
}
+ Expression agg = namedExpression.child(0);
+ return agg instanceof Count || agg instanceof Ndv;
+ }
- AggregateFunction agg = (AggregateFunction) namedExpression.child(0);
- if (agg instanceof Count || agg instanceof Ndv) {
- fdBuilder.addUniformSlot(namedExpression.toSlot());
+ @Override
+ public void computeUnique(FunctionalDependencies.Builder fdBuilder) {
+ if (this.sourceRepeat.isPresent()) {
+ // roll up may generate new data
+ return;
+ }
+ FunctionalDependencies childFd =
child(0).getLogicalProperties().getFunctionalDependencies();
+ ImmutableSet<Slot> groupByKeys = groupByExpressions.stream()
+ .map(s -> (Slot) s)
+ .collect(ImmutableSet.toImmutableSet());
+ // when group by all tuples, the result only have one row
+ if (groupByExpressions.isEmpty() ||
childFd.isUniformAndNotNull(groupByKeys)) {
+ getOutput().forEach(fdBuilder::addUniqueSlot);
return;
}
- if (ExpressionUtils.isInjectiveAgg(agg)
- &&
child().getLogicalProperties().getFunctionalDependencies().isUniqueAndNotNull(agg.getInputSlots()))
{
- fdBuilder.addUniqueSlot(namedExpression.toSlot());
+ // propagate all unique slots
+ fdBuilder.addUniqueSlot(childFd);
+
+ // group by keys is unique
+ fdBuilder.addUniqueSlot(groupByKeys);
+
+ // group by unique may has unique aggregate result
+ if (childFd.isUniqueAndNotNull(groupByKeys)) {
+ for (NamedExpression namedExpression : getOutputExpressions()) {
+ if (isUniqueGroupByUnique(namedExpression)) {
+ fdBuilder.addUniqueSlot(namedExpression.toSlot());
+ }
+ }
}
}
@Override
- public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>>
outputSupplier) {
+ public void computeUniform(FunctionalDependencies.Builder fdBuilder) {
+ // always propagate uniform
FunctionalDependencies childFd =
child(0).getLogicalProperties().getFunctionalDependencies();
- Set<Slot> outputSet = new HashSet<>(outputSupplier.get());
- Builder fdBuilder = new Builder();
- // when group by all tuples, the result only have one row
- if (groupByExpressions.isEmpty()) {
- outputSet.forEach(s -> {
- fdBuilder.addUniformSlot(s);
- fdBuilder.addUniqueSlot(s);
- });
- return fdBuilder.build();
- }
+ fdBuilder.addUniformSlot(childFd);
- // when group by complicate expression or virtual slot, just propagate
uniform slots
- if (groupByExpressions.stream()
- .anyMatch(s -> !(s instanceof SlotReference) || s instanceof
VirtualSlotReference)) {
- fdBuilder.addUniformSlot(childFd);
- fdBuilder.pruneSlots(outputSet);
- return fdBuilder.build();
+ if (this.sourceRepeat.isPresent()) {
+ // roll up may generate new data
+ return;
}
-
- // when group by uniform slot, the result only have one row
ImmutableSet<Slot> groupByKeys = groupByExpressions.stream()
.map(s -> (Slot) s)
.collect(ImmutableSet.toImmutableSet());
- if (childFd.isUniformAndNotNull(groupByKeys)) {
- outputSupplier.get().forEach(s -> {
- fdBuilder.addUniformSlot(s);
- fdBuilder.addUniqueSlot(s);
- });
+ // when group by all tuples, the result only have one row
+ if (groupByExpressions.isEmpty() ||
childFd.isUniformAndNotNull(groupByKeys)) {
+ getOutput().forEach(fdBuilder::addUniformSlot);
+ return;
}
- // when group by unique slot, the result depends on agg func
if (childFd.isUniqueAndNotNull(groupByKeys)) {
for (NamedExpression namedExpression : getOutputExpressions()) {
- updateFuncDepsGroupByUnique(namedExpression, fdBuilder);
+ if (isUniformGroupByUnique(namedExpression)) {
+ fdBuilder.addUniformSlot(namedExpression.toSlot());
+ }
}
}
-
- // group by keys is unique
- fdBuilder.addUniqueSlot(groupByKeys);
- fdBuilder.pruneSlots(outputSet);
-
- ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
- fdBuilder.addFdItems(fdItems);
-
- return fdBuilder.build();
}
@Override
- public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>>
outputSupplier) {
+ public ImmutableSet<FdItem> computeFdItems() {
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
ImmutableSet<SlotReference> groupByExprs =
getGroupByExpressions().stream()
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java
index 0279c9701ca..cad8d6e14d6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java
@@ -18,18 +18,21 @@
package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.FdItem;
+import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.AssertNumRowsElement;
+import
org.apache.doris.nereids.trees.expressions.AssertNumRowsElement.Assertion;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
-import org.apache.doris.nereids.trees.plans.PropagateFuncDeps;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Objects;
@@ -42,8 +45,7 @@ import java.util.stream.Collectors;
* If the number of rows is more than the desired num of rows, the query will
be cancelled.
* The cancelled reason will be reported by Backend and displayed back to the
user.
*/
-public class LogicalAssertNumRows<CHILD_TYPE extends Plan> extends
LogicalUnary<CHILD_TYPE> implements
- PropagateFuncDeps {
+public class LogicalAssertNumRows<CHILD_TYPE extends Plan> extends
LogicalUnary<CHILD_TYPE> {
private final AssertNumRowsElement assertNumRowsElement;
@@ -118,4 +120,29 @@ public class LogicalAssertNumRows<CHILD_TYPE extends Plan>
extends LogicalUnary<
public List<Slot> computeOutput() {
return child().getOutput().stream().map(o ->
o.withNullable(true)).collect(Collectors.toList());
}
+
+ @Override
+ public ImmutableSet<FdItem> computeFdItems() {
+ return ImmutableSet.of();
+ }
+
+ @Override
+ public void computeUnique(Builder fdBuilder) {
+ if (assertNumRowsElement.getDesiredNumOfRows() == 1
+ && (assertNumRowsElement.getAssertion() == Assertion.EQ
+ || assertNumRowsElement.getAssertion() == Assertion.LT
+ || assertNumRowsElement.getAssertion() == Assertion.LE)) {
+ getOutput().forEach(fdBuilder::addUniqueSlot);
+ }
+ }
+
+ @Override
+ public void computeUniform(Builder fdBuilder) {
+ if (assertNumRowsElement.getDesiredNumOfRows() == 1
+ && (assertNumRowsElement.getAssertion() == Assertion.EQ
+ || assertNumRowsElement.getAssertion() == Assertion.LT
+ || assertNumRowsElement.getAssertion() == Assertion.LE)) {
+ getOutput().forEach(fdBuilder::addUniformSlot);
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java
index 5f7982aae46..433feb741ba 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java
@@ -49,7 +49,6 @@ import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
-import java.util.function.Supplier;
/**
* abstract class catalog relation for logical relation
@@ -128,9 +127,16 @@ public abstract class LogicalCatalogRelation extends
LogicalRelation implements
}
@Override
- public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>>
outputSupplier) {
+ public FunctionalDependencies computeFuncDeps() {
Builder fdBuilder = new Builder();
- Set<Slot> outputSet = Utils.fastToImmutableSet(outputSupplier.get());
+ computeUnique(fdBuilder);
+
fdBuilder.addFdItems(computeFdItems(Utils.fastToImmutableSet(getOutputSet())));
+ return fdBuilder.build();
+ }
+
+ @Override
+ public void computeUnique(FunctionalDependencies.Builder fdBuilder) {
+ Set<Slot> outputSet = Utils.fastToImmutableSet(getOutputSet());
if (table instanceof OlapTable && ((OlapTable)
table).getKeysType().isAggregationFamily()) {
ImmutableSet.Builder<Slot> uniqSlots =
ImmutableSet.builderWithExpectedSize(outputSet.size());
for (Slot slot : outputSet) {
@@ -154,13 +160,16 @@ public abstract class LogicalCatalogRelation extends
LogicalRelation implements
Set<Column> columns = c.getUniqueKeys(table);
fdBuilder.addUniqueSlot((ImmutableSet)
findSlotsByColumn(outputSet, columns));
}
- fdBuilder.addFdItems(computeFdItems(outputSet));
- return fdBuilder.build();
}
@Override
- public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>>
outputSupplier) {
- return computeFdItems(Utils.fastToImmutableSet(outputSupplier.get()));
+ public void computeUniform(FunctionalDependencies.Builder fdBuilder) {
+ // No uniform slot for catalog relation
+ }
+
+ @Override
+ public ImmutableSet<FdItem> computeFdItems() {
+ return computeFdItems(Utils.fastToImmutableSet(getOutputSet()));
}
private ImmutableSet<FdItem> computeFdItems(Set<Slot> outputSet) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java
index 5bf0afe1fae..06c03f17f04 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java
@@ -22,7 +22,6 @@ import org.apache.doris.nereids.properties.ExprFdItem;
import org.apache.doris.nereids.properties.FdFactory;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
-import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.OrderKey;
import org.apache.doris.nereids.trees.expressions.ExprId;
@@ -43,7 +42,6 @@ import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
-import java.util.function.Supplier;
/**
* use for defer materialize top n
@@ -120,26 +118,29 @@ public class LogicalDeferMaterializeTopN<CHILD_TYPE
extends Plan> extends Logica
}
@Override
- public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>>
outputSupplier) {
- FunctionalDependencies fd =
child(0).getLogicalProperties().getFunctionalDependencies();
+ public void computeUnique(FunctionalDependencies.Builder fdBuilder) {
if (getLimit() == 1) {
- Builder builder = new Builder();
- List<Slot> output = outputSupplier.get();
- output.forEach(builder::addUniformSlot);
- output.forEach(builder::addUniqueSlot);
- ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
- builder.addFdItems(fdItems);
- fd = builder.build();
+ getOutput().forEach(fdBuilder::addUniqueSlot);
+ } else {
+
fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
- return fd;
}
@Override
- public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>>
outputSupplier) {
+ public void computeUniform(FunctionalDependencies.Builder fdBuilder) {
+ if (getLimit() == 1) {
+ getOutput().forEach(fdBuilder::addUniformSlot);
+ } else {
+
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
+ }
+ }
+
+ @Override
+ public ImmutableSet<FdItem> computeFdItems() {
ImmutableSet<FdItem> fdItems =
child(0).getLogicalProperties().getFunctionalDependencies().getFdItems();
if (getLimit() == 1) {
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
- List<Slot> output = outputSupplier.get();
+ List<Slot> output = getOutput();
ImmutableSet<SlotReference> slotSet = output.stream()
.filter(SlotReference.class::isInstance)
.map(SlotReference.class::cast)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java
index 707da68fe8f..d0223954411 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExcept.java
@@ -21,7 +21,7 @@ import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.ExprFdItem;
import org.apache.doris.nereids.properties.FdFactory;
import org.apache.doris.nereids.properties.FdItem;
-import org.apache.doris.nereids.properties.FunctionalDependencies;
+import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
@@ -39,7 +39,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
-import java.util.function.Supplier;
/**
* Logical Except.
@@ -109,29 +108,8 @@ public class LogicalExcept extends LogicalSetOperation {
}
@Override
- public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>>
outputSupplier) {
- FunctionalDependencies.Builder builder = new FunctionalDependencies
-
.Builder(child(0).getLogicalProperties().getFunctionalDependencies());
- Map<Slot, Slot> replaceMap = new HashMap<>();
- List<Slot> output = outputSupplier.get();
- List<? extends Slot> originalOutputs = regularChildrenOutputs.isEmpty()
- ? child(0).getOutput()
- : regularChildrenOutputs.get(0);
- for (int i = 0; i < output.size(); i++) {
- replaceMap.put(originalOutputs.get(i), output.get(i));
- }
- builder.replace(replaceMap);
- if (qualifier == Qualifier.DISTINCT) {
- builder.addUniqueSlot(ImmutableSet.copyOf(outputSupplier.get()));
- }
- ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
- builder.addFdItems(fdItems);
- return builder.build();
- }
-
- @Override
- public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>>
outputSupplier) {
- Set<NamedExpression> output =
ImmutableSet.copyOf(outputSupplier.get());
+ public ImmutableSet<FdItem> computeFdItems() {
+ Set<NamedExpression> output = ImmutableSet.copyOf(getOutput());
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
ImmutableSet<SlotReference> exprs = output.stream()
@@ -152,4 +130,35 @@ public class LogicalExcept extends LogicalSetOperation {
return builder.build();
}
+
+ @Override
+ public void computeUnique(Builder fdBuilder) {
+
fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies());
+ if (qualifier == Qualifier.DISTINCT) {
+ fdBuilder.addUniqueSlot(ImmutableSet.copyOf(getOutput()));
+ }
+ Map<Slot, Slot> replaceMap = new HashMap<>();
+ List<Slot> output = getOutput();
+ List<? extends Slot> originalOutputs = regularChildrenOutputs.isEmpty()
+ ? child(0).getOutput()
+ : regularChildrenOutputs.get(0);
+ for (int i = 0; i < output.size(); i++) {
+ replaceMap.put(originalOutputs.get(i), output.get(i));
+ }
+ fdBuilder.replace(replaceMap);
+ }
+
+ @Override
+ public void computeUniform(Builder fdBuilder) {
+
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
+ Map<Slot, Slot> replaceMap = new HashMap<>();
+ List<Slot> output = getOutput();
+ List<? extends Slot> originalOutputs = regularChildrenOutputs.isEmpty()
+ ? child(0).getOutput()
+ : regularChildrenOutputs.get(0);
+ for (int i = 0; i < output.size(); i++) {
+ replaceMap.put(originalOutputs.get(i), output.get(i));
+ }
+ fdBuilder.replace(replaceMap);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java
index 4375124a224..69d378fd6a5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java
@@ -19,7 +19,6 @@ package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdItem;
-import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
@@ -41,7 +40,6 @@ import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
-import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -149,17 +147,7 @@ public class LogicalFilter<CHILD_TYPE extends Plan>
extends LogicalUnary<CHILD_T
}
@Override
- public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>>
outputSupplier) {
- Builder fdBuilder = new Builder(
- child().getLogicalProperties().getFunctionalDependencies());
- getConjuncts().forEach(e ->
fdBuilder.addUniformSlot(ExpressionUtils.extractUniformSlot(e)));
- ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
- fdBuilder.addFdItems(fdItems);
- return fdBuilder.build();
- }
-
- @Override
- public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>>
outputSupplier) {
+ public ImmutableSet<FdItem> computeFdItems() {
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
ImmutableSet<FdItem> childItems =
child().getLogicalProperties().getFunctionalDependencies().getFdItems();
@@ -167,4 +155,15 @@ public class LogicalFilter<CHILD_TYPE extends Plan>
extends LogicalUnary<CHILD_T
return builder.build();
}
+
+ @Override
+ public void computeUnique(Builder fdBuilder) {
+
fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies());
+ }
+
+ @Override
+ public void computeUniform(Builder fdBuilder) {
+ getConjuncts().forEach(e ->
fdBuilder.addUniformSlot(ExpressionUtils.extractUniformSlot(e)));
+
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java
index 0fb3964e511..a54a7514dbc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java
@@ -19,7 +19,7 @@ package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdItem;
-import org.apache.doris.nereids.properties.FunctionalDependencies;
+import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
@@ -38,7 +38,6 @@ import com.google.common.collect.Lists;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
-import java.util.function.Supplier;
/**
* plan for table generator, the statement like: SELECT * FROM tbl LATERAL
VIEW EXPLODE(c1) g as (gc1);
@@ -157,16 +156,17 @@ public class LogicalGenerate<CHILD_TYPE extends Plan>
extends LogicalUnary<CHILD
}
@Override
- public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>>
outputSupplier) {
- FunctionalDependencies.Builder builder = new
FunctionalDependencies.Builder();
-
builder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
- ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
- builder.addFdItems(fdItems);
- return builder.build();
+ public ImmutableSet<FdItem> computeFdItems() {
+ return ImmutableSet.of();
}
@Override
- public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>>
outputSupplier) {
- return ImmutableSet.of();
+ public void computeUnique(Builder fdBuilder) {
+ // don't generate and propagate unique
+ }
+
+ @Override
+ public void computeUniform(Builder fdBuilder) {
+
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java
index e4d01f12748..da526c33af9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java
@@ -19,7 +19,6 @@ package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdItem;
-import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
@@ -39,7 +38,6 @@ import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
-import java.util.function.Supplier;
/**
* Logical Having plan
@@ -120,17 +118,18 @@ public class LogicalHaving<CHILD_TYPE extends Plan>
extends LogicalUnary<CHILD_T
}
@Override
- public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>>
outputSupplier) {
- Builder fdBuilder = new Builder(
- child().getLogicalProperties().getFunctionalDependencies());
+ public void computeUnique(Builder fdBuilder) {
+
fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies());
+ }
+
+ @Override
+ public void computeUniform(Builder fdBuilder) {
getConjuncts().forEach(e ->
fdBuilder.addUniformSlot(ExpressionUtils.extractUniformSlot(e)));
- ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
- fdBuilder.addFdItems(fdItems);
- return fdBuilder.build();
+
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
@Override
- public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>>
outputSupplier) {
+ public ImmutableSet<FdItem> computeFdItems() {
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
ImmutableSet<FdItem> childItems =
child().getLogicalProperties().getFunctionalDependencies().getFdItems();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java
index 2fbdbd24561..e9e4889a8e5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java
@@ -22,6 +22,7 @@ import org.apache.doris.nereids.properties.ExprFdItem;
import org.apache.doris.nereids.properties.FdFactory;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
+import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
@@ -39,7 +40,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
-import java.util.function.Supplier;
/**
* Logical Intersect.
@@ -119,24 +119,29 @@ public class LogicalIntersect extends LogicalSetOperation
{
}
@Override
- public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>>
outputSupplier) {
- FunctionalDependencies.Builder builder = new
FunctionalDependencies.Builder();
+ public void computeUnique(Builder fdBuilder) {
for (Plan child : children) {
- builder.addFunctionalDependencies(
+ fdBuilder.addUniqueSlot(
child.getLogicalProperties().getFunctionalDependencies());
- replaceSlotInFuncDeps(builder, child.getOutput(),
outputSupplier.get());
+ replaceSlotInFuncDeps(fdBuilder, child.getOutput(), getOutput());
}
if (qualifier == Qualifier.DISTINCT) {
- builder.addUniqueSlot(ImmutableSet.copyOf(outputSupplier.get()));
+ fdBuilder.addUniqueSlot(ImmutableSet.copyOf(getOutput()));
+ }
+ }
+
+ @Override
+ public void computeUniform(Builder fdBuilder) {
+ for (Plan child : children) {
+ fdBuilder.addUniformSlot(
+ child.getLogicalProperties().getFunctionalDependencies());
+ replaceSlotInFuncDeps(fdBuilder, child.getOutput(), getOutput());
}
- ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
- builder.addFdItems(fdItems);
- return builder.build();
}
@Override
- public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>>
outputSupplier) {
- Set<NamedExpression> output =
ImmutableSet.copyOf(outputSupplier.get());
+ public ImmutableSet<FdItem> computeFdItems() {
+ Set<NamedExpression> output = ImmutableSet.copyOf(getOutput());
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
ImmutableSet<SlotReference> exprs = output.stream()
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
index d18a454fe73..ea92aeca22d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
@@ -24,7 +24,6 @@ import
org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdFactory;
import org.apache.doris.nereids.properties.FdItem;
-import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.TableFdItem;
@@ -58,7 +57,6 @@ import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
-import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.Nullable;
@@ -460,76 +458,7 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan,
RIGHT_CHILD_TYPE extends
}
@Override
- public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>>
outputSupplier) {
- if (isMarkJoin()) {
- // TODO disable function dependence calculation for mark join, but
need re-think this in future.
- return FunctionalDependencies.EMPTY_FUNC_DEPS;
- }
- //1. NALAJ and FOJ block functional dependencies
- if (joinType.isNullAwareLeftAntiJoin() || joinType.isFullOuterJoin()) {
- return FunctionalDependencies.EMPTY_FUNC_DEPS;
- }
-
- // left/right semi/anti join propagate left/right functional
dependencies
- if (joinType.isLeftAntiJoin() || joinType.isLefSemiJoin()) {
- return left().getLogicalProperties().getFunctionalDependencies();
- }
- if (joinType.isRightSemiJoin() || joinType.isRightAntiJoin()) {
- return right().getLogicalProperties().getFunctionalDependencies();
- }
-
- // if there is non-equal join conditions, block functional dependencies
- if (!otherJoinConjuncts.isEmpty()) {
- return FunctionalDependencies.EMPTY_FUNC_DEPS;
- }
-
- Pair<Set<Slot>, Set<Slot>> keys = extractNullRejectHashKeys();
- if (keys == null) {
- return FunctionalDependencies.EMPTY_FUNC_DEPS;
- }
-
- // Note here we only check whether the left is unique.
- // So the hash condition can't be null-safe
- // TODO: consider Null-safe hash condition when left and rigth is not
nullable
- boolean isLeftUnique = left().getLogicalProperties()
- .getFunctionalDependencies().isUnique(keys.first);
- boolean isRightUnique = right().getLogicalProperties()
- .getFunctionalDependencies().isUnique(keys.second);
- Builder fdBuilder = new Builder();
- if (joinType.isInnerJoin()) {
- // inner join propagate uniforms slots
- // And if the hash keys is unique, inner join can propagate all
functional dependencies
- if (isLeftUnique && isRightUnique) {
-
fdBuilder.addFunctionalDependencies(left().getLogicalProperties().getFunctionalDependencies());
-
fdBuilder.addFunctionalDependencies(right().getLogicalProperties().getFunctionalDependencies());
- } else {
-
fdBuilder.addUniformSlot(left().getLogicalProperties().getFunctionalDependencies());
-
fdBuilder.addUniformSlot(right().getLogicalProperties().getFunctionalDependencies());
- }
- }
-
- // left/right outer join propagate left/right uniforms slots
- // And if the right/left hash keys is unique,
- // join can propagate left/right functional dependencies
- if (joinType.isLeftOuterJoin()) {
- if (isRightUnique) {
- return
left().getLogicalProperties().getFunctionalDependencies();
- }
-
fdBuilder.addUniformSlot(left().getLogicalProperties().getFunctionalDependencies());
- }
- if (joinType.isRightOuterJoin()) {
- if (isLeftUnique) {
- return
left().getLogicalProperties().getFunctionalDependencies();
- }
-
fdBuilder.addUniformSlot(left().getLogicalProperties().getFunctionalDependencies());
- }
- ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
- fdBuilder.addFdItems(fdItems);
- return fdBuilder.build();
- }
-
- @Override
- public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>>
outputSupplier) {
+ public ImmutableSet<FdItem> computeFdItems() {
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
if (isMarkJoin() || joinType.isNullAwareLeftAntiJoin()
|| joinType.isFullOuterJoin()
@@ -710,4 +639,61 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan,
RIGHT_CHILD_TYPE extends
logicalJoin.put("Properties", properties);
return logicalJoin;
}
+
+ @Override
+ public void computeUnique(Builder fdBuilder) {
+ if (isMarkJoin()) {
+ // TODO disable function dependence calculation for mark join, but
need re-think this in future.
+ return;
+ }
+ if (joinType.isLeftSemiOrAntiJoin()) {
+
fdBuilder.addUniqueSlot(left().getLogicalProperties().getFunctionalDependencies());
+ } else if (joinType.isRightSemiOrAntiJoin()) {
+
fdBuilder.addUniqueSlot(right().getLogicalProperties().getFunctionalDependencies());
+ }
+ // if there is non-equal join conditions, don't propagate unique
+ if (hashJoinConjuncts.isEmpty()) {
+ return;
+ }
+ Pair<Set<Slot>, Set<Slot>> keys = extractNullRejectHashKeys();
+ if (keys == null) {
+ return;
+ }
+
+ // Note here we only check whether the left is unique.
+ // So the hash condition can't be null-safe
+ // TODO: consider Null-safe hash condition when left and rigth is not
nullable
+ boolean isLeftUnique = left().getLogicalProperties()
+ .getFunctionalDependencies().isUnique(keys.first);
+ boolean isRightUnique = right().getLogicalProperties()
+ .getFunctionalDependencies().isUnique(keys.second);
+
+ // left/right outer join propagate left/right uniforms slots
+ // And if the right/left hash keys is unique,
+ // join can propagate left/right functional dependencies
+ if (joinType.isLeftOuterJoin() && isRightUnique) {
+
fdBuilder.addUniqueSlot(left().getLogicalProperties().getFunctionalDependencies());
+ } else if (joinType.isRightOuterJoin() && isLeftUnique) {
+
fdBuilder.addUniqueSlot(right().getLogicalProperties().getFunctionalDependencies());
+ } else if (joinType.isInnerJoin() && isLeftUnique && isRightUnique) {
+ // inner join propagate uniforms slots
+ // And if the hash keys is unique, inner join can propagate all
functional dependencies
+
fdBuilder.addFunctionalDependencies(left().getLogicalProperties().getFunctionalDependencies());
+
fdBuilder.addFunctionalDependencies(right().getLogicalProperties().getFunctionalDependencies());
+ }
+ }
+
+ @Override
+ public void computeUniform(Builder fdBuilder) {
+ if (isMarkJoin()) {
+ // TODO disable function dependence calculation for mark join, but
need re-think this in future.
+ return;
+ }
+ if (!joinType.isLeftSemiOrAntiJoin()) {
+
fdBuilder.addUniformSlot(right().getLogicalProperties().getFunctionalDependencies());
+ }
+ if (!joinType.isRightSemiOrAntiJoin()) {
+
fdBuilder.addUniformSlot(left().getLogicalProperties().getFunctionalDependencies());
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java
index 482c7e26058..02558fe2ed2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java
@@ -22,7 +22,6 @@ import org.apache.doris.nereids.properties.ExprFdItem;
import org.apache.doris.nereids.properties.FdFactory;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
-import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
@@ -41,7 +40,6 @@ import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
-import java.util.function.Supplier;
/**
* Logical limit plan
@@ -150,25 +148,29 @@ public class LogicalLimit<CHILD_TYPE extends Plan>
extends LogicalUnary<CHILD_TY
}
@Override
- public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>>
outputSupplier) {
- FunctionalDependencies fd =
child(0).getLogicalProperties().getFunctionalDependencies();
- if (getLimit() == 1 && !phase.isLocal()) {
- Builder builder = new Builder();
- outputSupplier.get().forEach(builder::addUniformSlot);
- outputSupplier.get().forEach(builder::addUniqueSlot);
- ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
- builder.addFdItems(fdItems);
- fd = builder.build();
+ public void computeUnique(FunctionalDependencies.Builder fdBuilder) {
+ if (getLimit() == 1) {
+ getOutput().forEach(fdBuilder::addUniqueSlot);
+ } else {
+
fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies());
+ }
+ }
+
+ @Override
+ public void computeUniform(FunctionalDependencies.Builder fdBuilder) {
+ if (getLimit() == 1) {
+ getOutput().forEach(fdBuilder::addUniformSlot);
+ } else {
+
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
- return fd;
}
@Override
- public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>>
outputSupplier) {
+ public ImmutableSet<FdItem> computeFdItems() {
ImmutableSet<FdItem> fdItems =
child(0).getLogicalProperties().getFunctionalDependencies().getFdItems();
if (getLimit() == 1 && !phase.isLocal()) {
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
- List<Slot> output = outputSupplier.get();
+ List<Slot> output = getOutput();
ImmutableSet<SlotReference> slotSet = output.stream()
.filter(SlotReference.class::isInstance)
.map(SlotReference.class::cast)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java
index d338be5cc88..78a1e9a3a5e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java
@@ -41,7 +41,6 @@ import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
-import java.util.function.Supplier;
/**
* A relation that contains only one row consist of some constant expressions.
@@ -136,20 +135,18 @@ public class LogicalOneRowRelation extends
LogicalRelation implements OneRowRela
}
@Override
- public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>>
outputSupplier) {
- FunctionalDependencies.Builder builder = new
FunctionalDependencies.Builder();
- outputSupplier.get().forEach(s -> {
- builder.addUniformSlot(s);
- builder.addUniqueSlot(s);
- });
- ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
- builder.addFdItems(fdItems);
- return builder.build();
+ public void computeUnique(FunctionalDependencies.Builder fdBuilder) {
+ getOutput().forEach(fdBuilder::addUniqueSlot);
+ }
+
+ @Override
+ public void computeUniform(FunctionalDependencies.Builder fdBuilder) {
+ getOutput().forEach(fdBuilder::addUniformSlot);
}
@Override
- public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>>
outputSupplier) {
- Set<NamedExpression> output =
ImmutableSet.copyOf(outputSupplier.get());
+ public ImmutableSet<FdItem> computeFdItems() {
+ Set<NamedExpression> output = ImmutableSet.copyOf(getOutput());
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
ImmutableSet<SlotReference> slotSet = output.stream()
.filter(SlotReference.class::isInstance)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java
index 860bd263acb..c0fb6ae7365 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java
@@ -19,13 +19,11 @@ package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
-import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
-import java.util.List;
import java.util.Optional;
import java.util.function.BiFunction;
import java.util.function.Supplier;
@@ -61,7 +59,18 @@ public interface LogicalPlan extends Plan {
* - BlockFDPropagation: clean the fd
* - PropagateFD: propagate the fd
*/
- FunctionalDependencies computeFuncDeps(Supplier<List<Slot>>
outputSupplier);
+ default FunctionalDependencies computeFuncDeps() {
+ FunctionalDependencies.Builder fdBuilder = new
FunctionalDependencies.Builder();
+ computeUniform(fdBuilder);
+ computeUnique(fdBuilder);
+ ImmutableSet<FdItem> fdItems = computeFdItems();
+ fdBuilder.addFdItems(fdItems);
+ return fdBuilder.build();
+ }
+
+ ImmutableSet<FdItem> computeFdItems();
+
+ void computeUnique(FunctionalDependencies.Builder fdBuilder);
- ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>> outputSupplier);
+ void computeUniform(FunctionalDependencies.Builder fdBuilder);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java
index 89dd7d49677..ebecee0d3ae 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java
@@ -23,7 +23,6 @@ import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.LogicalProperties;
-import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.BoundStar;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -42,11 +41,9 @@ import com.google.common.collect.ImmutableList.Builder;
import com.google.common.collect.ImmutableSet;
import org.json.JSONObject;
-import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
-import java.util.function.Supplier;
/**
* Logical project plan.
@@ -239,36 +236,50 @@ public class LogicalProject<CHILD_TYPE extends Plan>
extends LogicalUnary<CHILD_
}
@Override
- public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>>
outputSupplier) {
- FunctionalDependencies childFuncDeps =
child().getLogicalProperties().getFunctionalDependencies();
- FunctionalDependencies.Builder builder = new
FunctionalDependencies.Builder(childFuncDeps);
- builder.pruneSlots(new HashSet<>(outputSupplier.get()));
- projects.stream().filter(Alias.class::isInstance).forEach(proj -> {
- if (proj.child(0).isConstant()) {
- builder.addUniformSlot(proj.toSlot());
- } else if (proj.child(0) instanceof Uuid) {
- builder.addUniqueSlot(proj.toSlot());
+ public ImmutableSet<FdItem> computeFdItems() {
+ ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
+
+ ImmutableSet<FdItem> childItems =
child().getLogicalProperties().getFunctionalDependencies().getFdItems();
+ builder.addAll(childItems);
+
+ return builder.build();
+ }
+
+ @Override
+ public void computeUnique(FunctionalDependencies.Builder fdBuilder) {
+
fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies());
+ for (NamedExpression proj : getProjects()) {
+ if (proj.children().isEmpty()) {
+ continue;
+ }
+ if (proj.child(0) instanceof Uuid) {
+ fdBuilder.addUniqueSlot(proj.toSlot());
} else if (ExpressionUtils.isInjective(proj.child(0))) {
ImmutableSet<Slot> inputs =
ImmutableSet.copyOf(proj.getInputSlots());
- if (childFuncDeps.isUnique(inputs)) {
- builder.addUniqueSlot(proj.toSlot());
- } else if (childFuncDeps.isUniform(inputs)) {
- builder.addUniformSlot(proj.toSlot());
+ if
(child(0).getLogicalProperties().getFunctionalDependencies().isUnique(inputs)) {
+ fdBuilder.addUniqueSlot(proj.toSlot());
}
}
- });
- ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
- builder.addFdItems(fdItems);
- return builder.build();
+ }
+ fdBuilder.pruneSlots(getOutputSet());
}
@Override
- public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>>
outputSupplier) {
- ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
-
- ImmutableSet<FdItem> childItems =
child().getLogicalProperties().getFunctionalDependencies().getFdItems();
- builder.addAll(childItems);
-
- return builder.build();
+ public void computeUniform(FunctionalDependencies.Builder fdBuilder) {
+
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
+ for (NamedExpression proj : getProjects()) {
+ if (proj.children().isEmpty()) {
+ continue;
+ }
+ if (proj.child(0).isConstant()) {
+ fdBuilder.addUniformSlot(proj.toSlot());
+ } else if (ExpressionUtils.isInjective(proj.child(0))) {
+ ImmutableSet<Slot> inputs =
ImmutableSet.copyOf(proj.getInputSlots());
+ if
(child(0).getLogicalProperties().getFunctionalDependencies().isUniform(inputs))
{
+ fdBuilder.addUniformSlot(proj.toSlot());
+ }
+ }
+ }
+ fdBuilder.pruneSlots(getOutputSet());
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java
index 5e0311afe1b..95ae92b6866 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java
@@ -39,7 +39,6 @@ import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
-import java.util.function.Supplier;
/**
* LogicalRepeat.
@@ -187,19 +186,17 @@ public class LogicalRepeat<CHILD_TYPE extends Plan>
extends LogicalUnary<CHILD_T
}
@Override
- public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>>
outputSupplier) {
- FunctionalDependencies.Builder builder = new
FunctionalDependencies.Builder();
- // Note uniform does not reject nullable slots
- outputSupplier.get().stream()
-
.filter(child(0).getLogicalProperties().getFunctionalDependencies()::isUniform)
- .forEach(builder::addUniformSlot);
- ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
- builder.addFdItems(fdItems);
- return builder.build();
+ public void computeUnique(FunctionalDependencies.Builder fdBuilder) {
+ // don't generate unique slot
+ }
+
+ @Override
+ public void computeUniform(FunctionalDependencies.Builder fdBuilder) {
+
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
@Override
- public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>>
outputSupplier) {
+ public ImmutableSet<FdItem> computeFdItems() {
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
ImmutableSet<FdItem> childItems =
child().getLogicalProperties().getFunctionalDependencies().getFdItems();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java
index 68a451f1df1..ea9fd143c94 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java
@@ -41,7 +41,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
-import java.util.function.Supplier;
/**
* The node of logical plan for sub query and alias
@@ -165,22 +164,29 @@ public class LogicalSubQueryAlias<CHILD_TYPE extends
Plan> extends LogicalUnary<
}
@Override
- public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>>
outputSupplier) {
- FunctionalDependencies.Builder builder = new FunctionalDependencies
-
.Builder(child(0).getLogicalProperties().getFunctionalDependencies());
+ public void computeUnique(FunctionalDependencies.Builder fdBuilder) {
+
fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies());
Map<Slot, Slot> replaceMap = new HashMap<>();
- List<Slot> outputs = outputSupplier.get();
+ List<Slot> outputs = getOutput();
for (int i = 0; i < outputs.size(); i++) {
replaceMap.put(child(0).getOutput().get(i), outputs.get(i));
}
- builder.replace(replaceMap);
- ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
- builder.addFdItems(fdItems);
- return builder.build();
+ fdBuilder.replace(replaceMap);
}
@Override
- public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>>
outputSupplier) {
+ public void computeUniform(FunctionalDependencies.Builder fdBuilder) {
+
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
+ Map<Slot, Slot> replaceMap = new HashMap<>();
+ List<Slot> outputs = getOutput();
+ for (int i = 0; i < outputs.size(); i++) {
+ replaceMap.put(child(0).getOutput().get(i), outputs.get(i));
+ }
+ fdBuilder.replace(replaceMap);
+ }
+
+ @Override
+ public ImmutableSet<FdItem> computeFdItems() {
// TODO: inherit from child with replaceMap
return ImmutableSet.of();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java
index 1fb5dbaab72..791ded58cce 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java
@@ -22,7 +22,6 @@ import org.apache.doris.nereids.properties.ExprFdItem;
import org.apache.doris.nereids.properties.FdFactory;
import org.apache.doris.nereids.properties.FdItem;
import org.apache.doris.nereids.properties.FunctionalDependencies;
-import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.OrderKey;
import org.apache.doris.nereids.trees.expressions.Expression;
@@ -35,6 +34,7 @@ import
org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import com.google.common.base.Preconditions;
+import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
@@ -42,7 +42,6 @@ import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
-import java.util.function.Supplier;
/**
* Logical top-N plan.
@@ -171,26 +170,29 @@ public class LogicalTopN<CHILD_TYPE extends Plan> extends
LogicalUnary<CHILD_TYP
}
@Override
- public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>>
outputSupplier) {
- FunctionalDependencies fd =
child(0).getLogicalProperties().getFunctionalDependencies();
+ public void computeUnique(FunctionalDependencies.Builder fdBuilder) {
if (getLimit() == 1) {
- Builder builder = new Builder();
- List<Slot> output = outputSupplier.get();
- output.forEach(builder::addUniformSlot);
- output.forEach(builder::addUniqueSlot);
- ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
- builder.addFdItems(fdItems);
- fd = builder.build();
+ getOutput().forEach(fdBuilder::addUniqueSlot);
+ } else {
+
fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
- return fd;
}
@Override
- public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>>
outputSupplier) {
+ public void computeUniform(FunctionalDependencies.Builder fdBuilder) {
+ if (getLimit() == 1) {
+ getOutput().forEach(fdBuilder::addUniformSlot);
+ } else {
+
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
+ }
+ }
+
+ @Override
+ public ImmutableSet<FdItem> computeFdItems() {
ImmutableSet<FdItem> fdItems =
child(0).getLogicalProperties().getFunctionalDependencies().getFdItems();
if (getLimit() == 1) {
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
- List<Slot> output = outputSupplier.get();
+ List<Slot> output = getOutput();
ImmutableSet<SlotReference> slotSet = output.stream()
.filter(SlotReference.class::isInstance)
.map(SlotReference.class::cast)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java
index dac6996c0ca..57d944bc2b9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java
@@ -25,7 +25,6 @@ import
org.apache.doris.nereids.properties.FunctionalDependencies;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
-import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
@@ -41,7 +40,6 @@ import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
-import java.util.function.Supplier;
/**
* Logical Union.
@@ -187,20 +185,20 @@ public class LogicalUnion extends LogicalSetOperation
implements Union, OutputPr
}
@Override
- public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>>
outputSupplier) {
- if (qualifier != Qualifier.DISTINCT) {
- return FunctionalDependencies.EMPTY_FUNC_DEPS;
+ public void computeUnique(FunctionalDependencies.Builder fdBuilder) {
+ if (qualifier == Qualifier.DISTINCT) {
+ fdBuilder.addUniqueSlot(ImmutableSet.copyOf(getOutput()));
}
- FunctionalDependencies.Builder builder = new
FunctionalDependencies.Builder();
- builder.addUniqueSlot(ImmutableSet.copyOf(outputSupplier.get()));
- ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
- builder.addFdItems(fdItems);
- return builder.build();
}
@Override
- public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>>
outputSupplier) {
- Set<NamedExpression> output =
ImmutableSet.copyOf(outputSupplier.get());
+ public void computeUniform(FunctionalDependencies.Builder fdBuilder) {
+ // don't propagate uniform slots
+ }
+
+ @Override
+ public ImmutableSet<FdItem> computeFdItems() {
+ Set<NamedExpression> output = ImmutableSet.copyOf(getOutput());
ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
ImmutableSet<SlotReference> exprs = output.stream()
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java
index e8621b1754c..200ccc2ffcb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java
@@ -21,7 +21,7 @@ import org.apache.doris.catalog.View;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdItem;
-import org.apache.doris.nereids.properties.FunctionalDependencies;
+import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
@@ -36,7 +36,6 @@ import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
-import java.util.function.Supplier;
/** LogicalView */
public class LogicalView<BODY extends Plan> extends LogicalUnary<BODY> {
@@ -129,17 +128,22 @@ public class LogicalView<BODY extends Plan> extends
LogicalUnary<BODY> {
}
@Override
- public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>>
outputSupplier) {
- return ((LogicalPlan) child()).computeFuncDeps(outputSupplier);
+ public ImmutableSet<FdItem> computeFdItems() {
+ return ((LogicalPlan) child()).computeFdItems();
}
@Override
- public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>>
outputSupplier) {
- return ((LogicalPlan) child()).computeFdItems(outputSupplier);
+ public Plan withChildren(List<Plan> children) {
+ return new LogicalView<>(view, (LogicalPlan) children.get(0));
}
@Override
- public Plan withChildren(List<Plan> children) {
- return new LogicalView<>(view, (LogicalPlan) children.get(0));
+ public void computeUnique(Builder fdBuilder) {
+
fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies());
+ }
+
+ @Override
+ public void computeUniform(Builder fdBuilder) {
+
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java
index 457678787bd..b03a3365402 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java
@@ -19,7 +19,7 @@ package org.apache.doris.nereids.trees.plans.logical;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.FdItem;
-import org.apache.doris.nereids.properties.FunctionalDependencies;
+import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -43,7 +43,6 @@ import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
-import java.util.function.Supplier;
/**
* logical node to deal with window functions;
@@ -228,35 +227,51 @@ public class LogicalWindow<CHILD_TYPE extends Plan>
extends LogicalUnary<CHILD_T
return Optional.ofNullable(window);
}
- private void updateFuncDepsByWindowExpr(NamedExpression namedExpression,
FunctionalDependencies.Builder builder) {
+ private boolean isUnique(NamedExpression namedExpression) {
if (namedExpression.children().size() != 1 ||
!(namedExpression.child(0) instanceof WindowExpression)) {
- return;
+ return false;
}
WindowExpression windowExpr = (WindowExpression)
namedExpression.child(0);
List<Expression> partitionKeys = windowExpr.getPartitionKeys();
// Now we only support slot type keys
if (!partitionKeys.stream().allMatch(Slot.class::isInstance)) {
- return;
+ return false;
}
ImmutableSet<Slot> slotSet = partitionKeys.stream()
.map(s -> (Slot) s)
.collect(ImmutableSet.toImmutableSet());
+ // if partition by keys are uniform or empty, output is unique
+ if
(child(0).getLogicalProperties().getFunctionalDependencies().isUniformAndNotNull(slotSet)
+ || slotSet.isEmpty()) {
+ if (windowExpr.getFunction() instanceof RowNumber) {
+ return true;
+ }
+ }
+ return false;
+ }
+ private boolean isUniform(NamedExpression namedExpression) {
+ if (namedExpression.children().size() != 1 ||
!(namedExpression.child(0) instanceof WindowExpression)) {
+ return false;
+ }
+ WindowExpression windowExpr = (WindowExpression)
namedExpression.child(0);
+ List<Expression> partitionKeys = windowExpr.getPartitionKeys();
+ // Now we only support slot type keys
+ if (!partitionKeys.stream().allMatch(Slot.class::isInstance)) {
+ return false;
+ }
+ ImmutableSet<Slot> slotSet = partitionKeys.stream()
+ .map(s -> (Slot) s)
+ .collect(ImmutableSet.toImmutableSet());
// if partition by keys are unique, output is uniform
if
(child(0).getLogicalProperties().getFunctionalDependencies().isUniqueAndNotNull(slotSet))
{
if (windowExpr.getFunction() instanceof RowNumber
|| windowExpr.getFunction() instanceof Rank
|| windowExpr.getFunction() instanceof DenseRank) {
- builder.addUniformSlot(namedExpression.toSlot());
- }
- }
-
- // if partition by keys are uniform, output is unique
- if
(child(0).getLogicalProperties().getFunctionalDependencies().isUniformAndNotNull(slotSet))
{
- if (windowExpr.getFunction() instanceof RowNumber) {
- builder.addUniqueSlot(namedExpression.toSlot());
+ return true;
}
}
+ return false;
}
private void updateFuncDepsByWindowExpr(NamedExpression namedExpression,
ImmutableSet.Builder<FdItem> builder) {
@@ -278,27 +293,35 @@ public class LogicalWindow<CHILD_TYPE extends Plan>
extends LogicalUnary<CHILD_T
}
@Override
- public FunctionalDependencies computeFuncDeps(Supplier<List<Slot>>
outputSupplier) {
- FunctionalDependencies.Builder builder = new
FunctionalDependencies.Builder(
- child(0).getLogicalProperties().getFunctionalDependencies());
+ public ImmutableSet<FdItem> computeFdItems() {
+ ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
+ ImmutableSet<FdItem> childItems =
child().getLogicalProperties().getFunctionalDependencies().getFdItems();
+ builder.addAll(childItems);
+
for (NamedExpression namedExpression : windowExpressions) {
updateFuncDepsByWindowExpr(namedExpression, builder);
}
- ImmutableSet<FdItem> fdItems = computeFdItems(outputSupplier);
- builder.addFdItems(fdItems);
+
return builder.build();
}
@Override
- public ImmutableSet<FdItem> computeFdItems(Supplier<List<Slot>>
outputSupplier) {
- ImmutableSet.Builder<FdItem> builder = ImmutableSet.builder();
- ImmutableSet<FdItem> childItems =
child().getLogicalProperties().getFunctionalDependencies().getFdItems();
- builder.addAll(childItems);
-
+ public void computeUnique(Builder fdBuilder) {
+
fdBuilder.addUniqueSlot(child(0).getLogicalProperties().getFunctionalDependencies());
for (NamedExpression namedExpression : windowExpressions) {
- updateFuncDepsByWindowExpr(namedExpression, builder);
+ if (isUnique(namedExpression)) {
+ fdBuilder.addUniqueSlot(namedExpression.toSlot());
+ }
}
+ }
- return builder.build();
+ @Override
+ public void computeUniform(Builder fdBuilder) {
+
fdBuilder.addUniformSlot(child(0).getLogicalProperties().getFunctionalDependencies());
+ for (NamedExpression namedExpression : windowExpressions) {
+ if (isUniform(namedExpression)) {
+ fdBuilder.addUniformSlot(namedExpression.toSlot());
+ }
+ }
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
index 64685bc55e3..401b8e0736b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
@@ -41,7 +41,6 @@ import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.WindowExpression;
-import
org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
import org.apache.doris.nereids.trees.expressions.functions.agg.Avg;
import org.apache.doris.nereids.trees.expressions.functions.agg.Max;
import org.apache.doris.nereids.trees.expressions.functions.agg.Min;
@@ -738,7 +737,8 @@ public class ExpressionUtils {
return expression instanceof Slot;
}
- public static boolean isInjectiveAgg(AggregateFunction agg) {
+ // if the input is unique, the output of agg is unique, too
+ public static boolean isInjectiveAgg(Expression agg) {
return agg instanceof Sum || agg instanceof Avg || agg instanceof Max
|| agg instanceof Min;
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/FunctionalDependenciesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/FunctionalDependenciesTest.java
index 5c4193d83a3..dce123cbff0 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/FunctionalDependenciesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/FunctionalDependenciesTest.java
@@ -258,12 +258,12 @@ class FunctionalDependenciesTest extends
TestWithFeService {
@Test
void testWindow() {
Plan plan = PlanChecker.from(connectContext)
- .analyze("select row_number() over(partition by id) from agg")
+ .analyze("select id, row_number() over(partition by id) from
agg")
.rewrite()
.getPlan();
System.out.println(plan.getLogicalProperties().getFunctionalDependencies());
Assertions.assertTrue(plan.getLogicalProperties()
-
.getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(0)));
+
.getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(1)));
plan = PlanChecker.from(connectContext)
.analyze("select row_number() over(partition by name) from agg
where name = '1'")
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/UniformTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/UniformTest.java
new file mode 100644
index 00000000000..edd5d1e4f0f
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/UniformTest.java
@@ -0,0 +1,207 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.properties;
+
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.types.IntegerType;
+import org.apache.doris.nereids.util.PlanChecker;
+import org.apache.doris.utframe.TestWithFeService;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class UniformTest extends TestWithFeService {
+ Slot slot1 = new SlotReference("1", IntegerType.INSTANCE, false);
+ Slot slot2 = new SlotReference("2", IntegerType.INSTANCE, false);
+ Slot slot3 = new SlotReference("1", IntegerType.INSTANCE, false);
+ Slot slot4 = new SlotReference("1", IntegerType.INSTANCE, false);
+
+ @Override
+ protected void runBeforeAll() throws Exception {
+ createDatabase("test");
+ createTable("create table test.agg (\n"
+ + "id int not null,\n"
+ + "name varchar(128) replace not null )\n"
+ + "AGGREGATE KEY(id)\n"
+ + "distributed by hash(id) buckets 10\n"
+ + "properties('replication_num' = '1');");
+ createTable("create table test.uni (\n"
+ + "id int not null,\n"
+ + "name varchar(128) not null)\n"
+ + "UNIQUE KEY(id)\n"
+ + "distributed by hash(id) buckets 10\n"
+ + "properties('replication_num' = '1');");
+ connectContext.setDatabase("test");
+ }
+
+ @Test
+ void testAgg() {
+ // group by unique
+ Plan plan = PlanChecker.from(connectContext)
+ .analyze("select count(id) from agg group by id")
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUniform(plan.getOutput().get(0)));
+
+ // propagate uniform
+ plan = PlanChecker.from(connectContext)
+ .analyze("select id from agg where id = 1 group by id ")
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUniform(plan.getOutput().get(0)));
+
+ // group by all/uniform
+ plan = PlanChecker.from(connectContext)
+ .analyze("select sum(id) from agg ")
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUniform(plan.getOutput().get(0)));
+ plan = PlanChecker.from(connectContext)
+ .analyze("select sum(id) from agg where id = 1 group by id")
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUniform(plan.getOutput().get(0)));
+
+ }
+
+ @Test
+ void testTopNLimit() {
+ Plan plan = PlanChecker.from(connectContext)
+ .analyze("select name from agg limit 1")
+ .rewrite()
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUniformAndNotNull(plan.getOutput().get(0)));
+ plan = PlanChecker.from(connectContext)
+ .analyze("select name from agg order by name limit 1 ")
+ .rewrite()
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUniformAndNotNull(plan.getOutput().get(0)));
+ }
+
+ @Test
+ void testSetOp() {
+ Plan plan = PlanChecker.from(connectContext)
+ .analyze("select name from agg limit 1 except select name from
agg")
+ .rewrite()
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUniform(plan.getOutput().get(0)));
+ plan = PlanChecker.from(connectContext)
+ .analyze("select id from agg intersect select name from agg
limit 1")
+ .rewrite()
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUniform(plan.getOutput().get(0)));
+ }
+
+ @Test
+ void testFilterHaving() {
+ Plan plan = PlanChecker.from(connectContext)
+ .analyze("select id from agg where id = 1")
+ .rewrite()
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUniformAndNotNull(plan.getOutput().get(0)));
+ plan = PlanChecker.from(connectContext)
+ .analyze("select name from uni group by name having name =
\"\"")
+ .rewrite()
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUniformAndNotNull(plan.getOutput().get(0)));
+ }
+
+ @Test
+ void testGenerate() {
+ Plan plan = PlanChecker.from(connectContext)
+ .analyze("select id from agg lateral view explode([1,2,3])
tmp1 as e1")
+ .rewrite()
+ .getPlan();
+ Assertions.assertTrue(plan.getLogicalProperties()
+ .getFunctionalDependencies().isEmpty());
+ }
+
+ @Test
+ void testJoin() {
+ // foj doesn't propagate any
+ Plan plan = PlanChecker.from(connectContext)
+ .analyze("select uni.id from agg join uni "
+ + "on agg.id = uni.id where uni.id = 1")
+ .rewrite()
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies().isUniform(plan.getOutput().get(0)));
+ }
+
+ @Test
+ void testOneRowRelation() {
+ Plan plan = PlanChecker.from(connectContext)
+ .analyze("select 1")
+ .rewrite()
+ .getPlan();
+ Assertions.assertTrue(plan.getLogicalProperties()
+
.getFunctionalDependencies().isUniform(plan.getOutput().get(0)));
+ }
+
+ @Test
+ void testProject() {
+ Plan plan = PlanChecker.from(connectContext)
+ .analyze("select id as id2, 1 as id3 from uni where id = 1")
+ .rewrite()
+ .getPlan();
+ Assertions.assertTrue(plan.getLogicalProperties()
+
.getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(0)));
+ Assertions.assertTrue(plan.getLogicalProperties()
+
.getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(1)));
+ }
+
+ @Test
+ void testSubQuery() {
+ Plan plan = PlanChecker.from(connectContext)
+ .analyze("select id from (select id from agg where id = 1) t")
+ .getPlan();
+ Assertions.assertTrue(plan.getLogicalProperties()
+
.getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(0)));
+ }
+
+ @Test
+ void testWindow() {
+ // partition by uniform
+ Plan plan = PlanChecker.from(connectContext)
+ .analyze("select row_number() over(partition by id) from agg")
+ .rewrite()
+ .getPlan();
+ Assertions.assertTrue(plan.getLogicalProperties()
+
.getFunctionalDependencies().isUniform(plan.getOutput().get(0)));
+ plan = PlanChecker.from(connectContext)
+ .analyze("select rank() over(partition by id) from agg")
+ .rewrite()
+ .getPlan();
+ Assertions.assertTrue(plan.getLogicalProperties()
+
.getFunctionalDependencies().isUniform(plan.getOutput().get(0)));
+ plan = PlanChecker.from(connectContext)
+ .analyze("select dense_rank() over(partition by id) from agg")
+ .rewrite()
+ .getPlan();
+ Assertions.assertTrue(plan.getLogicalProperties()
+
.getFunctionalDependencies().isUniform(plan.getOutput().get(0)));
+ }
+
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/FunctionalDependenciesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/UniqueTest.java
similarity index 50%
copy from
fe/fe-core/src/test/java/org/apache/doris/nereids/properties/FunctionalDependenciesTest.java
copy to
fe/fe-core/src/test/java/org/apache/doris/nereids/properties/UniqueTest.java
index 5c4193d83a3..27d64ad186c 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/FunctionalDependenciesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/UniqueTest.java
@@ -17,21 +17,17 @@
package org.apache.doris.nereids.properties;
-import org.apache.doris.nereids.properties.FunctionalDependencies.Builder;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
import org.apache.doris.nereids.types.IntegerType;
import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.utframe.TestWithFeService;
-import com.google.common.collect.ImmutableSet;
import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
-class FunctionalDependenciesTest extends TestWithFeService {
+class UniqueTest extends TestWithFeService {
Slot slot1 = new SlotReference("1", IntegerType.INSTANCE, false);
Slot slot2 = new SlotReference("2", IntegerType.INSTANCE, false);
Slot slot3 = new SlotReference("1", IntegerType.INSTANCE, false);
@@ -56,83 +52,146 @@ class FunctionalDependenciesTest extends TestWithFeService
{
}
@Test
- void testUniform() {
- Builder fdBuilder = new Builder();
- fdBuilder.addUniformSlot(slot1);
- FunctionalDependencies fd = fdBuilder.build();
- Assertions.assertTrue(fd.isUniformAndNotNull(slot1));
- Assertions.assertFalse(fd.isUniformAndNotNull(slot2));
- fdBuilder.addUniformSlot(ImmutableSet.of(slot2));
- fd = fdBuilder.build();
- Assertions.assertTrue(fd.isUniformAndNotNull(slot2));
- ImmutableSet<Slot> slotSet = ImmutableSet.of(slot1, slot2, slot3);
- fdBuilder.addUniformSlot(slotSet);
- fd = fdBuilder.build();
- Assertions.assertTrue(fd.isUniformAndNotNull(slotSet));
- Assertions.assertFalse(fd.isUniformAndNotNull(ImmutableSet.of(slot1,
slot2, slot3, slot4)));
- Assertions.assertTrue(fd.isUniformAndNotNull(ImmutableSet.of(slot1,
slot2)));
- Assertions.assertFalse(fd.isUniformAndNotNull(ImmutableSet.of(slot3,
slot2)));
+ void testAgg() {
+ Plan plan = PlanChecker.from(connectContext)
+ .analyze("select name from agg group by name")
+ .rewrite()
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUniqueAndNotNull(plan.getOutput().get(0)));
+ plan = PlanChecker.from(connectContext)
+ .analyze("select sum(id) from agg")
+ .rewrite()
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUnique(plan.getOutput().get(0)));
+ plan = PlanChecker.from(connectContext)
+ .analyze("select id, sum(id), avg(id), max(id), min(id) from
agg group by id")
+ .rewrite()
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUnique(plan.getOutput().get(0)));
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUnique(plan.getOutput().get(1)));
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUnique(plan.getOutput().get(2)));
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUnique(plan.getOutput().get(3)));
+
}
@Test
- void testUnique() {
- Builder fdBuilder = new Builder();
- fdBuilder.addUniqueSlot(slot1);
- FunctionalDependencies fd = fdBuilder.build();
- Assertions.assertTrue(fd.isUniqueAndNotNull(slot1));
- Assertions.assertFalse(fd.isUniqueAndNotNull(slot2));
- fdBuilder.addUniqueSlot(slot2);
- fd = fdBuilder.build();
- Assertions.assertTrue(fd.isUniqueAndNotNull(slot2));
- ImmutableSet<Slot> slotSet = ImmutableSet.of(slot1, slot2, slot3);
- fdBuilder.addUniqueSlot(slotSet);
- fd = fdBuilder.build();
- Assertions.assertTrue(fd.isUniqueAndNotNull(slotSet));
- Assertions.assertTrue(fd.isUniqueAndNotNull(ImmutableSet.of(slot1,
slot2, slot3, slot4)));
- Assertions.assertFalse(fd.isUniqueAndNotNull(ImmutableSet.of(slot3,
slot4)));
+ void testScan() throws Exception {
+ // test agg key
+ Plan plan = PlanChecker.from(connectContext)
+ .analyze("select id from agg")
+ .rewrite()
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUniqueAndNotNull(plan.getOutput().get(0)));
+
+ // test unique key
+ plan = PlanChecker.from(connectContext)
+ .analyze("select id from uni")
+ .rewrite()
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUniqueAndNotNull(plan.getOutput().get(0)));
+
+ // test unique constraint
+ addConstraint("alter table agg add constraint uq unique(name)");
+ plan = PlanChecker.from(connectContext)
+ .analyze("select name from agg")
+ .rewrite()
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUniqueAndNotNull(plan.getOutput().get(0)));
+ dropConstraint("alter table agg drop constraint uq");
+
+ // test primary constraint
+ addConstraint("alter table agg add constraint pk primary key(name)");
+ plan = PlanChecker.from(connectContext)
+ .analyze("select name from agg")
+ .rewrite()
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUniqueAndNotNull(plan.getOutput().get(0)));
+ dropConstraint("alter table agg drop constraint pk");
}
@Test
- void testMergeFD() {
- Builder fdBuilder1 = new Builder();
- fdBuilder1.addUniformSlot(slot1);
- Builder fdBuilder2 = new Builder();
- fdBuilder2.addUniformSlot(slot2);
-
- fdBuilder1.addFunctionalDependencies(fdBuilder2.build());
- FunctionalDependencies fd = fdBuilder1.build();
- Assertions.assertTrue(fd.isUniformAndNotNull(slot1));
- Assertions.assertTrue(fd.isUniformAndNotNull(slot2));
+ void testTopNLimit() {
+ Plan plan = PlanChecker.from(connectContext)
+ .analyze("select name from agg limit 1")
+ .rewrite()
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUniqueAndNotNull(plan.getOutput().get(0)));
+ plan = PlanChecker.from(connectContext)
+ .analyze("select name from agg order by name limit 1 ")
+ .rewrite()
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUniqueAndNotNull(plan.getOutput().get(0)));
}
@Test
- void testScan() {
+ void testSetOp() {
Plan plan = PlanChecker.from(connectContext)
- .analyze("select id from agg")
+ .analyze("select name from agg limit 1 except select name from
agg")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUniqueAndNotNull(plan.getOutput().get(0)));
plan = PlanChecker.from(connectContext)
- .analyze("select id from uni")
+ .analyze("select id from agg intersect select name from agg")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
.isUniqueAndNotNull(plan.getOutput().get(0)));
+ plan = PlanChecker.from(connectContext)
+ .analyze("select id from agg union all select name from agg")
+ .rewrite()
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isEmpty());
+ plan = PlanChecker.from(connectContext)
+ .analyze("select name, id from agg union select name, id from
agg")
+ .rewrite()
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUnique(plan.getOutputSet()));
}
@Test
- void testFilter() {
+ void testFilterHaving() {
Plan plan = PlanChecker.from(connectContext)
- .analyze("select name from agg where name = \"1\"")
+ .analyze("select id from agg where id = 1")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
- .isUniformAndNotNull(plan.getOutput().get(0)));
+ .isUniqueAndNotNull(plan.getOutput().get(0)));
+ plan = PlanChecker.from(connectContext)
+ .analyze("select name from uni group by name having name =
\"\"")
+ .rewrite()
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isUniqueAndNotNull(plan.getOutput().get(0)));
+ }
+
+ @Test
+ void testGenerate() {
+ Plan plan = PlanChecker.from(connectContext)
+ .analyze("select id from agg lateral view explode([1,2,3])
tmp1 as e1")
+ .rewrite()
+ .getPlan();
+ Assertions.assertTrue(plan.getLogicalProperties()
+ .getFunctionalDependencies().isEmpty());
}
@Test
void testJoin() {
+ // foj doesn't propagate any
Plan plan = PlanChecker.from(connectContext)
.analyze("select * from agg full outer join uni "
+ "on agg.id = uni.id")
@@ -140,248 +199,187 @@ class FunctionalDependenciesTest extends
TestWithFeService {
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies().isEmpty());
+ // loj propagate unique when right is unique
plan = PlanChecker.from(connectContext)
- .analyze("select agg.id, uni.id from agg full outer join uni "
- + "on agg.id = uni.id")
+ .analyze("select agg.id, uni.id from agg left outer join uni "
+ + "on agg.id = uni.name")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies().isEmpty());
-
plan = PlanChecker.from(connectContext)
- .analyze("select agg.id from agg left outer join uni "
+ .analyze("select agg.id, uni.id from agg left outer join uni "
+ "on agg.id = uni.id")
.rewrite()
.getPlan();
-
System.out.println(plan.getLogicalProperties().getFunctionalDependencies());
- Assertions.assertTrue(plan.getLogicalProperties()
-
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0)));
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies().isUnique(plan.getOutput().get(0)));
+
Assertions.assertFalse(plan.getLogicalProperties().getFunctionalDependencies().isUnique(plan.getOutput().get(1)));
+ // roj propagate unique when left is unique
plan = PlanChecker.from(connectContext)
- .analyze("select agg.id from agg left semi join uni "
+ .analyze("select agg.id, uni.id from agg right outer join uni "
+ + "on agg.id = uni.name")
+ .rewrite()
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies().isEmpty());
+ plan = PlanChecker.from(connectContext)
+ .analyze("select agg.id, uni.id from agg right outer join uni "
+ "on agg.id = uni.id")
.rewrite()
.getPlan();
-
System.out.println(plan.getLogicalProperties().getFunctionalDependencies());
- Assertions.assertTrue(plan.getLogicalProperties()
-
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0)));
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies().isUnique(plan.getOutput().get(1)));
+
Assertions.assertFalse(plan.getLogicalProperties().getFunctionalDependencies().isUnique(plan.getOutput().get(0)));
+ // semi/anti join propagate all
+ plan = PlanChecker.from(connectContext)
+ .analyze("select agg.id from agg left semi join uni "
+ + "on agg.id = uni.name")
+ .rewrite()
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies().isUnique(plan.getOutput().get(0)));
plan = PlanChecker.from(connectContext)
.analyze("select agg.id from agg left anti join uni "
- + "on agg.id = uni.id")
+ + "on agg.id = uni.name")
.rewrite()
.getPlan();
-
System.out.println(plan.getLogicalProperties().getFunctionalDependencies());
- Assertions.assertTrue(plan.getLogicalProperties()
-
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0)));
-
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies().isUnique(plan.getOutput().get(0)));
plan = PlanChecker.from(connectContext)
- .analyze("select uni.id from agg right outer join uni "
- + "on agg.id = uni.id")
+ .analyze("select uni.id from agg right semi join uni "
+ + "on agg.id = uni.name")
.rewrite()
.getPlan();
-
System.out.println(plan.getLogicalProperties().getFunctionalDependencies());
- Assertions.assertTrue(plan.getLogicalProperties()
-
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0)));
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies().isUnique(plan.getOutput().get(0)));
+ plan = PlanChecker.from(connectContext)
+ .analyze("select uni.id from agg right anti join uni "
+ + "on agg.id = uni.name")
+ .rewrite()
+ .getPlan();
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies().isUnique(plan.getOutput().get(0)));
+ // inner join propagate unique only when join key is unique
plan = PlanChecker.from(connectContext)
.analyze("select uni.id, agg.id from agg inner join uni "
+ "on agg.id = uni.id")
.rewrite()
.getPlan();
-
System.out.println(plan.getLogicalProperties().getFunctionalDependencies());
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0)));
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(1)));
- }
-
- @Test
- void testAgg() {
- Plan plan = PlanChecker.from(connectContext)
- .analyze("select count(1), name from agg group by name")
- .rewrite()
- .getPlan();
-
System.out.println(plan.getLogicalProperties().getFunctionalDependencies());
- Assertions.assertTrue(plan.getLogicalProperties()
-
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(1)));
plan = PlanChecker.from(connectContext)
- .analyze("select count(1), max(id), id from agg group by id")
+ .analyze("select uni.id, agg.id from agg inner join uni "
+ + "on agg.id = uni.name")
.rewrite()
.getPlan();
-
System.out.println(plan.getLogicalProperties().getFunctionalDependencies());
- Assertions.assertTrue(plan.getLogicalProperties()
-
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(1)));
Assertions.assertTrue(plan.getLogicalProperties()
-
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(2)));
+ .getFunctionalDependencies().isEmpty());
Assertions.assertTrue(plan.getLogicalProperties()
-
.getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(0)));
-
+ .getFunctionalDependencies().isEmpty());
plan = PlanChecker.from(connectContext)
- .analyze("select count(1), id from agg where id = 1 group by
id")
+ .analyze("select uni.id, agg.id from agg inner join uni "
+ + "on agg.id < uni.id")
.rewrite()
.getPlan();
-
System.out.println(plan.getLogicalProperties().getFunctionalDependencies());
Assertions.assertTrue(plan.getLogicalProperties()
-
.getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(1)));
+ .getFunctionalDependencies().isEmpty());
Assertions.assertTrue(plan.getLogicalProperties()
-
.getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(0)));
-
+ .getFunctionalDependencies().isEmpty());
plan = PlanChecker.from(connectContext)
- .analyze("select count(name) from agg")
+ .analyze("select uni.id, agg.id from agg inner join uni "
+ + "on agg.id = uni.id or agg.name > uni.name")
.rewrite()
.getPlan();
-
System.out.println(plan.getLogicalProperties().getFunctionalDependencies());
Assertions.assertTrue(plan.getLogicalProperties()
-
.getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(0)));
-
+ .getFunctionalDependencies().isEmpty());
+ Assertions.assertTrue(plan.getLogicalProperties()
+ .getFunctionalDependencies().isEmpty());
plan = PlanChecker.from(connectContext)
- .analyze("select id from agg where id = 1 group by cube(id,
name)")
+ .analyze("select uni.id, agg.id from agg inner join uni "
+ + "on agg.id = uni.id and agg.name > uni.name")
.rewrite()
.getPlan();
-
System.out.println(plan.getLogicalProperties().getFunctionalDependencies());
Assertions.assertTrue(plan.getLogicalProperties()
-
.getFunctionalDependencies().isUniform(plan.getOutput().get(0)));
- }
-
- @Test
- void testGenerate() {
- Plan plan = PlanChecker.from(connectContext)
- .analyze("select k1 from (select 1 k1) as t lateral view
explode([1,2,3]) tmp1 as e1")
- .rewrite()
- .getPlan();
-
System.out.println(plan.getLogicalProperties().getFunctionalDependencies());
+
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0)));
Assertions.assertTrue(plan.getLogicalProperties()
-
.getFunctionalDependencies().isUniform(plan.getOutput().get(0)));
+
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(1)));
}
@Test
- void testWindow() {
+ void testOneRowRelation() {
Plan plan = PlanChecker.from(connectContext)
- .analyze("select row_number() over(partition by id) from agg")
- .rewrite()
- .getPlan();
-
System.out.println(plan.getLogicalProperties().getFunctionalDependencies());
- Assertions.assertTrue(plan.getLogicalProperties()
-
.getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(0)));
-
- plan = PlanChecker.from(connectContext)
- .analyze("select row_number() over(partition by name) from agg
where name = '1'")
+ .analyze("select 1")
.rewrite()
.getPlan();
-
System.out.println(plan.getLogicalProperties().getFunctionalDependencies());
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0)));
-
- plan = PlanChecker.from(connectContext)
- .analyze("select row_number() over(partition by name) from agg
where name = '1' limit 1")
- .rewrite()
- .getPlan();
- LogicalPartitionTopN<?> ptopn = (LogicalPartitionTopN<?>)
plan.child(0).child(0).child(0).child(0).child(0);
-
System.out.println(ptopn.getLogicalProperties().getFunctionalDependencies());
- System.out.println(ptopn.getOutput());
- Assertions.assertTrue(ptopn.getLogicalProperties()
-
.getFunctionalDependencies().isUniformAndNotNull(ImmutableSet.copyOf(ptopn.getOutputSet())));
-
- plan = PlanChecker.from(connectContext)
- .analyze("select row_number() over(partition by name) from agg
limit 1")
- .rewrite()
- .getPlan();
- ptopn = (LogicalPartitionTopN<?>)
plan.child(0).child(0).child(0).child(0).child(0);
- Assertions.assertTrue(ptopn.getLogicalProperties()
- .getFunctionalDependencies().isEmpty());
}
@Test
void testProject() {
Plan plan = PlanChecker.from(connectContext)
- .analyze("select name from agg where id = 1")
+ .analyze("select id as id2 from uni")
.rewrite()
.getPlan();
Assertions.assertTrue(plan.getLogicalProperties()
- .getFunctionalDependencies().isEmpty());
+
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0)));
}
@Test
- void testLimitAndTopN() {
+ void testRepeat() {
Plan plan = PlanChecker.from(connectContext)
- .analyze("select name from agg limit 1")
+ .analyze("select id from agg group by GROUPING SETS ((id,
name), (id))")
.rewrite()
.getPlan();
-
System.out.println(plan.getLogicalProperties().getFunctionalDependencies());
- Assertions.assertTrue(plan.getLogicalProperties()
-
.getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(0)));
-
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isEmpty());
plan = PlanChecker.from(connectContext)
- .analyze("select name from agg order by id limit 1")
+ .analyze("select id from agg group by rollup (id, name)")
.rewrite()
.getPlan();
-
System.out.println(plan.getLogicalProperties().getFunctionalDependencies());
- Assertions.assertTrue(plan.getLogicalProperties()
-
.getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(0)));
+
Assertions.assertTrue(plan.getLogicalProperties().getFunctionalDependencies()
+ .isEmpty());
}
@Test
void testSubQuery() {
Plan plan = PlanChecker.from(connectContext)
- .analyze("select id from (select * from agg) t")
- .rewrite()
+ .analyze("select id from (select id from agg) t")
.getPlan();
-
System.out.println(plan.getLogicalProperties().getFunctionalDependencies());
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0)));
}
- @Disabled
@Test
- void testCTE() {
+ void testAssertRows() {
Plan plan = PlanChecker.from(connectContext)
- .analyze("with t as (select * from agg) select id from t where
id = 1")
+ .analyze("select id from agg where id = (select id from uni)")
+ .rewrite()
.getPlan();
- System.out.println(plan.treeString());
-
System.out.println(plan.getLogicalProperties().getFunctionalDependencies());
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0)));
}
@Test
- void testSetOperation() {
+ void testWindow() {
+ // partition by uniform
Plan plan = PlanChecker.from(connectContext)
- .analyze("select * from agg union select * from uni")
- .rewrite()
- .getPlan();
-
System.out.println(plan.getLogicalProperties().getFunctionalDependencies());
- Assertions.assertTrue(plan.getLogicalProperties()
-
.getFunctionalDependencies().isUniqueAndNotNull(ImmutableSet.copyOf(plan.getOutput())));
-
- plan = PlanChecker.from(connectContext)
- .analyze("select * from agg union all select * from uni")
- .rewrite()
- .getPlan();
-
System.out.println(plan.getLogicalProperties().getFunctionalDependencies());
- Assertions.assertFalse(plan.getLogicalProperties()
-
.getFunctionalDependencies().isUniqueAndNotNull(ImmutableSet.copyOf(plan.getOutput())));
-
- plan = PlanChecker.from(connectContext)
- .analyze("select * from agg intersect select * from uni where
name = \"1\"")
+ .analyze("select id, row_number() over(partition by id) from
agg where id =1")
.rewrite()
.getPlan();
-
System.out.println(plan.getLogicalProperties().getFunctionalDependencies());
- Assertions.assertTrue(plan.getLogicalProperties()
-
.getFunctionalDependencies().isUniqueAndNotNull(ImmutableSet.copyOf(plan.getOutput())));
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0)));
Assertions.assertTrue(plan.getLogicalProperties()
-
.getFunctionalDependencies().isUniformAndNotNull(plan.getOutput().get(1)));
+
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(1)));
+ // partition by None
plan = PlanChecker.from(connectContext)
- .analyze("select * from agg except select * from uni")
+ .analyze("select id, row_number() over() from agg where id =1")
.rewrite()
.getPlan();
-
System.out.println(plan.getLogicalProperties().getFunctionalDependencies());
- Assertions.assertTrue(plan.getLogicalProperties()
-
.getFunctionalDependencies().isUniqueAndNotNull(ImmutableSet.copyOf(plan.getOutput())));
Assertions.assertTrue(plan.getLogicalProperties()
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(0)));
+ Assertions.assertTrue(plan.getLogicalProperties()
+
.getFunctionalDependencies().isUniqueAndNotNull(plan.getOutput().get(1)));
}
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
