This is an automated email from the ASF dual-hosted git repository. amashenkov pushed a commit to branch ignite-21580 in repository https://gitbox.apache.org/repos/asf/ignite-3.git
commit 9970fd765243264c72d0c062e3c4bd92fbd8681b Author: amashenkov <[email protected]> AuthorDate: Wed Mar 27 16:28:05 2024 +0300 Fix wrong memory for IgniteAggregate and distinct count estimation. --- .../engine/metadata/IgniteMdDistinctRowCount.java | 34 ++++++++++++++++++---- .../internal/sql/engine/rel/IgniteAggregate.java | 23 +++++++++++---- 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/metadata/IgniteMdDistinctRowCount.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/metadata/IgniteMdDistinctRowCount.java index 9530ee5059..19b8115545 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/metadata/IgniteMdDistinctRowCount.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/metadata/IgniteMdDistinctRowCount.java @@ -17,7 +17,11 @@ package org.apache.ignite.internal.sql.engine.metadata; +import org.apache.calcite.plan.Convention; import org.apache.calcite.plan.volcano.RelSubset; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.metadata.CyclicMetadataException; import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider; import org.apache.calcite.rel.metadata.RelMdDistinctRowCount; import org.apache.calcite.rel.metadata.RelMetadataProvider; @@ -25,6 +29,8 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.BuiltInMethod; import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.NumberUtil; +import org.checkerframework.checker.nullness.qual.Nullable; /** * IgniteMdDistinctRowCount. @@ -36,6 +42,11 @@ public class IgniteMdDistinctRowCount extends RelMdDistinctRowCount { ReflectiveRelMetadataProvider.reflectiveSource( BuiltInMethod.DISTINCT_ROW_COUNT.method, new IgniteMdDistinctRowCount()); + @Override + public @Nullable Double getDistinctRowCount(Aggregate rel, RelMetadataQuery mq, ImmutableBitSet groupKey, @Nullable RexNode predicate) { + return rel.estimateRowCount(mq); + } + /** {@inheritDoc} */ @Override public Double getDistinctRowCount( @@ -44,14 +55,25 @@ public class IgniteMdDistinctRowCount extends RelMdDistinctRowCount { ImmutableBitSet groupKey, RexNode predicate ) { - if (groupKey.cardinality() == 0) { - return 1d; + RelNode best = rel.getBest(); + if (best != null) { + return mq.getDistinctRowCount(best, groupKey, predicate); } - double rowCount = mq.getRowCount(rel); - - rowCount *= 1.0 - Math.pow(.5, groupKey.cardinality()); + if (rel.getConvention() == Convention.NONE) { + return null; + } - return rowCount; + Double d = null; + for (RelNode r2 : rel.getRels()) { + try { + Double d2 = mq.getDistinctRowCount(r2, groupKey, predicate); + d = NumberUtil.min(d, d2); + } catch (CyclicMetadataException e) { + // Ignore this relational expression; there will be non-cyclic ones + // in this set. + } + } + return d; } } diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rel/IgniteAggregate.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rel/IgniteAggregate.java index 637d2aca19..43c8080d68 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rel/IgniteAggregate.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rel/IgniteAggregate.java @@ -64,9 +64,7 @@ public abstract class IgniteAggregate extends Aggregate implements IgniteRel { return groupsCnt; } - // Estimation of the groups count is not available. - // Use heuristic estimation for result rows count. - return super.estimateRowCount(mq); + return guessDistinctRows(mq, groupSet.cardinality()); } /** @@ -78,11 +76,14 @@ public abstract class IgniteAggregate extends Aggregate implements IgniteRel { if (!aggCalls.isEmpty()) { double grps = estimateRowCount(mq); - double rows = input.estimateRowCount(mq); for (AggregateCall aggCall : aggCalls) { if (aggCall.isDistinct()) { - mem += IgniteCost.AGG_CALL_MEM_COST * rows / grps; + ImmutableBitSet aggGroup = ImmutableBitSet.of(aggCall.getArgList()); + + double distinctRows = guessDistinctRows(mq, aggGroup.cardinality()); + + mem += IgniteCost.AGG_CALL_MEM_COST * distinctRows / grps; } else { mem += IgniteCost.AGG_CALL_MEM_COST; } @@ -92,6 +93,18 @@ public abstract class IgniteAggregate extends Aggregate implements IgniteRel { return mem; } + private double guessDistinctRows(RelMetadataQuery mq, int groupSize) { + // Estimation of the groups count is not available. + // Use heuristic estimation for result rows count. + if (groupSize == 0) { + return 1; + } else { + double rowCount = mq.getRowCount(getInput()); + rowCount *= (1.0 - Math.pow(.8, groupSize)); + return rowCount; + } + } + /** * ComputeSelfCostHash. * TODO Documentation https://issues.apache.org/jira/browse/IGNITE-15859
