This is an automated email from the ASF dual-hosted git repository. caogaofei pushed a commit to branch beyyes/TableModelGrammar_0627 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a6181335df3382d87108425d7426f763d3ead899 Author: Beyyes <[email protected]> AuthorDate: Thu Jun 27 17:22:08 2024 +0800 add class SplitExpression for PushPredicateIntoTableScan --- .../plan/planner/plan/node/PlanNode.java | 2 +- .../optimizations/PushPredicateIntoTableScan.java | 84 ++++++++++++++-------- 2 files changed, 55 insertions(+), 31 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNode.java index ae5340e335e..3e6942dd147 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNode.java @@ -218,7 +218,7 @@ public abstract class PlanNode implements IConsensusRequest { return Objects.hash(id); } - // =========================== Used for Relational Model ============================ + // =========================== Used for Table Model ============================ public List<Symbol> getOutputSymbols() { throw new UnsupportedOperationException("This planNode does not support getOutputSymbols()."); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index e9fb6ead873..58f8130a960 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -43,12 +43,12 @@ import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Pair; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import static java.util.Objects.requireNonNull; import static org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory.ATTRIBUTE; import static org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory.MEASUREMENT; import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.getTimePartitionSlotList; @@ -82,10 +82,10 @@ public class PushPredicateIntoTableScan implements RelationalPlanOptimizer { Metadata metadata, SessionInfo sessionInfo, MPPQueryContext queryContext) { - return planNode.accept(new Rewriter(queryContext, analysis, metadata), new RewriterContext()); + return planNode.accept(new Rewriter(queryContext, analysis, metadata), null); } - private static class Rewriter extends PlanVisitor<PlanNode, RewriterContext> { + private static class Rewriter extends PlanVisitor<PlanNode, Void> { private final MPPQueryContext queryContext; private final Analysis analysis; private final Metadata metadata; @@ -98,20 +98,20 @@ public class PushPredicateIntoTableScan implements RelationalPlanOptimizer { } @Override - public PlanNode visitPlan(PlanNode node, RewriterContext context) { + public PlanNode visitPlan(PlanNode node, Void context) { throw new IllegalArgumentException( String.format("Unexpected plan node: %s in rule PushPredicateIntoTableScan", node)); } @Override - public PlanNode visitSingleChildProcess(SingleChildProcessNode node, RewriterContext context) { + public PlanNode visitSingleChildProcess(SingleChildProcessNode node, Void context) { PlanNode rewrittenChild = node.getChild().accept(this, context); node.setChild(rewrittenChild); return node; } @Override - public PlanNode visitMultiChildProcess(MultiChildProcessNode node, RewriterContext context) { + public PlanNode visitMultiChildProcess(MultiChildProcessNode node, Void context) { List<PlanNode> rewrittenChildren = new ArrayList<>(); for (PlanNode child : node.getChildren()) { rewrittenChildren.add(child.accept(this, context)); @@ -121,7 +121,7 @@ public class PushPredicateIntoTableScan implements RelationalPlanOptimizer { } @Override - public PlanNode visitFilter(FilterNode node, RewriterContext context) { + public PlanNode visitFilter(FilterNode node, Void context) { if (node.getPredicate() != null) { @@ -149,14 +149,11 @@ public class PushPredicateIntoTableScan implements RelationalPlanOptimizer { } public PlanNode combineFilterAndScan(TableScanNode tableScanNode) { - List<List<Expression>> splitPredicates = splitPredicate(tableScanNode); - - // exist indexed metadata expressions - tableMetadataIndexScan(tableScanNode, splitPredicates.get(0)); + SplitExpression splitExpression = splitPredicate(tableScanNode); // exist expressions can push down to scan operator - if (!splitPredicates.get(1).isEmpty()) { - List<Expression> expressions = splitPredicates.get(1); + if (!splitExpression.getExpressionsCanPushDown().isEmpty()) { + List<Expression> expressions = splitExpression.getExpressionsCanPushDown(); Expression pushDownPredicate = expressions.size() == 1 ? expressions.get(0) @@ -174,9 +171,12 @@ public class PushPredicateIntoTableScan implements RelationalPlanOptimizer { tableScanNode.setPushDownPredicate(null); } + // do index scan after expressionCanPushDown is processed + tableMetadataIndexScan(tableScanNode, splitExpression.getMetadataExpressions()); + // exist expressions can not push down to scan operator - if (!splitPredicates.get(2).isEmpty()) { - List<Expression> expressions = splitPredicates.get(2); + if (!splitExpression.getExpressionsCannotPushDown().isEmpty()) { + List<Expression> expressions = splitExpression.getExpressionsCannotPushDown(); return new FilterNode( queryContext.getQueryId().genPlanNodeId(), tableScanNode, @@ -188,12 +188,7 @@ public class PushPredicateIntoTableScan implements RelationalPlanOptimizer { return tableScanNode; } - /** - * splits predicate expression in table model into three parts, index 0 represents - * metadataExpressions, index 1 represents expressionsCanPushDownToOperator, index 2 represents - * expressionsCannotPushDownToOperator - */ - private List<List<Expression>> splitPredicate(TableScanNode node) { + private SplitExpression splitPredicate(TableScanNode node) { Set<String> idOrAttributeColumnNames = node.getIdAndAttributeIndexMap().keySet().stream() @@ -225,7 +220,7 @@ public class PushPredicateIntoTableScan implements RelationalPlanOptimizer { } } - return Arrays.asList( + return new SplitExpression( metadataExpressions, expressionsCanPushDown, expressionsCannotPushDown); } @@ -237,11 +232,12 @@ public class PushPredicateIntoTableScan implements RelationalPlanOptimizer { expressionsCannotPushDown.add(predicate); } - return Arrays.asList(metadataExpressions, expressionsCanPushDown, expressionsCannotPushDown); + return new SplitExpression( + metadataExpressions, expressionsCanPushDown, expressionsCannotPushDown); } @Override - public PlanNode visitTableScan(TableScanNode node, RewriterContext context) { + public PlanNode visitTableScan(TableScanNode node, Void context) { tableMetadataIndexScan(node, Collections.emptyList()); return node; } @@ -263,11 +259,9 @@ public class PushPredicateIntoTableScan implements RelationalPlanOptimizer { analysis.setEmptyDataSource(true); } else { Filter timeFilter = - node.getTimePredicate().isPresent() - ? node.getTimePredicate() - .get() - .accept(new ConvertPredicateToTimeFilterVisitor(), null) - : null; + node.getTimePredicate() + .map(value -> value.accept(new ConvertPredicateToTimeFilterVisitor(), null)) + .orElse(null); node.setTimeFilter(timeFilter); String treeModelDatabase = "root." + node.getQualifiedObjectName().getDatabaseName(); DataPartition dataPartition = @@ -333,5 +327,35 @@ public class PushPredicateIntoTableScan implements RelationalPlanOptimizer { return false; } - private static class RewriterContext {} + private static class SplitExpression { + // indexed tag expressions, such as `tag1 = 'A'` + List<Expression> metadataExpressions; + // expressions can push down into TableScan, such as `time > 1 and s_1 = 1` + List<Expression> expressionsCanPushDown; + // expressions can not push down into TableScan, such as `s_1 is null` + List<Expression> expressionsCannotPushDown; + + public SplitExpression( + List<Expression> metadataExpressions, + List<Expression> expressionsCanPushDown, + List<Expression> expressionsCannotPushDown) { + this.metadataExpressions = requireNonNull(metadataExpressions, "metadataExpressions is null"); + this.expressionsCanPushDown = + requireNonNull(expressionsCanPushDown, "expressionsCanPushDown is null"); + this.expressionsCannotPushDown = + requireNonNull(expressionsCannotPushDown, "expressionsCannotPushDown is null"); + } + + public List<Expression> getMetadataExpressions() { + return this.metadataExpressions; + } + + public List<Expression> getExpressionsCanPushDown() { + return this.expressionsCanPushDown; + } + + public List<Expression> getExpressionsCannotPushDown() { + return this.expressionsCannotPushDown; + } + } }
