This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 4952a7c54b7 branch-3.0: [fix](mtmv) Fix data wrong when query with
table operator such as TABLESAMPLE or tablet and so on #43030 (#44873)
4952a7c54b7 is described below
commit 4952a7c54b7c40e5144974c538975332c0404edd
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Fri Dec 6 10:17:51 2024 +0800
branch-3.0: [fix](mtmv) Fix data wrong when query with table operator such
as TABLESAMPLE or tablet and so on #43030 (#44873)
Cherry-picked from #43030
Co-authored-by: seawinde <w...@selectdb.com>
---
.../java/org/apache/doris/mtmv/MTMVPlanUtil.java | 6 +
.../doris/nereids/rules/analysis/BindRelation.java | 4 +
.../mv/InitMaterializationContextHook.java | 5 +
.../exploration/mv/MaterializedViewUtils.java | 20 ++-
.../nereids/rules/exploration/mv/StructInfo.java | 7 +
.../nereids/rules/rewrite/PruneEmptyPartition.java | 7 +-
.../nereids/rules/rewrite/PruneOlapScanTablet.java | 6 +-
.../trees/plans/commands/info/CreateMTMVInfo.java | 3 +
.../trees/plans/logical/LogicalOlapScan.java | 55 ++++--
.../rules/rewrite/PruneOlapScanTabletTest.java | 21 ++-
.../apache/doris/nereids/sqltest/SqlTestBase.java | 3 +
.../org/apache/doris/regression/suite/Suite.groovy | 16 ++
.../mv/is_in_debug_mode/is_in_debug_mode.groovy | 156 +++++++++++++++++
.../with_table_operator/with_table_operator.groovy | 195 +++++++++++++++++++++
14 files changed, 482 insertions(+), 22 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java
index c0cd47bd5a0..576e87b44f8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java
@@ -58,6 +58,12 @@ public class MTMVPlanUtil {
ctx.setCurrentUserIdentity(UserIdentity.ADMIN);
ctx.getState().reset();
ctx.setThreadLocalInfo();
+ // Debug session variable should be disabled when refreshed
+ ctx.getSessionVariable().skipDeletePredicate = false;
+ ctx.getSessionVariable().skipDeleteBitmap = false;
+ ctx.getSessionVariable().skipDeleteSign = false;
+ ctx.getSessionVariable().skipStorageEngineMerge = false;
+ ctx.getSessionVariable().showHiddenColumns = false;
ctx.getSessionVariable().allowModifyMaterializedViewData = true;
// Disable add default limit rule to avoid refresh data wrong
ctx.getSessionVariable().setDisableNereidsRules(
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
index 2bc9fb4c729..c803f0aac31 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -243,6 +243,10 @@ public class BindRelation extends OneAnalysisRuleFactory {
unboundRelation.getTableSample());
}
}
+ if (!tabletIds.isEmpty()) {
+ // This tabletIds is set manually, so need to set
specifiedTabletIds
+ scan = scan.withManuallySpecifiedTabletIds(tabletIds);
+ }
if (needGenerateLogicalAggForRandomDistAggTable(scan)) {
// it's a random distribution agg table
// add agg on olap scan
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
index 2e8baecf165..f9ea00e178b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
@@ -79,6 +79,11 @@ public class InitMaterializationContextHook implements
PlannerHook {
* @param cascadesContext current cascadesContext in the planner
*/
protected void doInitMaterializationContext(CascadesContext
cascadesContext) {
+ if
(cascadesContext.getConnectContext().getSessionVariable().isInDebugMode()) {
+ LOG.info(String.format("MaterializationContext init return because
is in debug mode, current queryId is %s",
+ cascadesContext.getConnectContext().getQueryIdentifier()));
+ return;
+ }
// Only collect the table or mv which query use directly, to avoid
useless mv partition in rewrite
TableCollectorContext collectorContext = new
TableCollectorContext(Sets.newHashSet(), false);
try {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
index 484abd11f01..ee4b002007e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
@@ -347,7 +347,15 @@ public class MaterializedViewUtils {
planner.getCascadesContext().getMemo().getRoot().getStatistics(), null);
}
- private static final class TableQueryOperatorChecker extends
DefaultPlanVisitor<Boolean, Void> {
+ /**
+ * Check the query if Contains query operator
+ * Such sql as following should return true
+ * select * from orders TABLET(10098) because TABLET(10098) should return
true
+ * select * from orders_partition PARTITION (day_2) because PARTITION
(day_2)
+ * select * from orders index query_index_test because index
query_index_test
+ * select * from orders TABLESAMPLE(20 percent) because TABLESAMPLE(20
percent)
+ * */
+ public static final class TableQueryOperatorChecker extends
DefaultPlanVisitor<Boolean, Void> {
public static final TableQueryOperatorChecker INSTANCE = new
TableQueryOperatorChecker();
@Override
@@ -358,12 +366,20 @@ public class MaterializedViewUtils {
if (relation instanceof LogicalOlapScan) {
LogicalOlapScan logicalOlapScan = (LogicalOlapScan) relation;
if (logicalOlapScan.getTableSample().isPresent()) {
+ // Contain sample, select * from orders TABLESAMPLE(20
percent)
return true;
}
- if (!logicalOlapScan.getSelectedTabletIds().isEmpty()) {
+ if
(!logicalOlapScan.getManuallySpecifiedTabletIds().isEmpty()) {
+ // Contain tablets, select * from orders TABLET(10098)
because TABLET(10098)
return true;
}
if
(!logicalOlapScan.getManuallySpecifiedPartitions().isEmpty()) {
+ // Contain specified partitions, select * from
orders_partition PARTITION (day_2)
+ return true;
+ }
+ if (logicalOlapScan.getSelectedIndexId() !=
logicalOlapScan.getTable().getBaseIndexId()) {
+ // Contains select index or use sync mv in rbo rewrite
+ // select * from orders index query_index_test
return true;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
index 526ec7030d2..5a84ab787d7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
@@ -28,6 +28,7 @@ import
org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.JoinEdge;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.GroupExpression;
+import
org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils.TableQueryOperatorChecker;
import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate;
import org.apache.doris.nereids.trees.copier.DeepCopierContext;
import org.apache.doris.nereids.trees.copier.LogicalPlanDeepCopier;
@@ -36,6 +37,7 @@ import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.plans.AbstractPlan;
import org.apache.doris.nereids.trees.plans.GroupPlan;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.ObjectId;
@@ -323,6 +325,11 @@ public class StructInfo {
cascadesContext);
valid = valid
&& hyperGraph.getNodes().stream().allMatch(n ->
((StructInfoNode) n).getExpressions() != null);
+ // if relationList has any relation which contains table operator,
+ // such as query with sample, index, table, is invalid
+ boolean invalid = relationList.stream().anyMatch(relation ->
+ ((AbstractPlan)
relation).accept(TableQueryOperatorChecker.INSTANCE, null));
+ valid = valid && !invalid;
// collect predicate from top plan which not in hyper graph
Set<Expression> topPlanPredicates = new LinkedHashSet<>();
topPlan.accept(PREDICATE_COLLECTOR, topPlanPredicates);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java
index 5465211c73e..c7b8f452afb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java
@@ -36,11 +36,16 @@ public class PruneEmptyPartition extends
OneRewriteRuleFactory {
return logicalOlapScan().thenApply(ctx -> {
LogicalOlapScan scan = ctx.root;
OlapTable table = scan.getTable();
- List<Long> ids =
table.selectNonEmptyPartitionIds(scan.getSelectedPartitionIds());
+ List<Long> partitionIdsToPrune = scan.getSelectedPartitionIds();
+ List<Long> ids =
table.selectNonEmptyPartitionIds(partitionIdsToPrune);
if (ids.isEmpty()) {
return new
LogicalEmptyRelation(ConnectContext.get().getStatementContext().getNextRelationId(),
scan.getOutput());
}
+ if (partitionIdsToPrune.equals(ids)) {
+ // Not Prune actually, return directly
+ return null;
+ }
return scan.withSelectedPartitionIds(ids);
}).toRule(RuleType.PRUNE_EMPTY_PARTITION);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java
index 4b86b7a3706..fc212550b6f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java
@@ -53,7 +53,7 @@ public class PruneOlapScanTablet extends
OneRewriteRuleFactory {
LogicalOlapScan olapScan = filter.child();
OlapTable table = olapScan.getTable();
Builder<Long> selectedTabletIdsBuilder = ImmutableList.builder();
- if (olapScan.getSelectedTabletIds().isEmpty()) {
+ if (olapScan.getManuallySpecifiedTabletIds().isEmpty()) {
for (Long id : olapScan.getSelectedPartitionIds()) {
Partition partition = table.getPartition(id);
MaterializedIndex index =
partition.getIndex(olapScan.getSelectedIndexId());
@@ -64,10 +64,10 @@ public class PruneOlapScanTablet extends
OneRewriteRuleFactory {
partition.getDistributionInfo()));
}
} else {
-
selectedTabletIdsBuilder.addAll(olapScan.getSelectedTabletIds());
+
selectedTabletIdsBuilder.addAll(olapScan.getManuallySpecifiedTabletIds());
}
List<Long> selectedTabletIds = selectedTabletIdsBuilder.build();
- if (new HashSet<>(selectedTabletIds).equals(new
HashSet<>(olapScan.getSelectedTabletIds()))) {
+ if (new HashSet<>(selectedTabletIds).equals(new
HashSet<>(olapScan.getManuallySpecifiedTabletIds()))) {
return null;
}
return
filter.withChildren(olapScan.withSelectedTabletIds(selectedTabletIds));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
index 8c44b42a5cc..fbca38d47a4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
@@ -156,6 +156,9 @@ public class CreateMTMVInfo {
if (!InternalCatalog.INTERNAL_CATALOG_NAME.equals(mvName.getCtl())) {
throw new AnalysisException("Only support creating asynchronous
materialized views in internal catalog");
}
+ if (ctx.getSessionVariable().isInDebugMode()) {
+ throw new AnalysisException("Create materialized view fail,
because is in debug mode");
+ }
try {
FeNameFormat.checkTableName(mvName.getTbl());
} catch (org.apache.doris.common.AnalysisException e) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
index 90ceb242313..2216e58c4fa 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
@@ -99,6 +99,12 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan
*/
private final List<Long> selectedTabletIds;
+ /**
+ * Selected tablet ids to read data from, this would be set if user query
with tablets manually
+ * Such as select * from orders TABLET(100);
+ */
+ private final List<Long> manuallySpecifiedTabletIds;
+
///////////////////////////////////////////////////////////////////////////
// Members for partition ids.
///////////////////////////////////////////////////////////////////////////
@@ -127,12 +133,16 @@ public class LogicalOlapScan extends
LogicalCatalogRelation implements OlapScan
this(id, table, ImmutableList.of());
}
+ /**
+ * LogicalOlapScan construct method
+ */
public LogicalOlapScan(RelationId id, OlapTable table, List<String>
qualifier) {
this(id, table, qualifier, Optional.empty(), Optional.empty(),
table.getPartitionIds(), false,
ImmutableList.of(),
-1, false, PreAggStatus.unset(), ImmutableList.of(),
ImmutableList.of(),
- Maps.newHashMap(), Optional.empty(), false, ImmutableMap.of());
+ Maps.newHashMap(), Optional.empty(), false, ImmutableMap.of(),
+ ImmutableList.of());
}
public LogicalOlapScan(RelationId id, OlapTable table, List<String>
qualifier, List<Long> tabletIds,
@@ -140,7 +150,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan
this(id, table, qualifier, Optional.empty(), Optional.empty(),
table.getPartitionIds(), false, tabletIds,
-1, false, PreAggStatus.unset(), ImmutableList.of(), hints,
Maps.newHashMap(),
- tableSample, false, ImmutableMap.of());
+ tableSample, false, ImmutableMap.of(), ImmutableList.of());
}
public LogicalOlapScan(RelationId id, OlapTable table, List<String>
qualifier, List<Long> specifiedPartitions,
@@ -149,7 +159,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan
// must use specifiedPartitions here for prune partition by
sql like 'select * from t partition p1'
specifiedPartitions, false, tabletIds,
-1, false, PreAggStatus.unset(), specifiedPartitions, hints,
Maps.newHashMap(),
- tableSample, false, ImmutableMap.of());
+ tableSample, false, ImmutableMap.of(), ImmutableList.of());
}
public LogicalOlapScan(RelationId id, OlapTable table, List<String>
qualifier, List<Long> tabletIds,
@@ -158,7 +168,8 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan
this(id, table, qualifier, Optional.empty(), Optional.empty(),
selectedPartitionIds, false, tabletIds,
selectedIndexId, true, preAggStatus,
- specifiedPartitions, hints, Maps.newHashMap(), tableSample,
true, ImmutableMap.of());
+ specifiedPartitions, hints, Maps.newHashMap(), tableSample,
true, ImmutableMap.of(),
+ ImmutableList.of());
}
/**
@@ -171,7 +182,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan
PreAggStatus preAggStatus, List<Long> specifiedPartitions,
List<String> hints, Map<Pair<Long, String>, Slot>
cacheSlotWithSlotName,
Optional<TableSample> tableSample, boolean directMvScan,
- Map<String, Set<List<String>>> colToSubPathsMap) {
+ Map<String, Set<List<String>>> colToSubPathsMap, List<Long>
specifiedTabletIds) {
super(id, PlanType.LOGICAL_OLAP_SCAN, table, qualifier,
groupExpression, logicalProperties);
Preconditions.checkArgument(selectedPartitionIds != null,
@@ -182,6 +193,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan
this.indexSelected = indexSelected;
this.preAggStatus = preAggStatus;
this.manuallySpecifiedPartitions =
ImmutableList.copyOf(specifiedPartitions);
+ this.manuallySpecifiedTabletIds =
ImmutableList.copyOf(specifiedTabletIds);
if (selectedPartitionIds.isEmpty()) {
this.selectedPartitionIds = ImmutableList.of();
@@ -240,6 +252,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan
&& partitionPruned == that.partitionPruned &&
Objects.equals(preAggStatus, that.preAggStatus)
&& Objects.equals(selectedTabletIds, that.selectedTabletIds)
&& Objects.equals(manuallySpecifiedPartitions,
that.manuallySpecifiedPartitions)
+ && Objects.equals(manuallySpecifiedTabletIds,
that.manuallySpecifiedTabletIds)
&& Objects.equals(selectedPartitionIds,
that.selectedPartitionIds)
&& Objects.equals(hints, that.hints)
&& Objects.equals(tableSample, that.tableSample);
@@ -248,8 +261,8 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), selectedIndexId, indexSelected,
preAggStatus, cacheSlotWithSlotName,
- selectedTabletIds, partitionPruned,
manuallySpecifiedPartitions, selectedPartitionIds, hints,
- tableSample);
+ selectedTabletIds, partitionPruned,
manuallySpecifiedTabletIds, manuallySpecifiedPartitions,
+ selectedPartitionIds, hints, tableSample);
}
@Override
@@ -258,7 +271,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan
groupExpression, Optional.of(getLogicalProperties()),
selectedPartitionIds, partitionPruned, selectedTabletIds,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
- hints, cacheSlotWithSlotName, tableSample, directMvScan,
colToSubPathsMap);
+ hints, cacheSlotWithSlotName, tableSample, directMvScan,
colToSubPathsMap, manuallySpecifiedTabletIds);
}
@Override
@@ -267,7 +280,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan
return new LogicalOlapScan(relationId, (Table) table, qualifier,
groupExpression, logicalProperties,
selectedPartitionIds, partitionPruned, selectedTabletIds,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
- hints, cacheSlotWithSlotName, tableSample, directMvScan,
colToSubPathsMap);
+ hints, cacheSlotWithSlotName, tableSample, directMvScan,
colToSubPathsMap, manuallySpecifiedTabletIds);
}
public LogicalOlapScan withSelectedPartitionIds(List<Long>
selectedPartitionIds) {
@@ -275,7 +288,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan
Optional.empty(), Optional.of(getLogicalProperties()),
selectedPartitionIds, true, selectedTabletIds,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
- hints, cacheSlotWithSlotName, tableSample, directMvScan,
colToSubPathsMap);
+ hints, cacheSlotWithSlotName, tableSample, directMvScan,
colToSubPathsMap, manuallySpecifiedTabletIds);
}
public LogicalOlapScan withMaterializedIndexSelected(long indexId) {
@@ -283,7 +296,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan
Optional.empty(), Optional.of(getLogicalProperties()),
selectedPartitionIds, partitionPruned, selectedTabletIds,
indexId, true, PreAggStatus.unset(),
manuallySpecifiedPartitions, hints, cacheSlotWithSlotName,
- tableSample, directMvScan, colToSubPathsMap);
+ tableSample, directMvScan, colToSubPathsMap,
manuallySpecifiedTabletIds);
}
public LogicalOlapScan withSelectedTabletIds(List<Long> selectedTabletIds)
{
@@ -291,7 +304,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan
Optional.empty(), Optional.of(getLogicalProperties()),
selectedPartitionIds, partitionPruned, selectedTabletIds,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
- hints, cacheSlotWithSlotName, tableSample, directMvScan,
colToSubPathsMap);
+ hints, cacheSlotWithSlotName, tableSample, directMvScan,
colToSubPathsMap, manuallySpecifiedTabletIds);
}
public LogicalOlapScan withPreAggStatus(PreAggStatus preAggStatus) {
@@ -299,7 +312,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan
Optional.empty(), Optional.of(getLogicalProperties()),
selectedPartitionIds, partitionPruned, selectedTabletIds,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
- hints, cacheSlotWithSlotName, tableSample, directMvScan,
colToSubPathsMap);
+ hints, cacheSlotWithSlotName, tableSample, directMvScan,
colToSubPathsMap, manuallySpecifiedTabletIds);
}
public LogicalOlapScan withColToSubPathsMap(Map<String, Set<List<String>>>
colToSubPathsMap) {
@@ -307,7 +320,15 @@ public class LogicalOlapScan extends
LogicalCatalogRelation implements OlapScan
Optional.empty(), Optional.empty(),
selectedPartitionIds, partitionPruned, selectedTabletIds,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
- hints, cacheSlotWithSlotName, tableSample, directMvScan,
colToSubPathsMap);
+ hints, cacheSlotWithSlotName, tableSample, directMvScan,
colToSubPathsMap, manuallySpecifiedTabletIds);
+ }
+
+ public LogicalOlapScan withManuallySpecifiedTabletIds(List<Long>
manuallySpecifiedTabletIds) {
+ return new LogicalOlapScan(relationId, (Table) table, qualifier,
+ Optional.empty(), Optional.of(getLogicalProperties()),
+ selectedPartitionIds, partitionPruned, selectedTabletIds,
+ selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
+ hints, cacheSlotWithSlotName, tableSample, directMvScan,
colToSubPathsMap, manuallySpecifiedTabletIds);
}
@Override
@@ -317,7 +338,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan
Optional.empty(), Optional.empty(),
selectedPartitionIds, false, selectedTabletIds,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
- hints, Maps.newHashMap(), tableSample, directMvScan,
colToSubPathsMap);
+ hints, Maps.newHashMap(), tableSample, directMvScan,
colToSubPathsMap, selectedTabletIds);
}
@Override
@@ -333,6 +354,10 @@ public class LogicalOlapScan extends
LogicalCatalogRelation implements OlapScan
return selectedTabletIds;
}
+ public List<Long> getManuallySpecifiedTabletIds() {
+ return manuallySpecifiedTabletIds;
+ }
+
@Override
public long getSelectedIndexId() {
return selectedIndexId;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java
index e7940cdfb9b..385657cd3f2 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java
@@ -28,6 +28,7 @@ import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.nereids.sqltest.SqlTestBase;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
import org.apache.doris.nereids.trees.expressions.InPredicate;
@@ -51,8 +52,9 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.List;
+import java.util.Objects;
-class PruneOlapScanTabletTest implements MemoPatternMatchSupported {
+class PruneOlapScanTabletTest extends SqlTestBase implements
MemoPatternMatchSupported {
@Test
void testPruneOlapScanTablet(@Mocked OlapTable olapTable,
@@ -154,4 +156,21 @@ class PruneOlapScanTabletTest implements
MemoPatternMatchSupported {
)
);
}
+
+ @Test
+ void testPruneOlapScanTabletWithManually() {
+ String sql = "select * from T4 TABLET(110) where id > 8";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .applyTopDown(new PruneOlapScanTablet())
+ .matches(
+ logicalFilter(
+ logicalOlapScan().when(s ->
+
Objects.equals(s.getSelectedTabletIds(), Lists.newArrayList(110L))
+ &&
Objects.equals(s.getManuallySpecifiedTabletIds(),
+ Lists.newArrayList(110L))
+ )
+ )
+ );
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java
index f5b301dcc49..9ebbc22a85c 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java
@@ -87,6 +87,9 @@ public abstract class SqlTestBase extends TestWithFeService
implements MemoPatte
+ " score bigint\n"
+ ")\n"
+ "DUPLICATE KEY(id)\n"
+ + "AUTO PARTITION BY LIST(`id`)\n"
+ + "(\n"
+ + ")\n"
+ "DISTRIBUTED BY HASH(id) BUCKETS 1\n"
+ "PROPERTIES (\n"
+ " \"replication_num\" = \"1\"\n"
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index 650e949da85..953aebb10cb 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -1814,6 +1814,22 @@ class Suite implements GroovyInterceptable {
sql "analyze table ${db}.${mv_name} with sync;"
}
+ def create_async_partition_mv = { db, mv_name, mv_sql, partition_col ->
+
+ sql """DROP MATERIALIZED VIEW IF EXISTS ${db}.${mv_name}"""
+ sql"""
+ CREATE MATERIALIZED VIEW ${db}.${mv_name}
+ BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL
+ PARTITION BY ${partition_col}
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES ('replication_num' = '1')
+ AS ${mv_sql}
+ """
+ def job_name = getJobName(db, mv_name);
+ waitingMTMVTaskFinished(job_name)
+ sql "analyze table ${db}.${mv_name} with sync;"
+ }
+
// mv not part in rewrite process
void mv_not_part_in(query_sql, mv_name, sync_cbo_rewrite =
enable_sync_mv_cost_based_rewrite()) {
logger.info("query_sql = " + query_sql + ", mv_names = " + mv_name +
", sync_cbo_rewrite = " + sync_cbo_rewrite)
diff --git
a/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy
b/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy
new file mode 100644
index 00000000000..15d93e32f65
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy
@@ -0,0 +1,156 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Assert
+
+suite("is_in_debug_mode") {
+ String db = context.config.getDbNameByFile(context.file)
+ sql "use ${db}"
+ sql "set runtime_filter_mode=OFF";
+ sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'"
+
+ sql """
+ drop table if exists orders;
+ """
+
+ sql """
+ CREATE TABLE IF NOT EXISTS orders (
+ o_orderkey INTEGER NOT NULL,
+ o_custkey INTEGER NOT NULL,
+ o_orderstatus CHAR(1) NOT NULL,
+ o_totalprice DECIMALV3(15,2) NOT NULL,
+ o_orderdate DATE NOT NULL,
+ o_orderpriority CHAR(15) NOT NULL,
+ o_clerk CHAR(15) NOT NULL,
+ o_shippriority INTEGER NOT NULL,
+ O_COMMENT VARCHAR(79) NOT NULL
+ )
+ UNIQUE KEY(o_orderkey, o_custkey)
+ DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+
+ sql """
+ insert into orders values
+ (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+ (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+ (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+ (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+ (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+ (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+ (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+ (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+ (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+ (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+ (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+ (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+ (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+ (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+ (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+ (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+ (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+ (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi');
+ """
+
+ create_async_mv(db, "basic_mv", """
+ select * from orders where o_orderkey > 1;
+ """)
+
+ sql """set skip_delete_sign = true;"""
+ mv_not_part_in("""select * from orders where o_orderkey > 1;""",
"basic_mv")
+ try {
+ sql """
+ CREATE MATERIALIZED VIEW test_create_mv
+ BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES ('replication_num' = '1')
+ AS select * from orders where o_orderkey > 2;
+ """
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains("because is in debug mode"))
+ }
+ sql """set skip_delete_sign = false;"""
+
+
+ sql """set skip_storage_engine_merge = true;"""
+ mv_not_part_in("""select * from orders where o_orderkey > 1;""",
"basic_mv")
+ try {
+ sql """
+ CREATE MATERIALIZED VIEW test_create_mv
+ BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES ('replication_num' = '1')
+ AS select * from orders where o_orderkey > 2;
+ """
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains("because is in debug mode"))
+ }
+ sql """set skip_storage_engine_merge = false;"""
+
+
+ sql """set skip_delete_bitmap = true;"""
+ mv_not_part_in("""select * from orders where o_orderkey > 1;""",
"basic_mv")
+ try {
+ sql """
+ CREATE MATERIALIZED VIEW test_create_mv
+ BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES ('replication_num' = '1')
+ AS select * from orders where o_orderkey > 2;
+ """
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains("because is in debug mode"))
+ }
+ sql """set skip_delete_bitmap = false;"""
+
+
+ sql """set skip_delete_predicate = true;"""
+ mv_not_part_in("""select * from orders where o_orderkey > 1;""",
"basic_mv")
+ try {
+ sql """
+ CREATE MATERIALIZED VIEW test_create_mv
+ BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES ('replication_num' = '1')
+ AS select * from orders where o_orderkey > 2;
+ """
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains("because is in debug mode"))
+ }
+ sql """set skip_delete_predicate = false;"""
+
+
+ sql """set show_hidden_columns = true;"""
+ mv_not_part_in("""select * from orders where o_orderkey > 1;""",
"basic_mv")
+ try {
+ sql """
+ CREATE MATERIALIZED VIEW test_create_mv
+ BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES ('replication_num' = '1')
+ AS select * from orders where o_orderkey > 2;
+ """
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains("because is in debug mode"))
+ }
+ sql """set show_hidden_columns = false;"""
+
+ sql """drop materialized view if exists basic_mv"""
+}
+
diff --git
a/regression-test/suites/nereids_rules_p0/mv/with_table_operator/with_table_operator.groovy
b/regression-test/suites/nereids_rules_p0/mv/with_table_operator/with_table_operator.groovy
new file mode 100644
index 00000000000..5ba2c05053e
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/mv/with_table_operator/with_table_operator.groovy
@@ -0,0 +1,195 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("with_table_operator") {
+ String db = context.config.getDbNameByFile(context.file)
+ sql "use ${db}"
+ sql "set runtime_filter_mode=OFF";
+ sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'"
+
+ sql """
+ drop table if exists orders;
+ """
+
+ sql """
+ CREATE TABLE IF NOT EXISTS orders (
+ o_orderkey INTEGER NOT NULL,
+ o_custkey INTEGER NOT NULL,
+ o_orderstatus CHAR(1) NOT NULL,
+ o_totalprice DECIMALV3(15,2) NOT NULL,
+ o_orderdate DATE NOT NULL,
+ o_orderpriority CHAR(15) NOT NULL,
+ o_clerk CHAR(15) NOT NULL,
+ o_shippriority INTEGER NOT NULL,
+ O_COMMENT VARCHAR(79) NOT NULL
+ )
+ DUPLICATE KEY(o_orderkey, o_custkey)
+ DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+
+ sql """
+ insert into orders values
+ (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+ (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+ (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+ (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+ (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+ (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+ (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+ (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+ (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+ (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+ (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+ (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+ (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+ (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+ (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+ (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+ (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+ (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi');
+ """
+
+ sql """
+ drop table if exists orders_partition;
+ """
+
+ sql """
+ CREATE TABLE IF NOT EXISTS orders_partition (
+ o_orderkey INTEGER NOT NULL,
+ o_custkey INTEGER NOT NULL,
+ o_orderstatus CHAR(1) NOT NULL,
+ o_totalprice DECIMALV3(15,2) NOT NULL,
+ o_orderdate DATE NOT NULL,
+ o_orderpriority CHAR(15) NOT NULL,
+ o_clerk CHAR(15) NOT NULL,
+ o_shippriority INTEGER NOT NULL,
+ O_COMMENT VARCHAR(79) NOT NULL
+ )
+ DUPLICATE KEY(o_orderkey, o_custkey)
+ PARTITION BY RANGE(o_orderdate) (
+ PARTITION `day_2` VALUES LESS THAN ('2023-12-9'),
+ PARTITION `day_3` VALUES LESS THAN ("2023-12-11"),
+ PARTITION `day_4` VALUES LESS THAN ("2023-12-30")
+ )
+ DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+
+ sql """
+ insert into orders_partition values
+ (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+ (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+ (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+ (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+ (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+ (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+ (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+ (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+ (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+ (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+ (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+ (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+ (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+ (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+ (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+ (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+ (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+ (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi');
+ """
+
+ create_async_mv(db, "complete_agg_mv", """
+ select count(*),o_orderdate
+ from orders
+ group by o_orderdate;
+ """)
+
+ create_async_mv(db, "complete_mv", """
+ select * from orders;
+ """)
+
+ create_async_partition_mv(db, "partition_agg_mv", """
+ select count(*),o_orderdate
+ from orders_partition
+ group by o_orderdate;
+ """, "(o_orderdate)")
+
+ create_async_partition_mv(db, "partition_mv", """
+ select * from orders_partition;
+ """, "(o_orderdate)")
+
+ // query rewrite with tablet should fail
+ mv_rewrite_all_fail("""
+ select count(*),o_orderdate
+ from orders TABLET(110)
+ group by o_orderdate;
+ """, ["complete_mv", "complete_agg_mv"])
+
+ mv_rewrite_all_fail("""select * from orders TABLET(110);""",
["complete_mv", "complete_agg_mv"])
+
+ // query rewrite with index should fail
+ createMV("""
+ create materialized view query_index_test
+ as
+ select o_orderdate, count(*)
+ from orders
+ group by o_orderdate;
+ """)
+ mv_rewrite_all_fail("""
+ select * from orders index query_index_test;
+ """, ["complete_mv", "complete_agg_mv"])
+
+ // query rewrite with sample should fail
+ mv_rewrite_all_fail("""
+ select count(*),o_orderdate
+ from orders TABLESAMPLE(20 percent)
+ group by o_orderdate;
+ """, ["complete_mv", "complete_agg_mv"])
+
+ mv_rewrite_all_fail("""select * from orders TABLESAMPLE(20 percent);""",
["complete_mv", "complete_agg_mv"])
+
+ // query rewrite with partition should fail
+ mv_rewrite_all_fail("""
+ select count(*),o_orderdate
+ from orders_partition PARTITION (day_2)
+ group by o_orderdate;
+ """, ["partition_mv", "partition_agg_mv"])
+
+ mv_rewrite_all_fail("""select * from orders_partition PARTITION
(day_2);""", ["partition_mv", "partition_agg_mv"])
+
+
+ sql """
+ DROP MATERIALIZED VIEW IF EXISTS complete_agg_mv;
+ """
+ sql """
+ DROP MATERIALIZED VIEW IF EXISTS partition_agg_mv;
+ """
+ sql """
+ DROP MATERIALIZED VIEW IF EXISTS complete_mv;
+ """
+ sql """
+ DROP MATERIALIZED VIEW IF EXISTS partition_mv;
+ """
+ sql """
+ DROP MATERIALIZED VIEW IF EXISTS query_index_test on orders;
+ """
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
