This is an automated email from the ASF dual-hosted git repository.
zouxinyi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 03d59ba81e [Fix](Nereids) fix sql-cache for nereids. (#22808)
03d59ba81e is described below
commit 03d59ba81e54576cd7a3e6b9c5d160164e6c098b
Author: Xiangyu Wang <[email protected]>
AuthorDate: Fri Aug 18 09:36:07 2023 +0800
[Fix](Nereids) fix sql-cache for nereids. (#22808)
1. should not use
((LogicalPlanAdapter)parsedStmt).getStatementContext().getOriginStatement().originStmt.toLowerCase()
as the cache key (do not invoke toLowerCase()), for example: select * from
tbl1 where k1 = 'a' is different with select * from tbl1 where k1 = 'A', so the
cache should be missed.
2. according to issue 6735 , the cache key should contains all views' s ddl
sql (including nested views)
---
.../org/apache/doris/nereids/NereidsPlanner.java | 1 +
.../org/apache/doris/nereids/StatementContext.java | 12 +++
.../doris/nereids/glue/LogicalPlanAdapter.java | 10 ++
.../doris/nereids/rules/analysis/BindRelation.java | 2 +-
.../org/apache/doris/qe/cache/CacheAnalyzer.java | 26 +++--
.../java/org/apache/doris/qe/cache/SqlCache.java | 15 +--
.../org/apache/doris/qe/PartitionCacheTest.java | 112 ++++++++++++++++++++-
7 files changed, 160 insertions(+), 18 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index aa8315d64e..3065413814 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -133,6 +133,7 @@ public class NereidsPlanner extends Planner {
ArrayList<String> columnLabelList =
physicalPlan.getOutput().stream().map(NamedExpression::getName)
.collect(Collectors.toCollection(ArrayList::new));
logicalPlanAdapter.setColLabels(columnLabelList);
+ logicalPlanAdapter.setViews(statementContext.getViews());
}
@VisibleForTesting
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
index 4cb121ab56..59d23db292 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids;
import org.apache.doris.analysis.StatementBase;
+import org.apache.doris.catalog.View;
import org.apache.doris.common.IdGenerator;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.memo.Group;
@@ -36,7 +37,9 @@ import org.apache.doris.qe.OriginStatement;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
import java.util.HashMap;
import java.util.List;
@@ -78,6 +81,7 @@ public class StatementContext {
// Used to update consumer's stats
private final Map<CTEId, List<Pair<Map<Slot, Slot>, Group>>>
cteIdToConsumerGroup = new HashMap<>();
private final Map<CTEId, LogicalPlan> rewrittenCtePlan = new HashMap<>();
+ private final Set<View> views = Sets.newHashSet();
public StatementContext() {
this.connectContext = ConnectContext.get();
@@ -207,4 +211,12 @@ public class StatementContext {
public Map<CTEId, LogicalPlan> getRewrittenCtePlan() {
return rewrittenCtePlan;
}
+
+ public void addView(View view) {
+ this.views.add(view);
+ }
+
+ public List<View> getViews() {
+ return ImmutableList.copyOf(views);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/LogicalPlanAdapter.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/LogicalPlanAdapter.java
index 355cdfdec5..3c9c8d1a1b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/LogicalPlanAdapter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/LogicalPlanAdapter.java
@@ -23,6 +23,7 @@ import org.apache.doris.analysis.OutFileClause;
import org.apache.doris.analysis.Queriable;
import org.apache.doris.analysis.RedirectStatus;
import org.apache.doris.analysis.StatementBase;
+import org.apache.doris.catalog.View;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand;
@@ -43,6 +44,7 @@ public class LogicalPlanAdapter extends StatementBase
implements Queriable {
private final LogicalPlan logicalPlan;
private List<Expr> resultExprs;
private ArrayList<String> colLabels;
+ private List<View> views;
public LogicalPlanAdapter(LogicalPlan logicalPlan, StatementContext
statementContext) {
this.logicalPlan = logicalPlan;
@@ -79,6 +81,10 @@ public class LogicalPlanAdapter extends StatementBase
implements Queriable {
return colLabels;
}
+ public List<View> getViews() {
+ return views;
+ }
+
@Override
public List<Expr> getResultExprs() {
return resultExprs;
@@ -92,6 +98,10 @@ public class LogicalPlanAdapter extends StatementBase
implements Queriable {
this.colLabels = colLabels;
}
+ public void setViews(List<View> views) {
+ this.views = views;
+ }
+
public StatementContext getStatementContext() {
return statementContext;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
index 77538867a2..a6b0b34e8d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -203,6 +203,7 @@ public class BindRelation extends OneAnalysisRuleFactory {
case OLAP:
return makeOlapScan(table, unboundRelation, tableQualifier);
case VIEW:
+ cascadesContext.getStatementContext().addView((View) table);
Plan viewPlan = parseAndAnalyzeView(((View)
table).getDdlSql(), cascadesContext);
return new LogicalSubQueryAlias<>(tableQualifier, viewPlan);
case HMS_EXTERNAL_TABLE:
@@ -251,7 +252,6 @@ public class BindRelation extends OneAnalysisRuleFactory {
CascadesContext viewContext = CascadesContext.initContext(
parentContext.getStatementContext(), parsedViewPlan,
PhysicalProperties.ANY);
viewContext.newAnalyzer().analyze();
-
// we should remove all group expression of the plan which in other
memo, so the groupId would not conflict
return viewContext.getRewritePlan();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java
index 8fcf8811a8..980eb2d829 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java
@@ -59,6 +59,7 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.stream.Collectors;
/**
* Analyze which caching mode a SQL is suitable for
@@ -189,6 +190,10 @@ public class CacheAnalyzer {
cacheMode = innerCheckCacheMode(now);
}
+ public void checkCacheModeForNereids(long now) {
+ cacheMode = innerCheckCacheModeForNereids(now);
+ }
+
private CacheMode innerCheckCacheMode(long now) {
if (!enableCache()) {
LOG.debug("cache is disabled. queryid {}",
DebugUtil.printId(queryId));
@@ -232,7 +237,7 @@ public class CacheAnalyzer {
if (enableSqlCache()
&& (now - latestTable.latestTime) >=
Config.cache_last_version_interval_second * 1000L) {
if (LOG.isDebugEnabled()) {
- LOG.debug("TIME:{},{},{}", now, latestTable.latestTime,
+ LOG.debug("Query cache time:{},{},{}", now,
latestTable.latestTime,
Config.cache_last_version_interval_second * 1000);
}
cache = new SqlCache(this.queryId, this.selectStmt);
@@ -326,7 +331,7 @@ public class CacheAnalyzer {
latestTable.debug();
addAllViewStmt((SetOperationStmt) parsedStmt);
- String allViewExpandStmtListStr = parsedStmt.toSql() + "|" +
StringUtils.join(allViewStmtSet, "|");
+ String allViewExpandStmtListStr = StringUtils.join(allViewStmtSet,
"|");
if (now == 0) {
now = nowtime();
@@ -334,10 +339,10 @@ public class CacheAnalyzer {
if (enableSqlCache()
&& (now - latestTable.latestTime) >=
Config.cache_last_version_interval_second * 1000L) {
if (LOG.isDebugEnabled()) {
- LOG.debug("TIME:{},{},{}", now, latestTable.latestTime,
+ LOG.debug("Query cache time:{},{},{}", now,
latestTable.latestTime,
Config.cache_last_version_interval_second * 1000);
}
- cache = new SqlCache(this.queryId);
+ cache = new SqlCache(this.queryId, parsedStmt.toSql());
((SqlCache) cache).setCacheInfo(this.latestTable,
allViewExpandStmtListStr);
MetricRepo.COUNTER_CACHE_ADDED_SQL.increase(1L);
return CacheMode.Sql;
@@ -383,19 +388,22 @@ public class CacheAnalyzer {
return CacheMode.NoNeed;
}
- String cacheKey = ((LogicalPlanAdapter)
parsedStmt).getStatementContext()
- .getOriginStatement().originStmt.toLowerCase();
+ allViewStmtSet.addAll(((LogicalPlanAdapter) parsedStmt).getViews()
+ .stream().map(view ->
view.getDdlSql()).collect(Collectors.toSet()));
+ String allViewExpandStmtListStr = StringUtils.join(allViewStmtSet,
"|");
+
if (now == 0) {
now = nowtime();
}
if (enableSqlCache()
&& (now - latestTable.latestTime) >=
Config.cache_last_version_interval_second * 1000L) {
if (LOG.isDebugEnabled()) {
- LOG.debug("TIME:{},{},{}", now, latestTable.latestTime,
+ LOG.debug("Query cache time :{},{},{}", now,
latestTable.latestTime,
Config.cache_last_version_interval_second * 1000);
}
- cache = new SqlCache(this.queryId);
- ((SqlCache) cache).setCacheInfo(this.latestTable, cacheKey);
+ cache = new SqlCache(this.queryId, ((LogicalPlanAdapter)
parsedStmt).getStatementContext()
+ .getOriginStatement().originStmt);
+ ((SqlCache) cache).setCacheInfo(this.latestTable,
allViewExpandStmtListStr);
MetricRepo.COUNTER_CACHE_ADDED_SQL.increase(1L);
return CacheMode.Sql;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
index 25b7233770..04f0ed35cb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
@@ -31,12 +31,16 @@ import org.apache.logging.log4j.Logger;
public class SqlCache extends Cache {
private static final Logger LOG = LogManager.getLogger(SqlCache.class);
+ private String originSql;
+
public SqlCache(TUniqueId queryId, SelectStmt selectStmt) {
super(queryId, selectStmt);
}
- public SqlCache(TUniqueId queryId) {
+ // For SetOperationStmt and Nereids
+ public SqlCache(TUniqueId queryId, String originSql) {
super(queryId);
+ this.originSql = originSql;
}
public void setCacheInfo(CacheAnalyzer.CacheTable latestTable, String
allViewExpandStmtListStr) {
@@ -45,11 +49,10 @@ public class SqlCache extends Cache {
}
public String getSqlWithViewStmt() {
- if (selectStmt != null) {
- return selectStmt.toSql() + "|" + allViewExpandStmtListStr;
- } else {
- return allViewExpandStmtListStr;
- }
+ String originSql = selectStmt != null ? selectStmt.toSql() :
this.originSql;
+ String cacheKey = originSql + "|" + allViewExpandStmtListStr;
+ LOG.debug("Cache key: {}", cacheKey);
+ return cacheKey;
}
public InternalService.PFetchCacheResult getCacheData(Status status) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/qe/PartitionCacheTest.java
b/fe/fe-core/src/test/java/org/apache/doris/qe/PartitionCacheTest.java
index 87345bcfa6..5923b57ea2 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/qe/PartitionCacheTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/PartitionCacheTest.java
@@ -45,6 +45,7 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.UserException;
import org.apache.doris.common.jmockit.Deencapsulation;
+import org.apache.doris.common.telemetry.Telemetry;
import org.apache.doris.common.util.SqlParserUtils;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.CatalogMgr;
@@ -54,6 +55,11 @@ import org.apache.doris.mysql.MysqlChannel;
import org.apache.doris.mysql.MysqlSerializer;
import org.apache.doris.mysql.privilege.AccessControllerManager;
import org.apache.doris.mysql.privilege.MockedAuth;
+import org.apache.doris.nereids.NereidsPlanner;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.glue.LogicalPlanAdapter;
+import org.apache.doris.nereids.parser.NereidsParser;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.planner.OlapScanNode;
import org.apache.doris.planner.PlanNodeId;
import org.apache.doris.planner.ScanNode;
@@ -235,6 +241,9 @@ public class PartitionCacheTest {
QueryState state = new QueryState();
channel.reset();
+ SessionVariable sessionVariable = new SessionVariable();
+ Deencapsulation.setField(sessionVariable, "beNumber", 1);
+
new Expectations(channel) {
{
channel.getSerializer();
@@ -296,7 +305,6 @@ public class PartitionCacheTest {
minTimes = 0;
result = dbName;
- SessionVariable sessionVariable = new SessionVariable();
ctx.getSessionVariable();
minTimes = 0;
result = sessionVariable;
@@ -307,6 +315,18 @@ public class PartitionCacheTest {
ctx.getStmtId();
minTimes = 0;
result = 1L;
+
+ ctx.getTracer();
+ minTimes = 0;
+ result = Telemetry.getNoopTracer();
+
+ ctx.getCurrentCatalog();
+ minTimes = 0;
+ result = catalog;
+
+ ctx.getCatalog(anyString);
+ minTimes = 0;
+ result = catalog;
}
};
@@ -466,10 +486,14 @@ public class PartitionCacheTest {
List<Column> column = Lists.newArrayList();
column.add(column1);
+ column.add(column2);
+ column.add(column3);
+ column.add(column4);
+ column.add(column5);
table.setIndexMeta(new Long(2), "test", column, 1, 1,
shortKeyColumnCount, TStorageType.COLUMN,
KeysType.AGG_KEYS);
- Deencapsulation.setField(table, "baseIndexId", 1000);
+ Deencapsulation.setField(table, "baseIndexId", 2);
table.addPartition(part12);
table.addPartition(part13);
@@ -519,6 +543,24 @@ public class PartitionCacheTest {
return node;
}
+ private StatementBase parseSqlByNereids(String sql) {
+ StatementBase stmt = null;
+ try {
+ LogicalPlan plan = new NereidsParser().parseSingle(sql);
+ OriginStatement originStatement = new OriginStatement(sql, 0);
+ StatementContext statementContext = new StatementContext(ctx,
originStatement);
+ NereidsPlanner nereidsPlanner = new
NereidsPlanner(statementContext);
+ LogicalPlanAdapter adapter = new LogicalPlanAdapter(plan,
statementContext);
+ nereidsPlanner.plan(adapter);
+ statementContext.setParsedStatement(adapter);
+ stmt = adapter;
+ } catch (Throwable throwable) {
+ LOG.warn("Part,an_ex={}", throwable);
+ Assert.fail(throwable.getMessage());
+ }
+ return stmt;
+ }
+
private StatementBase parseSql(String sql) {
SqlParser parser = new SqlParser(new SqlScanner(new
StringReader(sql)));
StatementBase parseStmt = null;
@@ -1099,6 +1141,24 @@ public class PartitionCacheTest {
+ "FROM appevent WHERE eventdate>=\"2020-01-12\" and
eventdate<=\"2020-01-14\" GROUP BY eventdate");
}
+ @Test
+ public void testSqlCacheKeyWithViewForNereids() {
+ Env.getCurrentSystemInfo();
+ StatementBase parseStmt = parseSqlByNereids("SELECT * from
testDb.view1");
+ ArrayList<Long> selectedPartitionIds
+ = Lists.newArrayList(20200112L, 20200113L, 20200114L);
+ List<ScanNode> scanNodes =
Lists.newArrayList(createEventScanNode(selectedPartitionIds));
+ CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes);
+ ca.checkCacheModeForNereids(1579053661000L); //2020-1-15 10:01:01
+ Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql);
+
+ SqlCache sqlCache = (SqlCache) ca.getCache();
+ String cacheKey = sqlCache.getSqlWithViewStmt();
+ Assert.assertEquals(cacheKey, "SELECT * from testDb.view1"
+ + "|select eventdate, COUNT(userid) FROM appevent "
+ + "WHERE eventdate>=\"2020-01-12\" and
eventdate<=\"2020-01-14\" GROUP BY eventdate");
+ }
+
@Test
public void testSqlCacheKeyWithSubSelectView() {
Env.getCurrentSystemInfo();
@@ -1125,6 +1185,35 @@ public class PartitionCacheTest {
+ "<= '2020-01-14') origin|select eventdate, userid FROM
appevent");
}
+
+ @Test
+ public void testSqlCacheKeyWithSubSelectViewForNereids() {
+ Env.getCurrentSystemInfo();
+ StatementBase parseStmt = parseSqlByNereids(
+ "select origin.eventdate as eventdate, origin.userid as
userid\n"
+ + "from (\n"
+ + " select view2.eventdate as eventdate,
view2.userid as userid \n"
+ + " from testDb.view2 view2 \n"
+ + " where view2.eventdate >=\"2020-01-12\" and
view2.eventdate <= \"2020-01-14\"\n"
+ + ") origin"
+ );
+ ArrayList<Long> selectedPartitionIds
+ = Lists.newArrayList(20200112L, 20200113L, 20200114L);
+ List<ScanNode> scanNodes =
Lists.newArrayList(createEventScanNode(selectedPartitionIds));
+ CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes);
+ ca.checkCacheModeForNereids(1579053661000L); //2020-1-15 10:01:01
+ Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql);
+
+ SqlCache sqlCache = (SqlCache) ca.getCache();
+ String cacheKey = sqlCache.getSqlWithViewStmt();
+ Assert.assertEquals(cacheKey, "select origin.eventdate as eventdate,
origin.userid as userid\n"
+ + "from (\n"
+ + " select view2.eventdate as eventdate, view2.userid
as userid \n"
+ + " from testDb.view2 view2 \n"
+ + " where view2.eventdate >=\"2020-01-12\" and
view2.eventdate <= \"2020-01-14\"\n"
+ + ") origin" + "|select eventdate, userid FROM appevent");
+ }
+
@Test
public void testPartitionCacheKeyWithView() {
Env.getCurrentSystemInfo();
@@ -1203,6 +1292,25 @@ public class PartitionCacheTest {
+ "eventdate<=\"2020-01-14\" GROUP BY eventdate|select
eventdate, userid FROM appevent");
}
+ @Test
+ public void testSqlCacheKeyWithNestedViewForNereids() {
+ Env.getCurrentSystemInfo();
+ StatementBase parseStmt = parseSqlByNereids("SELECT * from
testDb.view4");
+ ArrayList<Long> selectedPartitionIds
+ = Lists.newArrayList(20200112L, 20200113L, 20200114L);
+ List<ScanNode> scanNodes =
Lists.newArrayList(createEventScanNode(selectedPartitionIds));
+ CacheAnalyzer ca = new CacheAnalyzer(context, parseStmt, scanNodes);
+ ca.checkCacheModeForNereids(1579053661000L); //2020-1-15 10:01:01
+ Assert.assertEquals(ca.getCacheMode(), CacheMode.Sql);
+
+ SqlCache sqlCache = (SqlCache) ca.getCache();
+ String cacheKey = sqlCache.getSqlWithViewStmt();
+ Assert.assertEquals(cacheKey, "SELECT * from testDb.view4"
+ + "|select eventdate, COUNT(userid) FROM view2 "
+ + "WHERE eventdate>=\"2020-01-12\" and
eventdate<=\"2020-01-14\" GROUP BY eventdate"
+ + "|select eventdate, userid FROM appevent");
+ }
+
@Test
public void testCacheLocalViewMultiOperand() {
Env.getCurrentSystemInfo();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
