This is an automated email from the ASF dual-hosted git repository.
jakevin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ce3c60d110a [refactor](Nereids): avoid ConnectContext.get() ASAP to
improve proformance (#29952)
ce3c60d110a is described below
commit ce3c60d110af82c72c64a6ec9e9b902bdafbf5e6
Author: jakevin <[email protected]>
AuthorDate: Mon Jan 15 10:00:47 2024 +0800
[refactor](Nereids): avoid ConnectContext.get() ASAP to improve proformance
(#29952)
---
.../org/apache/doris/nereids/cost/CostModelV1.java | 2 +-
.../glue/translator/PhysicalPlanTranslator.java | 7 +++--
.../glue/translator/PlanTranslatorContext.java | 4 +++
.../java/org/apache/doris/nereids/jobs/Job.java | 5 +++
.../nereids/jobs/cascades/CostAndEnforcerJob.java | 5 ---
.../org/apache/doris/nereids/metrics/Event.java | 7 +++--
.../doris/nereids/minidump/MinidumpUtils.java | 36 ++++++++++++----------
.../expression/rules/FoldConstantRuleOnFE.java | 2 +-
.../doris/nereids/rules/rewrite/CTEInline.java | 7 +++--
.../org/apache/doris/nereids/trees/plans/Plan.java | 2 +-
.../plans/commands/InsertIntoTableCommand.java | 4 +--
.../trees/plans/commands/info/AlterMTMVInfo.java | 4 +--
.../trees/plans/commands/info/CreateMTMVInfo.java | 4 +--
.../trees/plans/commands/info/DropMTMVInfo.java | 4 +--
.../trees/plans/commands/info/RefreshMTMVInfo.java | 4 +--
15 files changed, 53 insertions(+), 44 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
index 011afb46fc2..d56a9d8be2a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
@@ -75,7 +75,7 @@ class CostModelV1 extends PlanVisitor<Cost, PlanContext> {
beNumber = sessionVariable.getBeNumberForTest();
parallelInstance = 8;
} else {
- beNumber = Math.max(1,
ConnectContext.get().getEnv().getClusterInfo().getBackendsNumber(true));
+ beNumber = Math.max(1,
connectContext.getEnv().getClusterInfo().getBackendsNumber(true));
parallelInstance = Math.max(1,
connectContext.getSessionVariable().getParallelExecInstanceNum());
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 9f8f82ad679..6270c5795b3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -179,12 +179,12 @@ import
org.apache.doris.planner.external.iceberg.IcebergScanNode;
import org.apache.doris.planner.external.jdbc.JdbcScanNode;
import org.apache.doris.planner.external.odbc.OdbcScanNode;
import org.apache.doris.planner.external.paimon.PaimonScanNode;
-import org.apache.doris.qe.ConnectContext;
import org.apache.doris.statistics.StatisticConstants;
import org.apache.doris.tablefunction.TableValuedFunctionIf;
import org.apache.doris.thrift.TFetchOption;
import org.apache.doris.thrift.TPartitionType;
import org.apache.doris.thrift.TPushAggOp;
+import org.apache.doris.thrift.TResultSinkType;
import org.apache.doris.thrift.TRuntimeFilterType;
import com.google.common.base.Preconditions;
@@ -363,8 +363,9 @@ public class PhysicalPlanTranslator extends
DefaultPlanVisitor<PlanFragment, Pla
public PlanFragment visitPhysicalResultSink(PhysicalResultSink<? extends
Plan> physicalResultSink,
PlanTranslatorContext context) {
PlanFragment planFragment = physicalResultSink.child().accept(this,
context);
- planFragment.setSink(new ResultSink(planFragment.getPlanRoot().getId(),
- ConnectContext.get().getResultSinkType()));
+ TResultSinkType resultSinkType = context.getConnectContext() != null ?
context.getConnectContext()
+ .getResultSinkType() : null;
+ planFragment.setSink(new
ResultSink(planFragment.getPlanRoot().getId(), resultSinkType));
return planFragment;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
index 8f273d4ec08..77b386c79e3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
@@ -151,6 +151,10 @@ public class PlanTranslatorContext {
return connectContext == null ? null :
connectContext.getSessionVariable();
}
+ public ConnectContext getConnectContext() {
+ return connectContext;
+ }
+
public Set<ScanNode> getScanNodeWithUnknownColumnStats() {
return statsUnknownColumnsMap.keySet();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/Job.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/Job.java
index 0a355efd390..a9739cbb9e2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/Job.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/Job.java
@@ -33,6 +33,7 @@ import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleSet;
import org.apache.doris.nereids.trees.expressions.CTEId;
import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.statistics.Statistics;
@@ -84,6 +85,10 @@ public abstract class Job implements TracerSupplier {
return once;
}
+ public ConnectContext getConnectContext() {
+ return context.getCascadesContext().getConnectContext();
+ }
+
public abstract void execute();
public EventProducer getEventTracer() {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java
index 8126a0aee01..101b3ca6670 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java
@@ -31,7 +31,6 @@ import
org.apache.doris.nereids.properties.ChildrenPropertiesRegulator;
import org.apache.doris.nereids.properties.EnforceMissingPropertiesHelper;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.properties.RequestPropertyDeriver;
-import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.SessionVariable;
import com.google.common.collect.Lists;
@@ -81,10 +80,6 @@ public class CostAndEnforcerJob extends Job implements
Cloneable {
this.groupExpression = groupExpression;
}
- private ConnectContext getConnectContext() {
- return context.getCascadesContext().getConnectContext();
- }
-
private SessionVariable getSessionVariable() {
return
context.getCascadesContext().getConnectContext().getSessionVariable();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/metrics/Event.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/metrics/Event.java
index 1a310766582..43061c4879f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/metrics/Event.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/metrics/Event.java
@@ -35,9 +35,10 @@ public abstract class Event implements Cloneable {
}
protected static boolean checkConnectContext(Class<? extends Event>
targetClass) {
- return ConnectContext.get() != null
- &&
ConnectContext.get().getSessionVariable().isEnableNereidsTrace()
- &&
ConnectContext.get().getSessionVariable().getParsedNereidsEventMode().contains(targetClass);
+ ConnectContext connectContext = ConnectContext.get();
+ return connectContext != null
+ && connectContext.getSessionVariable().isEnableNereidsTrace()
+ &&
connectContext.getSessionVariable().getParsedNereidsEventMode().contains(targetClass);
}
public final String toJson() {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java
index b1a6865c901..4187e1fc4ce 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java
@@ -290,14 +290,14 @@ public class MinidumpUtils {
* serialize output plan to dump file and persistent into disk
*/
public static void serializeOutputToDumpFile(Plan resultPlan) {
- if (ConnectContext.get().getSessionVariable().isPlayNereidsDump()
- ||
!ConnectContext.get().getSessionVariable().isEnableMinidump()) {
+ ConnectContext connectContext = ConnectContext.get();
+ if (connectContext.getSessionVariable().isPlayNereidsDump()
+ || !connectContext.getSessionVariable().isEnableMinidump()) {
return;
}
- ConnectContext.get().getMinidump().put("ResultPlan", ((AbstractPlan)
resultPlan).toJson());
- if (ConnectContext.get().getSessionVariable().isEnableMinidump()) {
- saveMinidumpString(ConnectContext.get().getMinidump(),
- DebugUtil.printId(ConnectContext.get().queryId()));
+ connectContext.getMinidump().put("ResultPlan", ((AbstractPlan)
resultPlan).toJson());
+ if (connectContext.getSessionVariable().isEnableMinidump()) {
+ saveMinidumpString(connectContext.getMinidump(),
DebugUtil.printId(connectContext.queryId()));
}
}
@@ -433,15 +433,16 @@ public class MinidumpUtils {
* implementation of interface serializeInputsToDumpFile
*/
private static JSONObject serializeInputs(Plan parsedPlan, List<TableIf>
tables) throws IOException {
+ ConnectContext connectContext = ConnectContext.get();
// Create a JSON object
JSONObject jsonObj = new JSONObject();
- jsonObj.put("Sql",
ConnectContext.get().getStatementContext().getOriginStatement().originStmt);
+ jsonObj.put("Sql",
connectContext.getStatementContext().getOriginStatement().originStmt);
// add session variable
- int beNumber =
ConnectContext.get().getEnv().getClusterInfo().getBackendsNumber(true);
- ConnectContext.get().getSessionVariable().setBeNumberForTest(beNumber);
- jsonObj.put("SessionVariable",
serializeChangedSessionVariable(ConnectContext.get().getSessionVariable()));
+ int beNumber =
connectContext.getEnv().getClusterInfo().getBackendsNumber(true);
+ connectContext.getSessionVariable().setBeNumberForTest(beNumber);
+ jsonObj.put("SessionVariable",
serializeChangedSessionVariable(connectContext.getSessionVariable()));
// add tables
- jsonObj.put("DbName", ConnectContext.get().getDatabase());
+ jsonObj.put("DbName", connectContext.getDatabase());
JSONArray tablesJson = serializeTables(tables);
jsonObj.put("Tables", tablesJson);
// add colocate table index, used to indicate grouping of table
distribution
@@ -461,19 +462,20 @@ public class MinidumpUtils {
* @throws IOException this will write to disk, so io exception should be
dealed with
*/
public static void serializeInputsToDumpFile(Plan parsedPlan,
List<TableIf> tables) throws IOException {
+ ConnectContext connectContext = ConnectContext.get();
// when playing minidump file, we do not save input again.
- if (ConnectContext.get().getSessionVariable().isPlayNereidsDump()
- ||
!ConnectContext.get().getSessionVariable().isEnableMinidump()) {
+ if (connectContext.getSessionVariable().isPlayNereidsDump()
+ || !connectContext.getSessionVariable().isEnableMinidump()) {
return;
}
- if
(!ConnectContext.get().getSessionVariable().getMinidumpPath().equals("")) {
- MinidumpUtils.DUMP_PATH =
ConnectContext.get().getSessionVariable().getMinidumpPath();
+ if (!connectContext.getSessionVariable().getMinidumpPath().equals(""))
{
+ MinidumpUtils.DUMP_PATH =
connectContext.getSessionVariable().getMinidumpPath();
} else {
-
ConnectContext.get().getSessionVariable().setMinidumpPath("defaultMinidumpPath");
+
connectContext.getSessionVariable().setMinidumpPath("defaultMinidumpPath");
}
MinidumpUtils.init();
- ConnectContext.get().setMinidump(serializeInputs(parsedPlan, tables));
+ connectContext.setMinidump(serializeInputs(parsedPlan, tables));
}
/**
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java
index 9988a34da42..8d56dde938e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java
@@ -121,7 +121,7 @@ public class FoldConstantRuleOnFE extends
AbstractExpressionRewriteRule {
@Override
public Expression visitEncryptKeyRef(EncryptKeyRef encryptKeyRef,
ExpressionRewriteContext context) {
String dbName = encryptKeyRef.getDbName();
- ConnectContext connectContext = ConnectContext.get();
+ ConnectContext connectContext =
context.cascadesContext.getConnectContext();
if (Strings.isNullOrEmpty(dbName)) {
dbName = connectContext.getDatabase();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CTEInline.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CTEInline.java
index 6b4de6d0f9a..e0c2c3da171 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CTEInline.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CTEInline.java
@@ -78,9 +78,10 @@ public class CTEInline extends
DefaultPlanRewriter<LogicalCTEProducer<?>> implem
}
return false;
});
- if
(ConnectContext.get().getSessionVariable().getEnablePipelineEngine()
- &&
ConnectContext.get().getSessionVariable().enableCTEMaterialize
- && consumers.size() >
ConnectContext.get().getSessionVariable().inlineCTEReferencedThreshold) {
+ ConnectContext connectContext = ConnectContext.get();
+ if (connectContext.getSessionVariable().getEnablePipelineEngine()
+ && connectContext.getSessionVariable().enableCTEMaterialize
+ && consumers.size() >
connectContext.getSessionVariable().inlineCTEReferencedThreshold) {
// not inline
Plan right = cteAnchor.right().accept(this, null);
return cteAnchor.withChildren(cteAnchor.left(), right);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java
index aa87de0fcab..f31c6e97a03 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java
@@ -165,7 +165,7 @@ public interface Plan extends TreeNode<Plan> {
StringBuilder builder = new StringBuilder();
String me = this.getClass().getSimpleName();
String prefixTail = "";
- if (!
ConnectContext.get().getSessionVariable().getIgnoreShapePlanNodes().contains(me))
{
+ if
(!ConnectContext.get().getSessionVariable().getIgnoreShapePlanNodes().contains(me))
{
builder.append(prefix).append(shapeInfo()).append("\n");
prefixTail += "--";
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertIntoTableCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertIntoTableCommand.java
index d50ee2096ca..c1ec90fff73 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertIntoTableCommand.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertIntoTableCommand.java
@@ -251,8 +251,8 @@ public class InsertIntoTableCommand extends Command
implements ForwardWithSync,
|| ctx.getSessionVariable().isEnableUniqueKeyPartialUpdate()) {
return false;
}
- return ConnectContext.get().getSessionVariable().getSqlMode() !=
SqlModeHelper.MODE_NO_BACKSLASH_ESCAPES
- && physicalOlapTableSink.getTargetTable() instanceof OlapTable
&& !ConnectContext.get().isTxnModel()
+ return ctx.getSessionVariable().getSqlMode() !=
SqlModeHelper.MODE_NO_BACKSLASH_ESCAPES
+ && physicalOlapTableSink.getTargetTable() instanceof OlapTable
&& !ctx.isTxnModel()
&& sink.getFragment().getPlanRoot() instanceof UnionNode &&
physicalOlapTableSink.getPartitionIds()
.isEmpty() &&
physicalOlapTableSink.getTargetTable().getTableProperty().getUseSchemaLightChange();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterMTMVInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterMTMVInfo.java
index 542ebca6ac0..a2e96ee298d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterMTMVInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterMTMVInfo.java
@@ -44,10 +44,10 @@ public abstract class AlterMTMVInfo {
*/
public void analyze(ConnectContext ctx) throws AnalysisException {
mvName.analyze(ctx);
- if
(!Env.getCurrentEnv().getAccessManager().checkTblPriv(ConnectContext.get(),
mvName.getDb(),
+ if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(ctx,
mvName.getDb(),
mvName.getTbl(), PrivPredicate.ALTER)) {
String message =
ErrorCode.ERR_TABLEACCESS_DENIED_ERROR.formatErrorMsg("ALTER",
- ConnectContext.get().getQualifiedUser(),
ConnectContext.get().getRemoteIP(),
+ ctx.getQualifiedUser(), ctx.getRemoteIP(),
mvName.getDb() + ": " + mvName.getTbl());
throw new AnalysisException(message);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
index c7483a2c70c..40fc06a4434 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
@@ -140,10 +140,10 @@ public class CreateMTMVInfo {
public void analyze(ConnectContext ctx) {
// analyze table name
mvName.analyze(ctx);
- if
(!Env.getCurrentEnv().getAccessManager().checkTblPriv(ConnectContext.get(),
mvName.getDb(),
+ if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(ctx,
mvName.getDb(),
mvName.getTbl(), PrivPredicate.CREATE)) {
String message =
ErrorCode.ERR_TABLEACCESS_DENIED_ERROR.formatErrorMsg("CREATE",
- ConnectContext.get().getQualifiedUser(),
ConnectContext.get().getRemoteIP(),
+ ctx.getQualifiedUser(), ctx.getRemoteIP(),
mvName.getDb() + ": " + mvName.getTbl());
throw new AnalysisException(message);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropMTMVInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropMTMVInfo.java
index 650df2c96c3..c64fb7cf67d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropMTMVInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropMTMVInfo.java
@@ -46,10 +46,10 @@ public class DropMTMVInfo {
*/
public void analyze(ConnectContext ctx) {
mvName.analyze(ctx);
- if
(!Env.getCurrentEnv().getAccessManager().checkTblPriv(ConnectContext.get(),
mvName.getDb(),
+ if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(ctx,
mvName.getDb(),
mvName.getTbl(), PrivPredicate.DROP)) {
String message =
ErrorCode.ERR_TABLEACCESS_DENIED_ERROR.formatErrorMsg("DROP",
- ConnectContext.get().getQualifiedUser(),
ConnectContext.get().getRemoteIP(),
+ ctx.getQualifiedUser(), ctx.getRemoteIP(),
mvName.getDb() + ": " + mvName.getTbl());
throw new AnalysisException(message);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java
index 315b30ae87d..f280e867819 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java
@@ -56,10 +56,10 @@ public class RefreshMTMVInfo {
*/
public void analyze(ConnectContext ctx) {
mvName.analyze(ctx);
- if
(!Env.getCurrentEnv().getAccessManager().checkTblPriv(ConnectContext.get(),
mvName.getDb(),
+ if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(ctx,
mvName.getDb(),
mvName.getTbl(), PrivPredicate.CREATE)) {
String message =
ErrorCode.ERR_TABLEACCESS_DENIED_ERROR.formatErrorMsg("CREATE",
- ConnectContext.get().getQualifiedUser(),
ConnectContext.get().getRemoteIP(),
+ ctx.getQualifiedUser(), ctx.getRemoteIP(),
mvName.getDb() + ": " + mvName.getTbl());
throw new AnalysisException(message);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
