This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 79eb575d7cc [Improvement](nereids)Support ODBC table for new planner.
(#29129)
79eb575d7cc is described below
commit 79eb575d7cc6292f96d9d68c4668ca77f0e7c8fb
Author: zy-kkk <[email protected]>
AuthorDate: Wed Jan 3 12:51:07 2024 +0800
[Improvement](nereids)Support ODBC table for new planner. (#29129)
---
.../org/apache/doris/nereids/cost/CostModelV1.java | 7 ++
.../org/apache/doris/nereids/cost/CostModelV2.java | 6 ++
.../glue/translator/PhysicalPlanTranslator.java | 26 ++++++
.../doris/nereids/jobs/executor/Rewriter.java | 2 +
.../properties/ChildOutputPropertyDeriver.java | 6 ++
.../org/apache/doris/nereids/rules/RuleSet.java | 2 +
.../org/apache/doris/nereids/rules/RuleType.java | 2 +
.../doris/nereids/rules/analysis/BindRelation.java | 3 +
.../LogicalOdbcScanToPhysicalOdbcScan.java | 42 +++++++++
.../rules/rewrite/PushConjunctsIntoOdbcScan.java | 39 ++++++++
.../doris/nereids/stats/StatsCalculator.java | 13 +++
.../trees/copier/LogicalPlanDeepCopier.java | 13 +++
.../apache/doris/nereids/trees/plans/PlanType.java | 2 +
.../trees/plans/logical/LogicalOdbcScan.java | 104 +++++++++++++++++++++
.../trees/plans/physical/PhysicalOdbcScan.java | 102 ++++++++++++++++++++
.../trees/plans/visitor/RelationVisitor.java | 10 ++
.../doris/planner/external/odbc/OdbcScanNode.java | 44 +++++++--
17 files changed, 417 insertions(+), 6 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
index b2fe0f94578..011afb46fc2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
@@ -35,6 +35,7 @@ import
org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate;
import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
@@ -137,6 +138,12 @@ class CostModelV1 extends PlanVisitor<Cost, PlanContext> {
return CostV1.ofCpu(context.getSessionVariable(),
statistics.getRowCount());
}
+ @Override
+ public Cost visitPhysicalOdbcScan(PhysicalOdbcScan physicalOdbcScan,
PlanContext context) {
+ Statistics statistics = context.getStatisticsWithCheck();
+ return CostV1.ofCpu(context.getSessionVariable(),
statistics.getRowCount());
+ }
+
@Override
public Cost visitPhysicalEsScan(PhysicalEsScan physicalEsScan, PlanContext
context) {
Statistics statistics = context.getStatisticsWithCheck();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV2.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV2.java
index 7f1c6a6c5ac..96afa5123c1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV2.java
@@ -34,6 +34,7 @@ import
org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
@@ -142,6 +143,11 @@ class CostModelV2 extends PlanVisitor<Cost, PlanContext> {
return calculateScanWithoutRF(context.getStatisticsWithCheck());
}
+ @Override
+ public Cost visitPhysicalOdbcScan(PhysicalOdbcScan physicalOdbcScan,
PlanContext context) {
+ return calculateScanWithoutRF(context.getStatisticsWithCheck());
+ }
+
@Override
public Cost visitPhysicalEsScan(PhysicalEsScan physicalEsScan, PlanContext
context) {
return calculateScanWithoutRF(context.getStatisticsWithCheck());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index f8b23935318..da0a58f4f37 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -41,6 +41,7 @@ import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Function.NullableMode;
+import org.apache.doris.catalog.OdbcTable;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
@@ -112,6 +113,7 @@ import
org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect;
import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
@@ -176,6 +178,7 @@ import org.apache.doris.planner.external.MaxComputeScanNode;
import org.apache.doris.planner.external.hudi.HudiScanNode;
import org.apache.doris.planner.external.iceberg.IcebergScanNode;
import org.apache.doris.planner.external.jdbc.JdbcScanNode;
+import org.apache.doris.planner.external.odbc.OdbcScanNode;
import org.apache.doris.planner.external.paimon.PaimonScanNode;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.statistics.StatisticConstants;
@@ -577,6 +580,29 @@ public class PhysicalPlanTranslator extends
DefaultPlanVisitor<PlanFragment, Pla
return planFragment;
}
+ @Override
+ public PlanFragment visitPhysicalOdbcScan(PhysicalOdbcScan odbcScan,
PlanTranslatorContext context) {
+ List<Slot> slots = odbcScan.getOutput();
+ TableIf table = odbcScan.getTable();
+ TupleDescriptor tupleDescriptor = generateTupleDesc(slots, table,
context);
+ OdbcScanNode odbcScanNode = new
OdbcScanNode(odbcScan.translatePlanNodeId(), tupleDescriptor,
+ (OdbcTable) table);
+
odbcScanNode.addConjuncts(translateToLegacyConjuncts(odbcScan.getConjuncts()));
+ Utils.execWithUncheckedException(odbcScanNode::init);
+ context.addScanNode(odbcScanNode);
+ context.getRuntimeTranslator().ifPresent(
+ runtimeFilterGenerator ->
runtimeFilterGenerator.getContext().getTargetListByScan(odbcScan).forEach(
+ expr ->
runtimeFilterGenerator.translateRuntimeFilterTarget(expr, odbcScanNode, context)
+ )
+ );
+ Utils.execWithUncheckedException(odbcScanNode::finalizeForNereids);
+ DataPartition dataPartition = DataPartition.RANDOM;
+ PlanFragment planFragment = new PlanFragment(context.nextFragmentId(),
odbcScanNode, dataPartition);
+ context.addPlanFragment(planFragment);
+ updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), odbcScan);
+ return planFragment;
+ }
+
@Override
public PlanFragment visitPhysicalOlapScan(PhysicalOlapScan olapScan,
PlanTranslatorContext context) {
List<Slot> slots = olapScan.getOutput();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index a97bd61e5c6..e02c299e71f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -95,6 +95,7 @@ import
org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderLimit;
import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderTopN;
import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoEsScan;
import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoJdbcScan;
+import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoOdbcScan;
import org.apache.doris.nereids.rules.rewrite.PushDownCountThroughJoin;
import org.apache.doris.nereids.rules.rewrite.PushDownCountThroughJoinOneSide;
import org.apache.doris.nereids.rules.rewrite.PushDownDistinctThroughJoin;
@@ -344,6 +345,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
new PruneEmptyPartition(),
new PruneFileScanPartition(),
new PushConjunctsIntoJdbcScan(),
+ new PushConjunctsIntoOdbcScan(),
new PushConjunctsIntoEsScan()
)
),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
index 3b07f2bbe98..bed9f4fe2e4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
@@ -47,6 +47,7 @@ import
org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
@@ -149,6 +150,11 @@ public class ChildOutputPropertyDeriver extends
PlanVisitor<PhysicalProperties,
return PhysicalProperties.STORAGE_ANY;
}
+ @Override
+ public PhysicalProperties visitPhysicalOdbcScan(PhysicalOdbcScan odbcScan,
PlanContext context) {
+ return PhysicalProperties.STORAGE_ANY;
+ }
+
@Override
public PhysicalProperties visitPhysicalOlapScan(PhysicalOlapScan olapScan,
PlanContext context) {
return new PhysicalProperties(olapScan.getDistributionSpec());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index b7d07aa134d..92a5bb9ddb1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -69,6 +69,7 @@ import
org.apache.doris.nereids.rules.implementation.LogicalJdbcScanToPhysicalJd
import org.apache.doris.nereids.rules.implementation.LogicalJoinToHashJoin;
import
org.apache.doris.nereids.rules.implementation.LogicalJoinToNestedLoopJoin;
import
org.apache.doris.nereids.rules.implementation.LogicalLimitToPhysicalLimit;
+import
org.apache.doris.nereids.rules.implementation.LogicalOdbcScanToPhysicalOdbcScan;
import
org.apache.doris.nereids.rules.implementation.LogicalOlapScanToPhysicalOlapScan;
import
org.apache.doris.nereids.rules.implementation.LogicalOlapTableSinkToPhysicalOlapTableSink;
import
org.apache.doris.nereids.rules.implementation.LogicalOneRowRelationToPhysicalOneRowRelation;
@@ -165,6 +166,7 @@ public class RuleSet {
.add(new LogicalSchemaScanToPhysicalSchemaScan())
.add(new LogicalFileScanToPhysicalFileScan())
.add(new LogicalJdbcScanToPhysicalJdbcScan())
+ .add(new LogicalOdbcScanToPhysicalOdbcScan())
.add(new LogicalEsScanToPhysicalEsScan())
.add(new LogicalProjectToPhysicalProject())
.add(new LogicalLimitToPhysicalLimit())
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 0ffe509d8bc..cede463c924 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -233,6 +233,7 @@ public enum RuleType {
OLAP_SCAN_PARTITION_PRUNE(RuleTypeClass.REWRITE),
FILE_SCAN_PARTITION_PRUNE(RuleTypeClass.REWRITE),
PUSH_CONJUNCTS_INTO_JDBC_SCAN(RuleTypeClass.REWRITE),
+ PUSH_CONJUNCTS_INTO_ODBC_SCAN(RuleTypeClass.REWRITE),
PUSH_CONJUNCTS_INTO_ES_SCAN(RuleTypeClass.REWRITE),
OLAP_SCAN_TABLET_PRUNE(RuleTypeClass.REWRITE),
PUSH_AGGREGATE_TO_OLAP_SCAN(RuleTypeClass.REWRITE),
@@ -368,6 +369,7 @@ public enum RuleType {
LOGICAL_SCHEMA_SCAN_TO_PHYSICAL_SCHEMA_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_FILE_SCAN_TO_PHYSICAL_FILE_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_JDBC_SCAN_TO_PHYSICAL_JDBC_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
+ LOGICAL_ODBC_SCAN_TO_PHYSICAL_ODBC_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_ES_SCAN_TO_PHYSICAL_ES_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_OLAP_TABLE_SINK_TO_PHYSICAL_OLAP_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_RESULT_SINK_TO_PHYSICAL_RESULT_SINK_RULE(RuleTypeClass.IMPLEMENTATION),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
index ef51de8ddc3..6274956ccea 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -53,6 +53,7 @@ import
org.apache.doris.nereids.trees.plans.logical.LogicalEsScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan;
@@ -242,6 +243,8 @@ public class BindRelation extends OneAnalysisRuleFactory {
case JDBC_EXTERNAL_TABLE:
case JDBC:
return new LogicalJdbcScan(unboundRelation.getRelationId(),
table, tableQualifier);
+ case ODBC:
+ return new LogicalOdbcScan(unboundRelation.getRelationId(),
table, tableQualifier);
case ES_EXTERNAL_TABLE:
return new LogicalEsScan(unboundRelation.getRelationId(),
(EsExternalTable) table, tableQualifier);
default:
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOdbcScanToPhysicalOdbcScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOdbcScanToPhysicalOdbcScan.java
new file mode 100644
index 00000000000..22aea72f736
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOdbcScanToPhysicalOdbcScan.java
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.implementation;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
+
+import java.util.Optional;
+
+/**
+ * Implementation rule that convert logical OdbcScan to physical OdbcScan.
+ */
+public class LogicalOdbcScanToPhysicalOdbcScan extends
OneImplementationRuleFactory {
+ @Override
+ public Rule build() {
+ return logicalOdbcScan().then(odbcScan ->
+ new PhysicalOdbcScan(
+ odbcScan.getRelationId(),
+ odbcScan.getTable(),
+ odbcScan.getQualifier(),
+ Optional.empty(),
+ odbcScan.getLogicalProperties(),
+ odbcScan.getConjuncts())
+ ).toRule(RuleType.LOGICAL_ODBC_SCAN_TO_PHYSICAL_ODBC_SCAN_RULE);
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoOdbcScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoOdbcScan.java
new file mode 100644
index 00000000000..8a1ba4d1b67
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoOdbcScan.java
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
+
+/**
+ * Rewrite odbc plan to set the conjuncts.
+ */
+public class PushConjunctsIntoOdbcScan extends OneRewriteRuleFactory {
+
+ @Override
+ public Rule build() {
+ return logicalFilter(logicalOdbcScan()).thenApply(ctx -> {
+ LogicalFilter<LogicalOdbcScan> filter = ctx.root;
+ LogicalOdbcScan scan = filter.child();
+ LogicalOdbcScan rewrittenScan =
scan.withConjuncts(filter.getConjuncts());
+ return new LogicalFilter<>(filter.getConjuncts(), rewrittenScan);
+ }).toRule(RuleType.PUSH_CONJUNCTS_INTO_ODBC_SCAN);
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index 84954117459..a475d716fdb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -67,6 +67,7 @@ import
org.apache.doris.nereids.trees.plans.logical.LogicalIntersect;
import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
@@ -98,6 +99,7 @@ import
org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect;
import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
@@ -317,6 +319,12 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
return computeCatalogRelation(jdbcScan);
}
+ @Override
+ public Statistics visitLogicalOdbcScan(LogicalOdbcScan odbcScan, Void
context) {
+ odbcScan.getExpressions();
+ return computeCatalogRelation(odbcScan);
+ }
+
@Override
public Statistics visitLogicalEsScan(LogicalEsScan esScan, Void context) {
esScan.getExpressions();
@@ -460,6 +468,11 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
return computeCatalogRelation(jdbcScan);
}
+ @Override
+ public Statistics visitPhysicalOdbcScan(PhysicalOdbcScan odbcScan, Void
context) {
+ return computeCatalogRelation(odbcScan);
+ }
+
@Override
public Statistics visitPhysicalEsScan(PhysicalEsScan esScan, Void context)
{
return computeCatalogRelation(esScan);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
index 11354221875..3ae47c3a3ed 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
@@ -49,6 +49,7 @@ import
org.apache.doris.nereids.trees.plans.logical.LogicalIntersect;
import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
@@ -238,6 +239,18 @@ public class LogicalPlanDeepCopier extends
DefaultPlanRewriter<DeepCopierContext
return newJdbcScan;
}
+ @Override
+ public Plan visitLogicalOdbcScan(LogicalOdbcScan odbcScan,
DeepCopierContext context) {
+ if
(context.getRelationReplaceMap().containsKey(odbcScan.getRelationId())) {
+ return
context.getRelationReplaceMap().get(odbcScan.getRelationId());
+ }
+ LogicalOdbcScan newOdbcScan = new
LogicalOdbcScan(StatementScopeIdGenerator.newRelationId(),
+ odbcScan.getTable(), odbcScan.getQualifier());
+ updateReplaceMapWithOutput(odbcScan, newOdbcScan,
context.exprIdReplaceMap);
+ context.putRelation(odbcScan.getRelationId(), newOdbcScan);
+ return newOdbcScan;
+ }
+
@Override
public Plan visitLogicalEsScan(LogicalEsScan esScan, DeepCopierContext
context) {
if
(context.getRelationReplaceMap().containsKey(esScan.getRelationId())) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 3fe662b7baf..f1eae216630 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -33,6 +33,7 @@ public enum PlanType {
LOGICAL_EMPTY_RELATION,
LOGICAL_ES_SCAN,
LOGICAL_JDBC_SCAN,
+ LOGICAL_ODBC_SCAN,
LOGICAL_OLAP_SCAN,
LOGICAL_ONE_ROW_RELATION,
LOGICAL_SCHEMA_SCAN,
@@ -83,6 +84,7 @@ public enum PlanType {
PHYSICAL_ES_SCAN,
PHYSICAL_FILE_SCAN,
PHYSICAL_JDBC_SCAN,
+ PHYSICAL_ODBC_SCAN,
PHYSICAL_ONE_ROW_RELATION,
PHYSICAL_OLAP_SCAN,
PHYSICAL_SCHEMA_SCAN,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java
new file mode 100644
index 00000000000..840cb1113a4
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java
@@ -0,0 +1,104 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.logical;
+
+import org.apache.doris.catalog.OdbcTable;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.RelationId;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.Utils;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Logical scan for external odbc table.
+ */
+public class LogicalOdbcScan extends LogicalCatalogRelation {
+
+ private final Set<Expression> conjuncts;
+
+ public LogicalOdbcScan(RelationId id, TableIf table, List<String>
qualifier,
+ Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties,
+ Set<Expression> conjuncts) {
+ super(id, PlanType.LOGICAL_ODBC_SCAN, table, qualifier,
+ groupExpression, logicalProperties);
+ this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts,
"conjuncts should not be null"));
+ }
+
+ public LogicalOdbcScan(RelationId id, TableIf table, List<String>
qualifier) {
+ this(id, table, qualifier, Optional.empty(), Optional.empty(),
ImmutableSet.of());
+ }
+
+ @Override
+ public TableIf getTable() {
+ Preconditions.checkArgument(table instanceof OdbcTable,
+ String.format("Table %s is not OdbcTable", table.getName()));
+ return table;
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toSqlString("LogicalOdbcScan",
+ "qualified", qualifiedName(),
+ "output", getOutput()
+ );
+ }
+
+ @Override
+ public LogicalOdbcScan withGroupExpression(Optional<GroupExpression>
groupExpression) {
+ return new LogicalOdbcScan(relationId, table, qualifier,
groupExpression,
+ Optional.of(getLogicalProperties()), conjuncts);
+ }
+
+ public LogicalOdbcScan withConjuncts(Set<Expression> conjuncts) {
+ return new LogicalOdbcScan(relationId, table, qualifier,
groupExpression,
+ Optional.of(getLogicalProperties()), conjuncts);
+ }
+
+ @Override
+ public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression>
groupExpression,
+ Optional<LogicalProperties> logicalProperties, List<Plan>
children) {
+ return new LogicalOdbcScan(relationId, table, qualifier,
groupExpression, logicalProperties, conjuncts);
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitLogicalOdbcScan(this, context);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return super.equals(o) && Objects.equals(conjuncts, ((LogicalOdbcScan)
o).conjuncts);
+ }
+
+ public Set<Expression> getConjuncts() {
+ return this.conjuncts;
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java
new file mode 100644
index 00000000000..f63b5a8d420
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java
@@ -0,0 +1,102 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.physical;
+
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.RelationId;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.statistics.Statistics;
+
+import com.google.common.collect.ImmutableSet;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Physical odbc scan for odbc table.
+ */
+public class PhysicalOdbcScan extends PhysicalCatalogRelation {
+
+ private final Set<Expression> conjuncts;
+
+ /**
+ * Constructor for PhysicalOdbcScan.
+ */
+ public PhysicalOdbcScan(RelationId id, TableIf table, List<String>
qualifier,
+ Optional<GroupExpression> groupExpression, LogicalProperties
logicalProperties, Set<Expression> conjuncts) {
+ this(id, table, qualifier, groupExpression, logicalProperties,
+ null, null, conjuncts);
+ }
+
+ /**
+ * Constructor for PhysicalOdbcScan.
+ */
+ public PhysicalOdbcScan(RelationId id, TableIf table, List<String>
qualifier,
+ Optional<GroupExpression> groupExpression,
+ LogicalProperties logicalProperties, PhysicalProperties
physicalProperties, Statistics statistics,
+ Set<Expression> conjuncts) {
+ super(id, PlanType.PHYSICAL_ODBC_SCAN, table, qualifier,
groupExpression,
+ logicalProperties, physicalProperties, statistics);
+ this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts,
"conjuncts should not be null"));
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toSqlString("PhysicalOdbcScan",
+ "qualified", Utils.qualifiedName(qualifier, table.getName()),
+ "output", getOutput(),
+ "stats", statistics
+ );
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitPhysicalOdbcScan(this, context);
+ }
+
+ @Override
+ public PhysicalOdbcScan withGroupExpression(Optional<GroupExpression>
groupExpression) {
+ return new PhysicalOdbcScan(relationId, table, qualifier,
groupExpression, getLogicalProperties(), conjuncts);
+ }
+
+ @Override
+ public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression>
groupExpression,
+ Optional<LogicalProperties> logicalProperties, List<Plan>
children) {
+ return new PhysicalOdbcScan(relationId, table, qualifier,
groupExpression, logicalProperties.get(), conjuncts);
+ }
+
+ @Override
+ public PhysicalOdbcScan withPhysicalPropertiesAndStats(PhysicalProperties
physicalProperties,
+ Statistics
statistics) {
+ return new PhysicalOdbcScan(relationId, table, qualifier,
groupExpression,
+ getLogicalProperties(), physicalProperties, statistics,
conjuncts);
+ }
+
+ public Set<Expression> getConjuncts() {
+ return this.conjuncts;
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java
index 65a03505021..adfe4991665 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java
@@ -25,6 +25,7 @@ import
org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalEsScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
@@ -35,6 +36,7 @@ import
org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalEsScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalFileScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
@@ -90,6 +92,10 @@ public interface RelationVisitor<R, C> {
return visitLogicalRelation(jdbcScan, context);
}
+ default R visitLogicalOdbcScan(LogicalOdbcScan odbcScan, C context) {
+ return visitLogicalRelation(odbcScan, context);
+ }
+
default R visitLogicalOlapScan(LogicalOlapScan olapScan, C context) {
return visitLogicalRelation(olapScan, context);
}
@@ -131,6 +137,10 @@ public interface RelationVisitor<R, C> {
return visitPhysicalRelation(jdbcScan, context);
}
+ default R visitPhysicalOdbcScan(PhysicalOdbcScan odbcScan, C context) {
+ return visitPhysicalRelation(odbcScan, context);
+ }
+
default R visitPhysicalOlapScan(PhysicalOlapScan olapScan, C context) {
return visitPhysicalRelation(olapScan, context);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java
index 68950ebab83..d827ec4a4eb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java
@@ -22,6 +22,7 @@ import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.ExprSubstitutionMap;
import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.SlotDescriptor;
+import org.apache.doris.analysis.SlotId;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.catalog.Column;
@@ -31,6 +32,7 @@ import org.apache.doris.catalog.OdbcTable;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.UserException;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
import org.apache.doris.planner.PlanNodeId;
import org.apache.doris.planner.external.ExternalScanNode;
import org.apache.doris.planner.external.jdbc.JdbcScanNode;
@@ -52,6 +54,7 @@ import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.List;
+import java.util.Set;
/**
* Full scan of an ODBC table.
@@ -78,6 +81,22 @@ public class OdbcScanNode extends ExternalScanNode {
this.tbl = tbl;
}
+ @Override
+ public void init(Analyzer analyzer) throws UserException {
+ super.init(analyzer);
+ }
+
+ /**
+ * Used for Nereids. Should NOT use this function in anywhere else.
+ */
+ @Override
+ public void init() throws UserException {
+ super.init();
+ numNodes = numNodes <= 0 ? 1 : numNodes;
+
StatsRecursiveDerive.getStatsRecursiveDerive().statsRecursiveDerive(this);
+ cardinality = (long) statsDeriveResult.getRowCount();
+ }
+
@Override
protected String debugString() {
MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper(this);
@@ -87,11 +106,24 @@ public class OdbcScanNode extends ExternalScanNode {
@Override
public void finalize(Analyzer analyzer) throws UserException {
// Convert predicates to Odbc columns and filters.
- createOdbcColumns(analyzer);
- createOdbcFilters(analyzer);
+ createOdbcColumns();
+ createOdbcFilters();
+ createScanRangeLocations();
+ }
+
+ @Override
+ public void finalizeForNereids() throws UserException {
+ createOdbcColumns();
+ createOdbcFilters();
createScanRangeLocations();
}
+ @Override
+ public void updateRequiredSlots(PlanTranslatorContext context, Set<SlotId>
requiredByProjectSlotIdSet)
+ throws UserException {
+ createOdbcColumns();
+ }
+
@Override
protected void createScanRangeLocations() throws UserException {
scanRangeLocations =
Lists.newArrayList(createSingleScanRangeLocations(backendPolicy));
@@ -152,7 +184,8 @@ public class OdbcScanNode extends ExternalScanNode {
return sql.toString();
}
- private void createOdbcColumns(Analyzer analyzer) {
+ private void createOdbcColumns() {
+ columns.clear();
for (SlotDescriptor slot : desc.getSlots()) {
if (!slot.isMaterialized()) {
continue;
@@ -161,16 +194,15 @@ public class OdbcScanNode extends ExternalScanNode {
columns.add(JdbcTable.databaseProperName(odbcType, col.getName()));
}
// this happens when count(*)
- if (0 == columns.size()) {
+ if (columns.isEmpty()) {
columns.add("*");
}
}
// We convert predicates of the form <slotref> op <constant> to Odbc
filters
- private void createOdbcFilters(Analyzer analyzer) {
+ private void createOdbcFilters() {
if (conjuncts.isEmpty()) {
return;
-
}
List<SlotRef> slotRefs = Lists.newArrayList();
Expr.collectList(conjuncts, SlotRef.class, slotRefs);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
