This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 6644eb40a2 [Improvement](multi catalog) Support Iceberg, Paimon and
MaxCompute table in nereids. (#22338) (#22376)
6644eb40a2 is described below
commit 6644eb40a27cb00dfd5ae7e040c48a6cddd1714e
Author: Jibing-Li <[email protected]>
AuthorDate: Sun Jul 30 21:13:54 2023 +0800
[Improvement](multi catalog) Support Iceberg, Paimon and MaxCompute table
in nereids. (#22338) (#22376)
---
.../glue/translator/PhysicalPlanTranslator.java | 4 ++
.../doris/nereids/rules/analysis/BindRelation.java | 8 ++-
.../doris/planner/external/MaxComputeScanNode.java | 4 ++
.../test_external_catalog_icebergv2_nereids.out | 74 +++++++++++++++++++
.../test_external_catalog_icebergv2_nereids.groovy | 84 ++++++++++++++++++++++
5 files changed, 172 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 8f9b336585..4b460e944d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -48,6 +48,7 @@ import org.apache.doris.catalog.external.ExternalTable;
import org.apache.doris.catalog.external.HMSExternalTable;
import org.apache.doris.catalog.external.IcebergExternalTable;
import org.apache.doris.catalog.external.JdbcExternalTable;
+import org.apache.doris.catalog.external.MaxComputeExternalTable;
import org.apache.doris.catalog.external.PaimonExternalTable;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.Util;
@@ -157,6 +158,7 @@ import org.apache.doris.planner.SortNode;
import org.apache.doris.planner.TableFunctionNode;
import org.apache.doris.planner.UnionNode;
import org.apache.doris.planner.external.HiveScanNode;
+import org.apache.doris.planner.external.MaxComputeScanNode;
import org.apache.doris.planner.external.hudi.HudiScanNode;
import org.apache.doris.planner.external.iceberg.IcebergScanNode;
import org.apache.doris.planner.external.jdbc.JdbcScanNode;
@@ -403,6 +405,8 @@ public class PhysicalPlanTranslator extends
DefaultPlanVisitor<PlanFragment, Pla
scanNode = new IcebergScanNode(context.nextPlanNodeId(),
tupleDescriptor, false);
} else if (table instanceof PaimonExternalTable) {
scanNode = new PaimonScanNode(context.nextPlanNodeId(),
tupleDescriptor, false);
+ } else if (table instanceof MaxComputeExternalTable) {
+ scanNode = new MaxComputeScanNode(context.nextPlanNodeId(),
tupleDescriptor, false);
} else {
throw new RuntimeException("do not support table type " +
table.getType());
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
index e23078ce54..0775fc7acb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -23,6 +23,7 @@ import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.View;
import org.apache.doris.catalog.external.EsExternalTable;
+import org.apache.doris.catalog.external.ExternalTable;
import org.apache.doris.catalog.external.HMSExternalTable;
import org.apache.doris.common.Config;
import org.apache.doris.common.util.Util;
@@ -212,8 +213,11 @@ public class BindRelation extends OneAnalysisRuleFactory {
return new LogicalSubQueryAlias<>(tableQualifier,
hiveViewPlan);
}
}
- return new LogicalFileScan(RelationUtil.newRelationId(),
- (HMSExternalTable) table, ImmutableList.of(dbName));
+ return new LogicalFileScan(RelationUtil.newRelationId(),
(HMSExternalTable) table, tableQualifier);
+ case ICEBERG_EXTERNAL_TABLE:
+ case PAIMON_EXTERNAL_TABLE:
+ case MAX_COMPUTE_EXTERNAL_TABLE:
+ return new LogicalFileScan(RelationUtil.newRelationId(),
(ExternalTable) table, tableQualifier);
case SCHEMA:
return new LogicalSchemaScan(RelationUtil.newRelationId(),
table, ImmutableList.of(dbName));
case JDBC_EXTERNAL_TABLE:
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java
index 11e9bafd86..1030a67a30 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java
@@ -44,6 +44,10 @@ public class MaxComputeScanNode extends FileQueryScanNode {
private final MaxComputeExternalCatalog catalog;
public static final int MIN_SPLIT_SIZE = 4096;
+ public MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc, boolean
needCheckColumnPriv) {
+ this(id, desc, "MCScanNode", StatisticalType.MAX_COMPUTE_SCAN_NODE,
needCheckColumnPriv);
+ }
+
public MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc, String
planNodeName,
StatisticalType statisticalType, boolean
needCheckColumnPriv) {
super(id, desc, planNodeName, statisticalType, needCheckColumnPriv);
diff --git
a/regression-test/data/external_table_emr_p2/iceberg/test_external_catalog_icebergv2_nereids.out
b/regression-test/data/external_table_emr_p2/iceberg/test_external_catalog_icebergv2_nereids.out
new file mode 100644
index 0000000000..a6e68d2f62
--- /dev/null
+++
b/regression-test/data/external_table_emr_p2/iceberg/test_external_catalog_icebergv2_nereids.out
@@ -0,0 +1,74 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !q01 --
+149988
+
+-- !q02 --
+1
+3
+4
+7
+
+-- !q03 --
+8242263
+
+-- !q04 --
+0
+
+-- !q05 --
+1 Customer#000000001 IVhzIApeRb ot,c,E 15 25-989-741-2988
711.56 BUILDING to the even, regular platelets. regular, ironic
epitaphs nag e
+3 Customer#000000003 MG9kdTD2WBHm 1 11-719-748-3364 7498.12
AUTOMOBILE deposits eat slyly ironic, even instructions. express foxes
detect slyly. blithely even accounts abov
+4 Customer#000000004 XxVSJsLAGtn 4 14-128-190-5944 2866.83
MACHINERY requests. final, regular ideas sleep final accou
+
+-- !q06 --
+604519555
+604519557
+604519558
+
+-- !q07 --
+12979.65
+219204.52
+5908.20
+
+-- !q08 --
+120001848
+
+-- !q09 --
+1
+2
+3
+
+-- !q10 --
+150000000
+149999999
+149999996
+
+-- !q11 --
+1
+2
+3
+
+-- !q12 --
+150000000
+149999999
+149999996
+
+-- !q13 --
+1
+4
+7
+
+-- !q14 --
+Customer#000000004
+Customer#000000007
+
+-- !q15 --
+150000
+
+-- !q16 --
+150000
+
+-- !q17 --
+150000
+
+-- !q18 --
+150000
diff --git
a/regression-test/suites/external_table_emr_p2/iceberg/test_external_catalog_icebergv2_nereids.groovy
b/regression-test/suites/external_table_emr_p2/iceberg/test_external_catalog_icebergv2_nereids.groovy
new file mode 100644
index 0000000000..ab2d45d547
--- /dev/null
+++
b/regression-test/suites/external_table_emr_p2/iceberg/test_external_catalog_icebergv2_nereids.groovy
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_external_catalog_icebergv2_nereids", "p2") {
+ String enabled = context.config.otherConfigs.get("enableExternalHiveTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String extHiveHmsHost =
context.config.otherConfigs.get("extHiveHmsHost")
+ String extHiveHmsPort =
context.config.otherConfigs.get("extHiveHmsPort")
+ String hms_catalog_name = "test_external_hms_catalog_iceberg"
+ String iceberg_catalog_name = "test_external_iceberg_catalog_nereids"
+
+ sql """drop catalog if exists ${hms_catalog_name};"""
+ sql """
+ create catalog if not exists ${hms_catalog_name} properties (
+ 'type'='hms',
+ 'hive.metastore.uris' =
'thrift://${extHiveHmsHost}:${extHiveHmsPort}'
+ );
+ """
+
+ sql """drop catalog if exists ${iceberg_catalog_name};"""
+ sql """
+ create catalog if not exists ${iceberg_catalog_name} properties (
+ 'type'='iceberg',
+ 'iceberg.catalog.type'='hms',
+ 'hive.metastore.uris' =
'thrift://${extHiveHmsHost}:${extHiveHmsPort}'
+ );
+ """
+ sql """set enable_nereids_planner=true;"""
+ sql """set enable_fallback_to_original_planner=false;"""
+
+ sql """switch ${hms_catalog_name};"""
+ // test parquet format format
+ def q01 = {
+ qt_q01 """ select count(1) as c from customer_small """
+ qt_q02 """ select c_custkey from customer_small group by c_custkey
order by c_custkey limit 4 """
+ qt_q03 """ select count(1) from orders_small """
+ qt_q04 """ select count(1) from customer_small where c_name =
'Customer#000000005' or c_name = 'Customer#000000006' """
+ qt_q05 """ select * from customer_small order by c_custkey limit 3
"""
+ qt_q06 """ select o_orderkey from orders_small where o_orderkey >
652566 order by o_orderkey limit 3 """
+ qt_q07 """ select o_totalprice from orders_small where o_custkey <
3357 order by o_custkey limit 3 """
+ qt_q08 """ select count(1) as c from customer """
+ }
+ // test time travel stmt
+ def q02 = {
+ qt_q09 """ select c_custkey from customer for time as of
'2022-12-27 10:21:36' order by c_custkey limit 3 """
+ qt_q10 """ select c_custkey from customer for time as of
'2022-12-28 10:21:36' order by c_custkey desc limit 3 """
+ qt_q11 """ select c_custkey from customer for version as of
906874575350293177 order by c_custkey limit 3 """
+ qt_q12 """ select c_custkey from customer for version as of
6352416983354893547 order by c_custkey desc limit 3 """
+ }
+ // in predicate
+ def q03 = {
+ qt_q13 """ select c_custkey from customer_small where c_custkey in
(1, 2, 4, 7) order by c_custkey """
+ qt_q14 """ select c_name from customer_small where c_name in
('Customer#000000004', 'Customer#000000007') order by c_custkey """
+ }
+
+ // test for 'FOR TIME AS OF' and 'FOR VERSION AS OF'
+ def q04 = {
+ qt_q15 """ select count(*) from
${hms_catalog_name}.tpch_1000_icebergv2.customer_small FOR TIME AS OF
'2022-12-22 02:29:30' """
+ qt_q16 """ select count(*) from
${hms_catalog_name}.tpch_1000_icebergv2.customer_small FOR VERSION AS OF
6113938156088124425 """
+ qt_q17 """ select count(*) from
${iceberg_catalog_name}.tpch_1000_icebergv2.customer_small FOR TIME AS OF
'2022-12-22 02:29:30' """
+ qt_q18 """ select count(*) from
${iceberg_catalog_name}.tpch_1000_icebergv2.customer_small FOR VERSION AS OF
6113938156088124425 """
+ }
+
+ sql """ use `tpch_1000_icebergv2`; """
+ q01()
+ q02()
+ q03()
+ q04()
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
