This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 8d95e280cb [fix](hive) fix partition prune issue and some external
table test cases (#24335)
8d95e280cb is described below
commit 8d95e280cb11bc79b84bc969d0cdb9886faeec11
Author: Mingyu Chen <[email protected]>
AuthorDate: Thu Sep 14 11:01:58 2023 +0800
[fix](hive) fix partition prune issue and some external table test cases
(#24335)
---
.../rules/rewrite/PruneFileScanPartition.java | 33 +++++++++++++---------
.../trees/plans/logical/LogicalFileScan.java | 8 ++++++
.../external_table_p0/tvf/test_local_tvf.groovy | 10 +++----
.../hive/test_external_catalog_hive.groovy | 4 +--
4 files changed, 34 insertions(+), 21 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
index 4e35b8d82b..ca0e871320 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
@@ -55,22 +55,26 @@ public class PruneFileScanPartition extends
OneRewriteRuleFactory {
@Override
public Rule build() {
- return logicalFilter(logicalFileScan()).when(p ->
p.child().getSelectedPartitions() == null).thenApply(ctx -> {
- LogicalFilter<LogicalFileScan> filter = ctx.root;
- LogicalFileScan scan = filter.child();
- ExternalTable tbl = scan.getTable();
- SelectedPartitions selectedPartitions = new SelectedPartitions(0,
Maps.newHashMap(), false);
+ return logicalFilter(logicalFileScan()).when(p ->
!p.child().getSelectedPartitions().isPartitionPruned())
+ .thenApply(ctx -> {
+ LogicalFilter<LogicalFileScan> filter = ctx.root;
+ LogicalFileScan scan = filter.child();
+ ExternalTable tbl = scan.getTable();
+ SelectedPartitions selectedPartitions = new
SelectedPartitions(0, Maps.newHashMap(), false);
- // TODO(cmy): support other external table
- if (tbl instanceof HMSExternalTable && ((HMSExternalTable)
tbl).getDlaType() == DLAType.HIVE) {
- HMSExternalTable hiveTbl = (HMSExternalTable) tbl;
- selectedPartitions = pruneHivePartitions(hiveTbl, filter,
scan, ctx.cascadesContext);
- }
+ // TODO(cmy): support other external table
+ if (tbl instanceof HMSExternalTable && ((HMSExternalTable)
tbl).getDlaType() == DLAType.HIVE) {
+ HMSExternalTable hiveTbl = (HMSExternalTable) tbl;
+ selectedPartitions = pruneHivePartitions(hiveTbl,
filter, scan, ctx.cascadesContext);
+ } else {
+ // set isPartitionPruned so that it won't go pass the
partition prune again
+ selectedPartitions.isPartitionPruned = true;
+ }
- LogicalFileScan rewrittenScan =
scan.withConjuncts(filter.getConjuncts())
- .withSelectedPartitions(selectedPartitions);
- return new LogicalFilter<>(filter.getConjuncts(), rewrittenScan);
- }).toRule(RuleType.FILE_SCAN_PARTITION_PRUNE);
+ LogicalFileScan rewrittenScan =
scan.withConjuncts(filter.getConjuncts())
+ .withSelectedPartitions(selectedPartitions);
+ return new LogicalFilter<>(filter.getConjuncts(),
rewrittenScan);
+ }).toRule(RuleType.FILE_SCAN_PARTITION_PRUNE);
}
private SelectedPartitions pruneHivePartitions(HMSExternalTable hiveTbl,
@@ -104,3 +108,4 @@ public class PruneFileScanPartition extends
OneRewriteRuleFactory {
return new SelectedPartitions(idToPartitionItem.size(),
selectedPartitionItems, true);
}
}
+
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
index 4b46084a82..00c6849cdc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
@@ -149,6 +149,13 @@ public class LogicalFileScan extends
LogicalCatalogRelation {
}
}
+ /**
+ * Return true means this relation is partition prunned
+ */
+ public boolean isPartitionPruned() {
+ return this.totalPartitionNum > 0 || isPartitionPruned;
+ }
+
@Override
public boolean equals(Object o) {
return isPartitionPruned == ((SelectedPartitions)
o).isPartitionPruned && Objects.equals(
@@ -157,3 +164,4 @@ public class LogicalFileScan extends LogicalCatalogRelation
{
}
}
+
diff --git a/regression-test/suites/external_table_p0/tvf/test_local_tvf.groovy
b/regression-test/suites/external_table_p0/tvf/test_local_tvf.groovy
index 21d985b358..47f157a035 100644
--- a/regression-test/suites/external_table_p0/tvf/test_local_tvf.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_local_tvf.groovy
@@ -26,14 +26,14 @@ suite("test_local_tvf") {
List<List<Object>> doris_log = sql """ ADMIN SHOW FRONTEND CONFIG like
"sys_log_dir"; """
assertTrue(doris_log.size() > 0)
- def doris_log_path = doris_log[0][1]
+ def doris_log_path = "log/"
table = sql """
select count(*) from local(
- "file_path" = "${doris_log_path}/fe.out",
+ "file_path" = "${doris_log_path}/be.out",
"backend_id" = "${be_id}",
"format" = "csv")
- where c1 like "%FE type%";"""
+ where c1 like "%start time%";"""
assertTrue(table.size() > 0)
assertTrue(Long.valueOf(table[0][0]) > 0)
@@ -43,7 +43,7 @@ suite("test_local_tvf") {
"file_path" = "${doris_log_path}/*.out",
"backend_id" = "${be_id}",
"format" = "csv")
- where c1 like "%FE type%";"""
+ where c1 like "%start time%";"""
assertTrue(table.size() > 0)
assertTrue(Long.valueOf(table[0][0]) > 0)
@@ -51,7 +51,7 @@ suite("test_local_tvf") {
test {
sql """
select count(*) from local(
- "file_path" = "../fe.out",
+ "file_path" = "../be.out",
"backend_id" = "${be_id}",
"format" = "csv")
where c1 like "%FE type%";
diff --git
a/regression-test/suites/external_table_p2/hive/test_external_catalog_hive.groovy
b/regression-test/suites/external_table_p2/hive/test_external_catalog_hive.groovy
index fed3ba60af..b2ad3eeb88 100644
---
a/regression-test/suites/external_table_p2/hive/test_external_catalog_hive.groovy
+++
b/regression-test/suites/external_table_p2/hive/test_external_catalog_hive.groovy
@@ -94,8 +94,8 @@ suite("test_external_catalog_hive", "p2") {
qt_not_single_slot_filter_conjuncts_parquet """ select * from
multi_catalog.lineitem_string_date_orc where l_commitdate < l_receiptdate and
l_receiptdate = '1995-01-01' order by l_orderkey, l_partkey, l_suppkey,
l_linenumber limit 10; """
// test null expr with dict filter issue
- qt_null_expr_dict_filter_orc """ select count(*), count(distinct
user_no) from multi_catalog.dict_fitler_test_orc WHERE partitions in
('2023-08-21') and actual_intf_type = 'type1' and (REUSE_FLAG<> 'y' or
REUSE_FLAG is null); """
- qt_null_expr_dict_filter_parquet """ select count(*), count(distinct
user_no) from multi_catalog.dict_fitler_test_parquet WHERE partitions in
('2023-08-21') and actual_intf_type = 'type1' and (REUSE_FLAG<> 'y' or
REUSE_FLAG is null); """
+ qt_null_expr_dict_filter_orc """ select count(*), count(distinct
user_no) from multi_catalog.dict_fitler_test_orc WHERE `partitions` in
('2023-08-21') and actual_intf_type = 'type1' and (REUSE_FLAG<> 'y' or
REUSE_FLAG is null); """
+ qt_null_expr_dict_filter_parquet """ select count(*), count(distinct
user_no) from multi_catalog.dict_fitler_test_parquet WHERE `partitions` in
('2023-08-21') and actual_intf_type = 'type1' and (REUSE_FLAG<> 'y' or
REUSE_FLAG is null); """
// test par fields in file
qt_par_fields_in_file_orc1 """ select * from
multi_catalog.par_fields_in_file_orc where year = 2023 and month = 8 order by
id; """
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
