This is an automated email from the ASF dual-hosted git repository.
airborne pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new c9acd71ad6d [fix](inverted index) Fix errors caused by
enable_need_read_data_opt #42064 (#42247)
c9acd71ad6d is described below
commit c9acd71ad6de1751087c3d85ed9a026e926e74e4
Author: airborne12 <[email protected]>
AuthorDate: Wed Oct 23 09:18:19 2024 +0800
[fix](inverted index) Fix errors caused by enable_need_read_data_opt #42064
(#42247)
cherry pick from #42064
Co-authored-by: Sun Chenyang <[email protected]>
---
.../rules/implementation/AggregateStrategies.java | 22 ++-
.../data/inverted_index_p0/test_index_rqg_bug8.out | 4 +
.../inverted_index_p0/test_index_rqg_bug8.groovy | 149 +++++++++++++++++++++
3 files changed, 173 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
index 1cd9d8b783e..c93c3fc73ec 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
@@ -130,7 +130,8 @@ public class AggregateStrategies implements
ImplementationRuleFactory {
Set<Slot> aggSlots = funcs.stream()
.flatMap(f -> f.getInputSlots().stream())
.collect(Collectors.toSet());
- return conjuncts.stream().allMatch(expr ->
checkSlotInOrExpression(expr, aggSlots));
+ return conjuncts.stream().allMatch(expr ->
checkSlotInOrExpression(expr, aggSlots)
+ &&
checkIsNullExpr(expr, aggSlots));
})
.thenApply(ctx -> {
LogicalAggregate<LogicalFilter<LogicalOlapScan>> agg =
ctx.root;
@@ -163,7 +164,8 @@ public class AggregateStrategies implements
ImplementationRuleFactory {
Set<Slot> aggSlots = funcs.stream()
.flatMap(f -> f.getInputSlots().stream())
.collect(Collectors.toSet());
- return conjuncts.stream().allMatch(expr ->
checkSlotInOrExpression(expr, aggSlots));
+ return conjuncts.stream().allMatch(expr ->
checkSlotInOrExpression(expr, aggSlots)
+ &&
checkIsNullExpr(expr, aggSlots));
})
.thenApply(ctx -> {
LogicalAggregate<LogicalProject<LogicalFilter<LogicalOlapScan>>> agg = ctx.root;
@@ -452,6 +454,22 @@ public class AggregateStrategies implements
ImplementationRuleFactory {
return true;
}
+ private boolean checkIsNullExpr(Expression expr, Set<Slot> aggSlots) {
+ if (expr instanceof IsNull) {
+ Set<Slot> slots = expr.getInputSlots();
+ if (slots.stream().anyMatch(aggSlots::contains)) {
+ return false;
+ }
+ } else {
+ for (Expression child : expr.children()) {
+ if (!checkIsNullExpr(child, aggSlots)) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
private boolean isDupOrMowKeyTable(LogicalOlapScan logicalScan) {
if (logicalScan != null) {
KeysType keysType = logicalScan.getTable().getKeysType();
diff --git a/regression-test/data/inverted_index_p0/test_index_rqg_bug8.out
b/regression-test/data/inverted_index_p0/test_index_rqg_bug8.out
new file mode 100644
index 00000000000..a21f3b8748e
--- /dev/null
+++ b/regression-test/data/inverted_index_p0/test_index_rqg_bug8.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+0
+
diff --git
a/regression-test/suites/inverted_index_p0/test_index_rqg_bug8.groovy
b/regression-test/suites/inverted_index_p0/test_index_rqg_bug8.groovy
new file mode 100644
index 00000000000..3e54a10b326
--- /dev/null
+++ b/regression-test/suites/inverted_index_p0/test_index_rqg_bug8.groovy
@@ -0,0 +1,149 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+suite("test_index_rqg_bug8", "test_index_rqg_bug8"){
+ def table1 = "test_index_rqg_bug8"
+
+ sql "drop table if exists ${table1}"
+
+ sql """
+ CREATE TABLE ${table1} (
+ `pk` int NULL,
+ `col_int_undef_signed_index_inverted` int NULL,
+ `col_boolean_undef_signed` boolean NULL,
+ `col_boolean_undef_signed_not_null` boolean NOT NULL,
+ `col_tinyint_undef_signed` tinyint NULL,
+ `col_tinyint_undef_signed_index_inverted` tinyint NULL,
+ `col_tinyint_undef_signed_not_null` tinyint NOT NULL,
+ `col_tinyint_undef_signed_not_null_index_inverted` tinyint NOT NULL,
+ `col_smallint_undef_signed` smallint NULL,
+ `col_smallint_undef_signed_index_inverted` smallint NULL,
+ `col_smallint_undef_signed_not_null` smallint NOT NULL,
+ `col_smallint_undef_signed_not_null_index_inverted` smallint NOT NULL,
+ `col_int_undef_signed` int NULL,
+ `col_int_undef_signed_not_null` int NOT NULL,
+ `col_int_undef_signed_not_null_index_inverted` int NOT NULL,
+ `col_bigint_undef_signed` bigint NULL,
+ `col_bigint_undef_signed_index_inverted` bigint NULL,
+ `col_bigint_undef_signed_not_null` bigint NOT NULL,
+ `col_bigint_undef_signed_not_null_index_inverted` bigint NOT NULL,
+ `col_decimal_16__8__undef_signed` decimal(16,8) NULL,
+ `col_decimal_16__8__undef_signed_index_inverted` decimal(16,8) NULL,
+ `col_decimal_16__8__undef_signed_not_null` decimal(16,8) NOT NULL,
+ `col_decimal_16__8__undef_signed_not_null_index_inverted`
decimal(16,8) NOT NULL,
+ `col_decimal_38__9__undef_signed` decimal(38,9) NULL,
+ `col_decimal_38__9__undef_signed_index_inverted` decimal(38,9) NULL,
+ `col_decimal_38__9__undef_signed_not_null` decimal(38,9) NOT NULL,
+ `col_decimal_38__9__undef_signed_not_null_index_inverted`
decimal(38,9) NOT NULL,
+ `col_decimal_38__30__undef_signed` decimal(38,30) NULL,
+ `col_decimal_38__30__undef_signed_index_inverted` decimal(38,30) NULL,
+ `col_decimal_38__30__undef_signed_not_null` decimal(38,30) NOT NULL,
+ `col_decimal_38__30__undef_signed_not_null_index_inverted`
decimal(38,30) NOT NULL,
+ `col_date_undef_signed` date NULL,
+ `col_date_undef_signed_index_inverted` date NULL,
+ `col_date_undef_signed_not_null` date NOT NULL,
+ `col_date_undef_signed_not_null_index_inverted` date NOT NULL,
+ `col_datetime_undef_signed` datetime NULL,
+ `col_datetime_undef_signed_index_inverted` datetime NULL,
+ `col_datetime_undef_signed_not_null` datetime NOT NULL,
+ `col_datetime_undef_signed_not_null_index_inverted` datetime NOT NULL,
+ `col_datetime_3__undef_signed` datetime(3) NULL,
+ `col_datetime_3__undef_signed_index_inverted` datetime(3) NULL,
+ `col_datetime_3__undef_signed_not_null` datetime(3) NOT NULL,
+ `col_datetime_3__undef_signed_not_null_index_inverted` datetime(3) NOT
NULL,
+ `col_datetime_6__undef_signed` datetime(6) NULL,
+ `col_datetime_6__undef_signed_index_inverted` datetime(6) NULL,
+ `col_datetime_6__undef_signed_not_null` datetime(6) NOT NULL,
+ `col_datetime_6__undef_signed_not_null_index_inverted` datetime(6) NOT
NULL,
+ `col_char_255__undef_signed` character(255) NULL,
+ `col_char_255__undef_signed_index_inverted` character(255) NULL,
+ `col_char_255__undef_signed_index_inverted_p_e` character(255) NULL,
+ `col_char_255__undef_signed_index_inverted_p_u` character(255) NULL,
+ `col_char_255__undef_signed_not_null` character(255) NOT NULL,
+ `col_char_255__undef_signed_not_null_index_inverted` character(255)
NOT NULL,
+ `col_char_255__undef_signed_not_null_index_inverted_p_e`
character(255) NOT NULL,
+ `col_char_255__undef_signed_not_null_index_inverted_p_u`
character(255) NOT NULL,
+ `col_varchar_1024__undef_signed` varchar(1024) NULL,
+ `col_varchar_1024__undef_signed_index_inverted` varchar(1024) NULL,
+ `col_varchar_1024__undef_signed_index_inverted_p_e` varchar(1024) NULL,
+ `col_varchar_1024__undef_signed_index_inverted_p_u` varchar(1024) NULL,
+ `col_varchar_1024__undef_signed_not_null` varchar(1024) NOT NULL,
+ `col_varchar_1024__undef_signed_not_null_index_inverted` varchar(1024)
NOT NULL,
+ `col_varchar_1024__undef_signed_not_null_index_inverted_p_e`
varchar(1024) NOT NULL,
+ `col_varchar_1024__undef_signed_not_null_index_inverted_p_u`
varchar(1024) NOT NULL,
+ INDEX col_tinyint_undef_signed_index_inverted_idx
(`col_tinyint_undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_tinyint_undef_signed_not_null_index_inverted_idx
(`col_tinyint_undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_smallint_undef_signed_index_inverted_idx
(`col_smallint_undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_smallint_undef_signed_not_null_index_inverted_idx
(`col_smallint_undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_int_undef_signed_index_inverted_idx
(`col_int_undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_int_undef_signed_not_null_index_inverted_idx
(`col_int_undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_bigint_undef_signed_index_inverted_idx
(`col_bigint_undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_bigint_undef_signed_not_null_index_inverted_idx
(`col_bigint_undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_decimal_16__8__undef_signed_index_inverted_idx
(`col_decimal_16__8__undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_decimal_16__8__undef_signed_not_null_index_inverted_idx
(`col_decimal_16__8__undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_decimal_38__9__undef_signed_index_inverted_idx
(`col_decimal_38__9__undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_decimal_38__9__undef_signed_not_null_index_inverted_idx
(`col_decimal_38__9__undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_decimal_38__30__undef_signed_index_inverted_idx
(`col_decimal_38__30__undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_decimal_38__30__undef_signed_not_null_index_inverted_idx
(`col_decimal_38__30__undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_date_undef_signed_index_inverted_idx
(`col_date_undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_date_undef_signed_not_null_index_inverted_idx
(`col_date_undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_datetime_undef_signed_index_inverted_idx
(`col_datetime_undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_datetime_undef_signed_not_null_index_inverted_idx
(`col_datetime_undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_datetime_3__undef_signed_index_inverted_idx
(`col_datetime_3__undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_datetime_3__undef_signed_not_null_index_inverted_idx
(`col_datetime_3__undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_datetime_6__undef_signed_index_inverted_idx
(`col_datetime_6__undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_datetime_6__undef_signed_not_null_index_inverted_idx
(`col_datetime_6__undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_char_255__undef_signed_index_inverted_idx
(`col_char_255__undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_char_255__undef_signed_index_inverted_p_e_idx
(`col_char_255__undef_signed_index_inverted_p_e`) USING INVERTED
PROPERTIES("parser" = "english", "lower_case" = "true", "support_phrase" =
"true"),
+ INDEX col_char_255__undef_signed_index_inverted_p_u_idx
(`col_char_255__undef_signed_index_inverted_p_u`) USING INVERTED
PROPERTIES("parser" = "unicode", "lower_case" = "true", "support_phrase" =
"true"),
+ INDEX col_char_255__undef_signed_not_null_index_inverted_idx
(`col_char_255__undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_char_255__undef_signed_not_null_index_inverted_p_e_idx
(`col_char_255__undef_signed_not_null_index_inverted_p_e`) USING INVERTED
PROPERTIES("parser" = "english", "lower_case" = "true", "support_phrase" =
"true"),
+ INDEX col_char_255__undef_signed_not_null_index_inverted_p_u_idx
(`col_char_255__undef_signed_not_null_index_inverted_p_u`) USING INVERTED
PROPERTIES("parser" = "unicode", "lower_case" = "true", "support_phrase" =
"true"),
+ INDEX col_varchar_1024__undef_signed_index_inverted_idx
(`col_varchar_1024__undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_varchar_1024__undef_signed_index_inverted_p_e_idx
(`col_varchar_1024__undef_signed_index_inverted_p_e`) USING INVERTED
PROPERTIES("parser" = "english", "lower_case" = "true", "support_phrase" =
"true"),
+ INDEX col_varchar_1024__undef_signed_index_inverted_p_u_idx
(`col_varchar_1024__undef_signed_index_inverted_p_u`) USING INVERTED
PROPERTIES("parser" = "unicode", "lower_case" = "true", "support_phrase" =
"true"),
+ INDEX col_varchar_1024__undef_signed_not_null_index_inverted_idx
(`col_varchar_1024__undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_varchar_1024__undef_signed_not_null_index_inverted_p_e_idx
(`col_varchar_1024__undef_signed_not_null_index_inverted_p_e`) USING INVERTED
PROPERTIES("parser" = "english", "lower_case" = "true", "support_phrase" =
"true"),
+ INDEX col_varchar_1024__undef_signed_not_null_index_inverted_p_u_idx
(`col_varchar_1024__undef_signed_not_null_index_inverted_p_u`) USING INVERTED
PROPERTIES("parser" = "unicode", "lower_case" = "true", "support_phrase" =
"true")
+ ) ENGINE=OLAP
+ UNIQUE KEY(`pk`, `col_int_undef_signed_index_inverted`)
+ DISTRIBUTED BY HASH(`pk`) BUCKETS 10
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "min_load_replica_num" = "-1",
+ "bloom_filter_columns" = "col_varchar_1024__undef_signed,
col_date_undef_signed_not_null, col_date_undef_signed, col_int_undef_signed,
col_varchar_1024__undef_signed_not_null, col_int_undef_signed_not_null",
+ "is_being_synced" = "false",
+ "storage_medium" = "hdd",
+ "storage_format" = "V2",
+ "inverted_index_storage_format" = "V1",
+ "enable_unique_key_merge_on_write" = "true",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "false",
+ "enable_single_replica_compaction" = "false",
+ "group_commit_interval_ms" = "10000",
+ "group_commit_data_bytes" = "134217728",
+ "enable_mow_light_delete" = "false"
+ );
+ """
+
+ sql """ insert into ${table1}
(pk,col_boolean_undef_signed,col_boolean_undef_signed_not_null,col_tinyint_undef_signed,col_tinyint_undef_signed_index_inverted,col_tinyint_undef_signed_not_null,col_tinyint_undef_signed_not_null_index_inverted,col_smallint_undef_signed,col_smallint_undef_signed_index_inverted,col_smallint_undef_signed_not_null,col_smallint_undef_signed_not_null_index_inverted,col_int_undef_signed,col_int_undef_signed_index_inverted,col_int_undef_signed_not_null,col_int_
[...]
+
+ qt_sql """
+ SELECT count(col_decimal_16__8__undef_signed_index_inverted) FROM
${table1} where col_decimal_16__8__undef_signed_index_inverted is null;
+ """
+
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
