This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d7f1a4fb9a [Fix](inverted index) need_read_data only supports DUP_KEYS
(#23818)
d7f1a4fb9a is described below
commit d7f1a4fb9a3f7eac39097045a7cd70bd27bf8798
Author: airborne12 <[email protected]>
AuthorDate: Tue Sep 5 19:57:22 2023 +0800
[Fix](inverted index) need_read_data only supports DUP_KEYS (#23818)
need_read_data optimization does not work properly when table is UNQIUE_KEY
MOR
---
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 4 +
.../data/index_p0/test_bitmap_index.out | 54 ++++++
.../suites/index_p0/test_bitmap_index.groovy | 197 +++++++++++++++++++++
3 files changed, 255 insertions(+)
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index aa2d228f2e..bde5a533ae 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -946,6 +946,10 @@ Status
SegmentIterator::_apply_inverted_index_on_block_column_predicate(
}
bool SegmentIterator::_need_read_data(ColumnId cid) {
+ // for safety reason, only support DUP_KEYS
+ if (_opts.tablet_schema->keys_type() != KeysType::DUP_KEYS) {
+ return true;
+ }
if (_output_columns.count(-1)) {
// if _output_columns contains -1, it means that the light
// weight schema change may not be enabled or other reasons
diff --git a/regression-test/data/index_p0/test_bitmap_index.out
b/regression-test/data/index_p0/test_bitmap_index.out
index e490d7a803..7693c1ca47 100644
--- a/regression-test/data/index_p0/test_bitmap_index.out
+++ b/regression-test/data/index_p0/test_bitmap_index.out
@@ -113,3 +113,57 @@
default_cluster:regression_test_index_p0.test_bitmap_index_unique
index16 v1
-- !sql --
1 1 1 1 1 1 2022-05-31
2022-05-31T10:00 1 1 true 2022-05-31
2022-05-31T10:00 2022-05-31T10:00:00.111 2022-05-31T10:00:00.111111
1
+-- !sql --
+create_time DATETIME No true \N
+vid VARCHAR(64) No true \N
+report_time DATETIME Yes true \N
+block_version INT Yes false \N REPLACE
+vehicle_mode INT Yes false \N REPLACE
+usage_mode INT Yes false \N REPLACE
+
+-- !sql --
+default_cluster:regression_test_index_p0.test_bitmap_index_unique_mor_delete
vid_bitmap_index vid
BITMAP
+
+-- !sql --
+2
+
+-- !sql --
+2
+
+-- !sql --
+2
+
+-- !sql --
+2
+
+-- !sql --
+2023-08-25T10:00 123 2023-08-25T10:00 1 1 1
+2023-08-25T11:00 123 2023-08-25T11:00 2 2 2
+
+-- !sql --
+create_time DATETIME No true \N
+vid VARCHAR(64) No true \N
+report_time DATETIME Yes true \N
+block_version INT Yes false \N NONE
+vehicle_mode INT Yes false \N NONE
+usage_mode INT Yes false \N NONE
+
+-- !sql --
+default_cluster:regression_test_index_p0.test_bitmap_index_unique_mow_delete
vid_bitmap_index vid
BITMAP
+
+-- !sql --
+2
+
+-- !sql --
+3
+
+-- !sql --
+2
+
+-- !sql --
+2
+
+-- !sql --
+2023-08-25T10:00 123 2023-08-25T10:00 1 1 1
+2023-08-25T11:00 123 2023-08-25T11:00 2 2 2
+
diff --git a/regression-test/suites/index_p0/test_bitmap_index.groovy
b/regression-test/suites/index_p0/test_bitmap_index.groovy
index 73e5ab0d9e..cf1da84456 100644
--- a/regression-test/suites/index_p0/test_bitmap_index.groovy
+++ b/regression-test/suites/index_p0/test_bitmap_index.groovy
@@ -269,4 +269,201 @@ suite("test_bitmap_index") {
}
}
sql "DROP TABLE ${tbName3} FORCE;"
+
+ // test bitmap index on MOR, and delete row after insert
+ def tbName4 = "test_bitmap_index_unique_mor_delete"
+ sql "DROP TABLE IF EXISTS ${tbName4}"
+ sql """
+ CREATE TABLE ${tbName4} (
+ create_time datetime NOT NULL COMMENT '',
+ vid varchar(64) NOT NULL COMMENT '',
+ report_time datetime NULL COMMENT '',
+ block_version int(11) NULL COMMENT '',
+ vehicle_mode int(11) NULL COMMENT '',
+ usage_mode int(11) NULL COMMENT ''
+ ) ENGINE=OLAP
+ UNIQUE KEY(create_time, vid, report_time)
+ COMMENT 'OLAP'
+ PARTITION BY RANGE(create_time)
+ (PARTITION p20230820 VALUES [('2023-08-20 00:00:00'),
('2023-08-21 00:00:00')),
+ PARTITION p20230821 VALUES [('2023-08-21 00:00:00'),
('2023-08-22 00:00:00')),
+ PARTITION p20230822 VALUES [('2023-08-22 00:00:00'),
('2023-08-23 00:00:00')),
+ PARTITION p20230823 VALUES [('2023-08-23 00:00:00'),
('2023-08-24 00:00:00')),
+ PARTITION p20230824 VALUES [('2023-08-24 00:00:00'),
('2023-08-25 00:00:00')),
+ PARTITION p20230825 VALUES [('2023-08-25 00:00:00'),
('2023-08-26 00:00:00')),
+ PARTITION p20230826 VALUES [('2023-08-26 00:00:00'),
('2023-08-27 00:00:00')),
+ PARTITION p20230827 VALUES [('2023-08-27 00:00:00'),
('2023-08-28 00:00:00')),
+ PARTITION p20230828 VALUES [('2023-08-28 00:00:00'),
('2023-08-29 00:00:00')),
+ PARTITION p20230829 VALUES [('2023-08-29 00:00:00'),
('2023-08-30 00:00:00')))
+ DISTRIBUTED BY HASH(vid) BUCKETS AUTO
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "is_being_synced" = "false",
+ "dynamic_partition.enable" = "true",
+ "dynamic_partition.time_unit" = "DAY",
+ "dynamic_partition.time_zone" = "Asia/Shanghai",
+ "dynamic_partition.start" = "-30",
+ "dynamic_partition.end" = "1",
+ "dynamic_partition.prefix" = "p",
+ "dynamic_partition.replication_allocation" =
"tag.location.default: 1",
+ "dynamic_partition.buckets" = "10",
+ "dynamic_partition.create_history_partition" = "true",
+ "dynamic_partition.history_partition_num" = "-1",
+ "dynamic_partition.hot_partition_num" = "0",
+ "dynamic_partition.reserved_history_periods" = "NULL",
+ "dynamic_partition.storage_policy" = "",
+ "storage_format" = "V2",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "false",
+ "enable_single_replica_compaction" = "false"
+ );
+ """
+
+ sql """
+ ALTER TABLE ${tbName4} ADD INDEX vid_bitmap_index (vid) USING
BITMAP;
+ """
+ max_try_secs = 60
+ while (max_try_secs--) {
+ String res = getJobState(tbName3)
+ if (res == "FINISHED" || res == "CANCELLED") {
+ assertEquals("FINISHED", res)
+ sleep(3000)
+ break
+ } else {
+ Thread.sleep(1000)
+ if (max_try_secs < 1) {
+ println "test timeout," + "state:" + res
+ assertEquals("FINISHED",res)
+ }
+ }
+ }
+
+ sql "insert into
${tbName4}(create_time,vid,report_time,block_version,vehicle_mode,usage_mode)
values('2023-08-25 10:00:00','123','2023-08-25 10:00:00',1,1,1);"
+ sql "insert into
${tbName4}(create_time,vid,report_time,block_version,vehicle_mode,usage_mode)
values('2023-08-25 11:00:00','123','2023-08-25 11:00:00',2,2,2);"
+ sql "insert into
${tbName4}(create_time,vid,report_time,block_version,vehicle_mode,usage_mode)
values('2023-08-25 12:00:00','123','2023-08-25 12:00:00',3,3,3);"
+ qt_sql "desc ${tbName4};"
+ qt_sql "SHOW INDEX FROM ${tbName4};"
+ sql "delete from ${tbName4} where vid='123' and
report_time='2023-08-25 12:00:00' and create_time='2023-08-25 12:00:00';"
+ qt_sql "select count(*) from ${tbName4}; "
+ qt_sql "select count(*) from ${tbName4} where vid='123'; "
+ qt_sql "select count(*) from ${tbName4} where create_time>='2023-08-25
10:00:00';"
+ qt_sql "select count(CASE when vid='123' then 1 else null end) from
${tbName4} where vid='123';"
+ qt_sql "select * from ${tbName4} where vid='123' order by create_time;"
+
+ sql "DROP INDEX IF EXISTS index1 ON ${tbName4};"
+ max_try_secs = 60
+ while (max_try_secs--) {
+ String res = getJobState(tbName3)
+ if (res == "FINISHED" || res == "CANCELLED") {
+ assertEquals("FINISHED", res)
+ sleep(3000)
+ break
+ } else {
+ Thread.sleep(1000)
+ if (max_try_secs < 1) {
+ println "test timeout," + "state:" + res
+ assertEquals("FINISHED",res)
+ }
+ }
+ }
+ sql "DROP TABLE ${tbName4} FORCE;"
+
+ // test bitmap index on MOW, and delete row after insert
+ def tbName5 = "test_bitmap_index_unique_mow_delete"
+ sql "DROP TABLE IF EXISTS ${tbName5}"
+ sql """
+ CREATE TABLE ${tbName5} (
+ create_time datetime NOT NULL COMMENT '',
+ vid varchar(64) NOT NULL COMMENT '',
+ report_time datetime NULL COMMENT '',
+ block_version int(11) NULL COMMENT '',
+ vehicle_mode int(11) NULL COMMENT '',
+ usage_mode int(11) NULL COMMENT ''
+ ) ENGINE=OLAP
+ UNIQUE KEY(create_time, vid, report_time)
+ COMMENT 'OLAP'
+ PARTITION BY RANGE(create_time)
+ (PARTITION p20230820 VALUES [('2023-08-20 00:00:00'),
('2023-08-21 00:00:00')),
+ PARTITION p20230821 VALUES [('2023-08-21 00:00:00'),
('2023-08-22 00:00:00')),
+ PARTITION p20230822 VALUES [('2023-08-22 00:00:00'),
('2023-08-23 00:00:00')),
+ PARTITION p20230823 VALUES [('2023-08-23 00:00:00'),
('2023-08-24 00:00:00')),
+ PARTITION p20230824 VALUES [('2023-08-24 00:00:00'),
('2023-08-25 00:00:00')),
+ PARTITION p20230825 VALUES [('2023-08-25 00:00:00'),
('2023-08-26 00:00:00')),
+ PARTITION p20230826 VALUES [('2023-08-26 00:00:00'),
('2023-08-27 00:00:00')),
+ PARTITION p20230827 VALUES [('2023-08-27 00:00:00'),
('2023-08-28 00:00:00')),
+ PARTITION p20230828 VALUES [('2023-08-28 00:00:00'),
('2023-08-29 00:00:00')),
+ PARTITION p20230829 VALUES [('2023-08-29 00:00:00'),
('2023-08-30 00:00:00')))
+ DISTRIBUTED BY HASH(vid) BUCKETS AUTO
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "is_being_synced" = "false",
+ "dynamic_partition.enable" = "true",
+ "dynamic_partition.time_unit" = "DAY",
+ "dynamic_partition.time_zone" = "Asia/Shanghai",
+ "dynamic_partition.start" = "-30",
+ "dynamic_partition.end" = "1",
+ "dynamic_partition.prefix" = "p",
+ "dynamic_partition.replication_allocation" =
"tag.location.default: 1",
+ "dynamic_partition.buckets" = "10",
+ "dynamic_partition.create_history_partition" = "true",
+ "dynamic_partition.history_partition_num" = "-1",
+ "dynamic_partition.hot_partition_num" = "0",
+ "dynamic_partition.reserved_history_periods" = "NULL",
+ "dynamic_partition.storage_policy" = "",
+ "storage_format" = "V2",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "false",
+ "enable_unique_key_merge_on_write" = "true",
+ "enable_single_replica_compaction" = "false"
+ );
+ """
+
+ sql """
+ ALTER TABLE ${tbName5} ADD INDEX vid_bitmap_index (vid) USING
BITMAP;
+ """
+ max_try_secs = 60
+ while (max_try_secs--) {
+ String res = getJobState(tbName3)
+ if (res == "FINISHED" || res == "CANCELLED") {
+ assertEquals("FINISHED", res)
+ sleep(3000)
+ break
+ } else {
+ Thread.sleep(1000)
+ if (max_try_secs < 1) {
+ println "test timeout," + "state:" + res
+ assertEquals("FINISHED",res)
+ }
+ }
+ }
+
+ sql "insert into
${tbName5}(create_time,vid,report_time,block_version,vehicle_mode,usage_mode)
values('2023-08-25 10:00:00','123','2023-08-25 10:00:00',1,1,1);"
+ sql "insert into
${tbName5}(create_time,vid,report_time,block_version,vehicle_mode,usage_mode)
values('2023-08-25 11:00:00','123','2023-08-25 11:00:00',2,2,2);"
+ sql "insert into
${tbName5}(create_time,vid,report_time,block_version,vehicle_mode,usage_mode)
values('2023-08-25 12:00:00','123','2023-08-25 12:00:00',3,3,3);"
+ qt_sql "desc ${tbName5};"
+ qt_sql "SHOW INDEX FROM ${tbName5};"
+ sql "delete from ${tbName5} where vid='123' and
report_time='2023-08-25 12:00:00' and create_time='2023-08-25 12:00:00';"
+ qt_sql "select count(*) from ${tbName5}; "
+ qt_sql "select count(*) from ${tbName5} where vid='123'; "
+ qt_sql "select count(*) from ${tbName5} where create_time>='2023-08-25
10:00:00';"
+ qt_sql "select count(CASE when vid='123' then 1 else null end) from
${tbName5} where vid='123';"
+ qt_sql "select * from ${tbName5} where vid='123' order by create_time;"
+
+ sql "DROP INDEX IF EXISTS index1 ON ${tbName5};"
+ max_try_secs = 60
+ while (max_try_secs--) {
+ String res = getJobState(tbName3)
+ if (res == "FINISHED" || res == "CANCELLED") {
+ assertEquals("FINISHED", res)
+ sleep(3000)
+ break
+ } else {
+ Thread.sleep(1000)
+ if (max_try_secs < 1) {
+ println "test timeout," + "state:" + res
+ assertEquals("FINISHED",res)
+ }
+ }
+ }
+ sql "DROP TABLE ${tbName5} FORCE;"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
