This is an automated email from the ASF dual-hosted git repository.
gabriellee pushed a commit to branch opt_perf
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/opt_perf by this push:
new 677b837a15 [Improvement](string) Optimize scanning for string (#12909)
677b837a15 is described below
commit 677b837a15d55e51424f3d12d0b3edb9cd00cc5d
Author: Gabriel <[email protected]>
AuthorDate: Fri Sep 23 17:44:48 2022 +0800
[Improvement](string) Optimize scanning for string (#12909)
---
be/src/olap/comparison_predicate.h | 5 ++-
be/src/olap/rowset/segment_v2/binary_plain_page.h | 27 +++++++++---
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 13 ++++--
be/src/vec/columns/column_string.cpp | 49 ++++++++++++++++++++++
be/src/vec/columns/column_string.h | 21 +---------
be/src/vec/columns/predicate_column.h | 7 ++--
6 files changed, 88 insertions(+), 34 deletions(-)
diff --git a/be/src/olap/comparison_predicate.h
b/be/src/olap/comparison_predicate.h
index 6fcb45bd3c..00c69efe89 100644
--- a/be/src/olap/comparison_predicate.h
+++ b/be/src/olap/comparison_predicate.h
@@ -518,8 +518,9 @@ private:
uint16_t _base_loop(uint16_t* sel, uint16_t size, const uint8_t*
__restrict null_map,
const TArray* __restrict data_array, const TValue&
value) const {
uint16_t new_size = 0;
- for (uint16_t i = 0; i < size; ++i) {
- uint16_t idx = sel[i];
+ uint16_t* end = sel + size;
+ for (uint16_t* cursor = sel; cursor < end; ++cursor) {
+ uint16_t idx = *cursor;
if constexpr (is_nullable) {
if (_opposite ^ (!null_map[idx] && _operator(data_array[idx],
value))) {
sel[new_size++] = idx;
diff --git a/be/src/olap/rowset/segment_v2/binary_plain_page.h
b/be/src/olap/rowset/segment_v2/binary_plain_page.h
index 1242fd9b75..659df55fee 100644
--- a/be/src/olap/rowset/segment_v2/binary_plain_page.h
+++ b/be/src/olap/rowset/segment_v2/binary_plain_page.h
@@ -248,12 +248,14 @@ public:
return Status::OK();
}
const size_t max_fetch = std::min(*n, static_cast<size_t>(_num_elems -
_cur_idx));
-
uint32_t len_array[max_fetch];
uint32_t start_offset_array[max_fetch];
- for (int i = 0; i < max_fetch; i++, _cur_idx++) {
- const uint32_t start_offset = offset(_cur_idx);
- uint32_t len = offset(_cur_idx + 1) - start_offset;
+
+ uint32_t last_offset = guarded_offset(_cur_idx);
+ for (int i = 0; i < max_fetch - 1; i++, _cur_idx++) {
+ const uint32_t start_offset = last_offset;
+ last_offset = guarded_offset(_cur_idx + 1);
+ uint32_t len = last_offset - start_offset;
len_array[i] = len;
start_offset_array[i] = start_offset;
if constexpr (Type == OLAP_FIELD_TYPE_OBJECT) {
@@ -262,6 +264,14 @@ public:
}
}
}
+ _cur_idx++;
+ len_array[max_fetch - 1] = offset(_cur_idx) - last_offset;
+ start_offset_array[max_fetch - 1] = last_offset;
+ if constexpr (Type == OLAP_FIELD_TYPE_OBJECT) {
+ if (_options.need_check_bitmap) {
+ RETURN_IF_ERROR(BitmapTypeCode::validate(*(_data.data +
last_offset)));
+ }
+ }
dst->insert_many_binary_data(_data.mutable_data(), len_array,
start_offset_array,
max_fetch);
@@ -340,13 +350,20 @@ public:
}
private:
+ static constexpr size_t SIZE_OF_INT32 = sizeof(uint32_t);
// Return the offset within '_data' where the string value with index
'idx' can be found.
uint32_t offset(size_t idx) const {
if (idx >= _num_elems) {
return _offsets_pos;
}
const uint8_t* p =
- reinterpret_cast<const uint8_t*>(&_data[_offsets_pos + idx *
sizeof(uint32_t)]);
+ reinterpret_cast<const uint8_t*>(&_data[_offsets_pos + idx *
SIZE_OF_INT32]);
+ return decode_fixed32_le(p);
+ }
+
+ uint32_t guarded_offset(size_t idx) const {
+ const uint8_t* p =
+ reinterpret_cast<const uint8_t*>(&_data[_offsets_pos + idx *
SIZE_OF_INT32]);
return decode_fixed32_le(p);
}
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 98f2cfae27..e6435e8be1 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -1150,8 +1150,11 @@ Status SegmentIterator::next_batch(vectorized::Block*
block) {
}
if (!_lazy_materialization_read) {
- Status ret = _output_column_by_sel_idx(block,
_first_read_column_ids, sel_rowid_idx,
- selected_size);
+ Status ret = Status::OK();
+ if (selected_size > 0) {
+ ret = _output_column_by_sel_idx(block, _first_read_column_ids,
sel_rowid_idx,
+ selected_size);
+ }
if (!ret.ok()) {
return ret;
}
@@ -1176,8 +1179,10 @@ Status SegmentIterator::next_batch(vectorized::Block*
block) {
// when lazy materialization enables, _first_read_column_ids =
distinct(_short_cir_pred_column_ids + _vec_pred_column_ids)
// see _vec_init_lazy_materialization
// todo(wb) need to tell input columnids from output columnids
- RETURN_IF_ERROR(_output_column_by_sel_idx(block,
_first_read_column_ids, sel_rowid_idx,
- selected_size));
+ if (selected_size > 0) {
+ RETURN_IF_ERROR(_output_column_by_sel_idx(block,
_first_read_column_ids, sel_rowid_idx,
+ selected_size));
+ }
}
// shrink char_type suffix zero data
diff --git a/be/src/vec/columns/column_string.cpp
b/be/src/vec/columns/column_string.cpp
index d988791ca0..ddc21c31d8 100644
--- a/be/src/vec/columns/column_string.cpp
+++ b/be/src/vec/columns/column_string.cpp
@@ -440,6 +440,55 @@ void ColumnString::reserve(size_t n) {
chars.reserve(n);
}
+void ColumnString::insert_many_strings(const StringRef* strings, size_t num) {
+ DCHECK_NE(num, 0);
+ offsets.reserve(offsets.size() + num);
+ std::vector<const char*> start_points(1);
+ auto& head = strings[0];
+ start_points[0] = head.data;
+ size_t new_size = head.size;
+ const char* cursor = head.data + new_size;
+ std::vector<const char*> end_points;
+
+ const size_t old_size = chars.size();
+ size_t offset = old_size;
+ offset += new_size;
+ offsets.push_back(offset);
+ if (num == 1) {
+ end_points.push_back(cursor);
+ } else {
+ for (size_t i = 1; i < num; i++) {
+ auto& str = strings[i];
+ if (cursor != str.data) {
+ end_points.push_back(cursor);
+ start_points.push_back(str.data);
+ cursor = str.data;
+ }
+ size_t sz = str.size;
+ offset += sz;
+ new_size += sz;
+ cursor += sz;
+ offsets.push_back_without_reserve(offset);
+ }
+ end_points.push_back(cursor);
+ }
+ DCHECK_EQ(end_points.size(), start_points.size());
+
+ chars.resize(old_size + new_size);
+
+ size_t num_range = start_points.size();
+ Char* data = chars.data();
+
+ offset = old_size;
+ for (size_t i = 0; i < num_range; i++) {
+ uint32_t len = end_points[i] - start_points[i];
+ if (len) {
+ memcpy(data + offset, start_points[i], len);
+ offset += len;
+ }
+ }
+}
+
void ColumnString::resize(size_t n) {
auto origin_size = size();
if (origin_size > n) {
diff --git a/be/src/vec/columns/column_string.h
b/be/src/vec/columns/column_string.h
index 469bbdc6df..a62a16be42 100644
--- a/be/src/vec/columns/column_string.h
+++ b/be/src/vec/columns/column_string.h
@@ -186,26 +186,7 @@ public:
}
};
- void insert_many_strings(const StringRef* strings, size_t num) override {
- size_t new_size = 0;
- for (size_t i = 0; i < num; i++) {
- new_size += strings[i].size;
- }
-
- const size_t old_size = chars.size();
- chars.resize(old_size + new_size);
-
- Char* data = chars.data();
- size_t offset = old_size;
- for (size_t i = 0; i < num; i++) {
- uint32_t len = strings[i].size;
- if (len) {
- memcpy(data + offset, strings[i].data, len);
- offset += len;
- }
- offsets.push_back(offset);
- }
- }
+ void insert_many_strings(const StringRef* strings, size_t num) override;
void insert_many_dict_data(const int32_t* data_array, size_t start_index,
const StringRef* dict,
size_t num, uint32_t /*dict_num*/) override {
diff --git a/be/src/vec/columns/predicate_column.h
b/be/src/vec/columns/predicate_column.h
index d5ad52b6ac..4d44217b9a 100644
--- a/be/src/vec/columns/predicate_column.h
+++ b/be/src/vec/columns/predicate_column.h
@@ -279,9 +279,10 @@ public:
data.resize(org_elem_num + num);
uint32_t fragment_start_offset = start_offset_array[0];
size_t fragment_len = 0;
- for (size_t i = 0; i < num; i++) {
- data[org_elem_num + i].ptr = destination + fragment_len;
- data[org_elem_num + i].len = len_array[i];
+ StringValue* cursor = data.data() + org_elem_num;
+ for (size_t i = 0; i < num; i++, cursor++) {
+ cursor->ptr = destination + fragment_len;
+ cursor->len = len_array[i];
fragment_len += len_array[i];
// Compute the largest continuous memcpy block and copy them.
// If this is the last element in data array, then should copy
the current memory block.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
