This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 9eb2f90e27c [Optimize](inverted index) optimize inverted index bitmap
copy (#32279) (#32469)
9eb2f90e27c is described below
commit 9eb2f90e27c372af059ec9431bde8419ac0f96b8
Author: airborne12 <[email protected]>
AuthorDate: Tue Mar 19 17:28:59 2024 +0800
[Optimize](inverted index) optimize inverted index bitmap copy (#32279)
(#32469)
---
be/src/olap/comparison_predicate.h | 8 +--
be/src/olap/in_list_predicate.h | 6 +-
be/src/olap/match_predicate.cpp | 8 +--
.../segment_v2/inverted_index_file_reader.cpp | 11 ++-
.../rowset/segment_v2/inverted_index_file_reader.h | 1 +
.../rowset/segment_v2/inverted_index_reader.cpp | 83 +++++++++++-----------
.../olap/rowset/segment_v2/inverted_index_reader.h | 22 +++---
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 17 ++---
be/src/runtime/exec_env_init.cpp | 4 +-
9 files changed, 85 insertions(+), 75 deletions(-)
diff --git a/be/src/olap/comparison_predicate.h
b/be/src/olap/comparison_predicate.h
index 3673b89a1d0..19e92a41bca 100644
--- a/be/src/olap/comparison_predicate.h
+++ b/be/src/olap/comparison_predicate.h
@@ -99,11 +99,11 @@ public:
return Status::InvalidArgument("invalid comparison predicate type
{}", PT);
}
- roaring::Roaring roaring;
+ std::shared_ptr<roaring::Roaring> roaring =
std::make_shared<roaring::Roaring>();
auto&& value =
PrimitiveTypeConvertor<Type>::to_storage_field_type(_value);
RETURN_IF_ERROR(iterator->read_from_inverted_index(column_name,
&value, query_type,
- num_rows,
&roaring));
+ num_rows, roaring));
// mask out null_bitmap, since NULL cmp VALUE will produce NULL
// and be treated as false in WHERE
@@ -118,9 +118,9 @@ public:
}
if constexpr (PT == PredicateType::NE) {
- *bitmap -= roaring;
+ *bitmap -= *roaring;
} else {
- *bitmap &= roaring;
+ *bitmap &= *roaring;
}
return Status::OK();
diff --git a/be/src/olap/in_list_predicate.h b/be/src/olap/in_list_predicate.h
index 6800b563fae..ec0e770ebd7 100644
--- a/be/src/olap/in_list_predicate.h
+++ b/be/src/olap/in_list_predicate.h
@@ -194,10 +194,10 @@ public:
auto&& value = PrimitiveTypeConvertor<Type>::to_storage_field_type(
*reinterpret_cast<const T*>(ptr));
InvertedIndexQueryType query_type =
InvertedIndexQueryType::EQUAL_QUERY;
- roaring::Roaring index;
+ std::shared_ptr<roaring::Roaring> index =
std::make_shared<roaring::Roaring>();
RETURN_IF_ERROR(iterator->read_from_inverted_index(column_name,
&value, query_type,
- num_rows,
&index));
- indices |= index;
+ num_rows,
index));
+ indices |= *index;
iter->next();
}
diff --git a/be/src/olap/match_predicate.cpp b/be/src/olap/match_predicate.cpp
index 13fd982b0b5..c000d3b635d 100644
--- a/be/src/olap/match_predicate.cpp
+++ b/be/src/olap/match_predicate.cpp
@@ -57,7 +57,7 @@ Status MatchPredicate::evaluate(const
vectorized::NameAndTypePair& name_with_typ
}
auto type = name_with_type.second;
const std::string& name = name_with_type.first;
- roaring::Roaring roaring;
+ std::shared_ptr<roaring::Roaring> roaring =
std::make_shared<roaring::Roaring>();
auto inverted_index_query_type =
_to_inverted_index_query_type(_match_type);
TypeDescriptor column_desc = type->get_type_as_type_descriptor();
if (is_string_type(column_desc.type) ||
@@ -67,7 +67,7 @@ Status MatchPredicate::evaluate(const
vectorized::NameAndTypePair& name_with_typ
char* buffer = const_cast<char*>(_value.c_str());
match_value.replace(buffer, length); //is it safe?
RETURN_IF_ERROR(iterator->read_from_inverted_index(
- name, &match_value, inverted_index_query_type, num_rows,
&roaring));
+ name, &match_value, inverted_index_query_type, num_rows,
roaring));
} else if (column_desc.type == TYPE_ARRAY &&
is_numeric_type(
TabletColumn::get_field_type_by_type(column_desc.children[0].type))) {
@@ -76,7 +76,7 @@ Status MatchPredicate::evaluate(const
vectorized::NameAndTypePair& name_with_typ
TabletColumn::get_field_type_by_type(column_desc.children[0].type));
RETURN_IF_ERROR(type_info->from_string(buf, _value));
RETURN_IF_ERROR(iterator->read_from_inverted_index(name, buf,
inverted_index_query_type,
- num_rows, &roaring,
true));
+ num_rows, roaring,
true));
}
// mask out null_bitmap, since NULL cmp VALUE will produce NULL
@@ -91,7 +91,7 @@ Status MatchPredicate::evaluate(const
vectorized::NameAndTypePair& name_with_typ
}
}
- *bitmap &= roaring;
+ *bitmap &= *roaring;
return Status::OK();
}
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
index 03db293f613..ccbb0eb4c0b 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
@@ -192,12 +192,21 @@ Result<std::unique_ptr<DorisCompoundReader>>
InvertedIndexFileReader::open(
return _open(index_id, index_suffix);
}
-std::string InvertedIndexFileReader::get_index_file_path(const TabletIndex*
index_meta) const {
+std::string InvertedIndexFileReader::get_index_file_key(const TabletIndex*
index_meta) const {
return InvertedIndexDescriptor::get_index_file_name(_index_file_dir /
_segment_file_name,
index_meta->index_id(),
index_meta->get_index_suffix());
}
+std::string InvertedIndexFileReader::get_index_file_path(const TabletIndex*
index_meta) const {
+ if (_storage_format == InvertedIndexStorageFormatPB::V1) {
+ return InvertedIndexDescriptor::get_index_file_name(_index_file_dir /
_segment_file_name,
+
index_meta->index_id(),
+
index_meta->get_index_suffix());
+ }
+ return _index_file_dir / _index_file_name;
+}
+
Status InvertedIndexFileReader::index_file_exist(const TabletIndex*
index_meta, bool* res) const {
if (_storage_format == InvertedIndexStorageFormatPB::V1) {
auto index_file_path = _index_file_dir /
InvertedIndexDescriptor::get_index_file_name(
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h
b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h
index c87ea9b2372..d44976fc889 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h
@@ -66,6 +66,7 @@ public:
bool open_idx_file_cache = false);
Result<std::unique_ptr<DorisCompoundReader>> open(const TabletIndex*
index_meta) const;
void debug_file_entries();
+ std::string get_index_file_key(const TabletIndex* index_meta) const;
std::string get_index_file_path(const TabletIndex* index_meta) const;
Status index_file_exist(const TabletIndex* index_meta, bool* res) const;
Status has_null(const TabletIndex* index_meta, bool* res) const;
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
index 58810d472e3..769886d2e7c 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
@@ -149,9 +149,9 @@ Status
InvertedIndexReader::read_null_bitmap(InvertedIndexQueryCacheHandle* cach
bool owned_dir = false;
try {
// try to get query bitmap result from cache and return immediately on
cache hit
- auto index_file_path =
_inverted_index_file_reader->get_index_file_path(&_index_meta);
+ auto index_file_key =
_inverted_index_file_reader->get_index_file_key(&_index_meta);
InvertedIndexQueryCache::CacheKey cache_key {
- index_file_path, "", InvertedIndexQueryType::UNKNOWN_QUERY,
"null_bitmap"};
+ index_file_key, "", InvertedIndexQueryType::UNKNOWN_QUERY,
"null_bitmap"};
auto* cache = InvertedIndexQueryCache::instance();
if (cache->lookup(cache_key, cache_handle)) {
return Status::OK();
@@ -196,7 +196,7 @@ Status
InvertedIndexReader::read_null_bitmap(InvertedIndexQueryCacheHandle* cach
Status InvertedIndexReader::handle_searcher_cache(
InvertedIndexCacheHandle* inverted_index_cache_handle,
OlapReaderStatistics* stats) {
- auto index_file_key =
_inverted_index_file_reader->get_index_file_path(&_index_meta);
+ auto index_file_key =
_inverted_index_file_reader->get_index_file_key(&_index_meta);
InvertedIndexSearcherCache::CacheKey searcher_cache_key(index_file_key);
if (InvertedIndexSearcherCache::instance()->lookup(searcher_cache_key,
inverted_index_cache_handle)) {
@@ -252,7 +252,8 @@ Status
FullTextIndexReader::new_iterator(OlapReaderStatistics* stats, RuntimeSta
Status FullTextIndexReader::query(OlapReaderStatistics* stats, RuntimeState*
runtime_state,
const std::string& column_name, const void*
query_value,
- InvertedIndexQueryType query_type,
roaring::Roaring* bit_map) {
+ InvertedIndexQueryType query_type,
+ std::shared_ptr<roaring::Roaring>& bit_map) {
SCOPED_RAW_TIMER(&stats->inverted_index_query_timer);
std::string search_str = reinterpret_cast<const
StringRef*>(query_value)->to_string();
@@ -262,10 +263,10 @@ Status FullTextIndexReader::query(OlapReaderStatistics*
stats, RuntimeState* run
try {
std::vector<std::string> analyse_result;
InvertedIndexQueryCache::CacheKey cache_key;
- auto index_file_path =
_inverted_index_file_reader->get_index_file_path(&_index_meta);
+ auto index_file_key =
_inverted_index_file_reader->get_index_file_key(&_index_meta);
if (query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY) {
- cache_key = {index_file_path, column_name, query_type, search_str};
+ cache_key = {index_file_key, column_name, query_type, search_str};
analyse_result.emplace_back(search_str);
} else {
InvertedIndexCtxSPtr inverted_index_ctx =
std::make_shared<InvertedIndexCtx>(
@@ -301,7 +302,6 @@ Status FullTextIndexReader::query(OlapReaderStatistics*
stats, RuntimeState* run
std::unique_ptr<lucene::search::Query> query;
std::wstring field_ws = std::wstring(column_name.begin(),
column_name.end());
- roaring::Roaring query_match_bitmap;
if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY ||
query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY ||
query_type == InvertedIndexQueryType::MATCH_PHRASE_EDGE_QUERY ||
@@ -310,7 +310,7 @@ Status FullTextIndexReader::query(OlapReaderStatistics*
stats, RuntimeState* run
query_type == InvertedIndexQueryType::MATCH_ANY_QUERY) {
std::string str_tokens = join(analyse_result, " ");
- cache_key = {index_file_path, column_name, query_type, str_tokens};
+ cache_key = {index_file_key, column_name, query_type, str_tokens};
}
auto* cache = InvertedIndexQueryCache::instance();
InvertedIndexQueryCacheHandle cache_handler;
@@ -337,10 +337,8 @@ Status FullTextIndexReader::query(OlapReaderStatistics*
stats, RuntimeState* run
}
term_match_bitmap->runOptimize();
cache->insert(cache_key, term_match_bitmap, &cache_handler);
+ bit_map = term_match_bitmap;
}
- query_match_bitmap = *term_match_bitmap;
-
- bit_map->swap(query_match_bitmap);
return Status::OK();
} catch (const CLuceneError& e) {
return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
@@ -385,7 +383,7 @@ Status
StringTypeInvertedIndexReader::query(OlapReaderStatistics* stats,
RuntimeState* runtime_state,
const std::string& column_name,
const void* query_value,
InvertedIndexQueryType query_type,
- roaring::Roaring* bit_map) {
+ std::shared_ptr<roaring::Roaring>&
bit_map) {
SCOPED_RAW_TIMER(&stats->inverted_index_query_timer);
const auto* search_query = reinterpret_cast<const StringRef*>(query_value);
@@ -402,10 +400,10 @@ Status
StringTypeInvertedIndexReader::query(OlapReaderStatistics* stats,
[](lucene::index::Term* term) { _CLDECDELETE(term); }};
std::unique_ptr<lucene::search::Query> query;
- auto index_file_path =
_inverted_index_file_reader->get_index_file_path(&_index_meta);
+ auto index_file_key =
_inverted_index_file_reader->get_index_file_key(&_index_meta);
// try to get query bitmap result from cache and return immediately on
cache hit
- InvertedIndexQueryCache::CacheKey cache_key {index_file_path, column_name,
query_type,
+ InvertedIndexQueryCache::CacheKey cache_key {index_file_key, column_name,
query_type,
search_str};
auto* cache = InvertedIndexQueryCache::instance();
InvertedIndexQueryCacheHandle cache_handler;
@@ -498,7 +496,7 @@ Status
StringTypeInvertedIndexReader::query(OlapReaderStatistics* stats,
term_match_bitmap->runOptimize();
cache->insert(cache_key, term_match_bitmap, &cache_handler);
- bit_map->swap(result);
+ bit_map = term_match_bitmap;
}
return Status::OK();
}
@@ -590,12 +588,12 @@ Status BkdIndexReader::invoke_bkd_try_query(const void*
query_value,
Status BkdIndexReader::invoke_bkd_query(const void* query_value,
InvertedIndexQueryType query_type,
std::shared_ptr<lucene::util::bkd::bkd_reader> r,
- roaring::Roaring* bit_map) {
+ std::shared_ptr<roaring::Roaring>&
bit_map) {
switch (query_type) {
case InvertedIndexQueryType::LESS_THAN_QUERY: {
auto visitor =
std::make_unique<InvertedIndexVisitor<InvertedIndexQueryType::LESS_THAN_QUERY>>(
- r.get(), bit_map);
+ r.get(), bit_map.get());
RETURN_IF_ERROR(construct_bkd_query_value(query_value, r,
visitor.get()));
r->intersect(visitor.get());
break;
@@ -603,7 +601,7 @@ Status BkdIndexReader::invoke_bkd_query(const void*
query_value, InvertedIndexQu
case InvertedIndexQueryType::LESS_EQUAL_QUERY: {
auto visitor =
std::make_unique<InvertedIndexVisitor<InvertedIndexQueryType::LESS_EQUAL_QUERY>>(
- r.get(), bit_map);
+ r.get(), bit_map.get());
RETURN_IF_ERROR(construct_bkd_query_value(query_value, r,
visitor.get()));
r->intersect(visitor.get());
break;
@@ -611,7 +609,7 @@ Status BkdIndexReader::invoke_bkd_query(const void*
query_value, InvertedIndexQu
case InvertedIndexQueryType::GREATER_THAN_QUERY: {
auto visitor =
std::make_unique<InvertedIndexVisitor<InvertedIndexQueryType::GREATER_THAN_QUERY>>(
- r.get(), bit_map);
+ r.get(), bit_map.get());
RETURN_IF_ERROR(construct_bkd_query_value(query_value, r,
visitor.get()));
r->intersect(visitor.get());
break;
@@ -619,14 +617,14 @@ Status BkdIndexReader::invoke_bkd_query(const void*
query_value, InvertedIndexQu
case InvertedIndexQueryType::GREATER_EQUAL_QUERY: {
auto visitor =
std::make_unique<InvertedIndexVisitor<InvertedIndexQueryType::GREATER_EQUAL_QUERY>>(
- r.get(), bit_map);
+ r.get(), bit_map.get());
RETURN_IF_ERROR(construct_bkd_query_value(query_value, r,
visitor.get()));
r->intersect(visitor.get());
break;
}
case InvertedIndexQueryType::EQUAL_QUERY: {
auto visitor =
std::make_unique<InvertedIndexVisitor<InvertedIndexQueryType::EQUAL_QUERY>>(
- r.get(), bit_map);
+ r.get(), bit_map.get());
RETURN_IF_ERROR(construct_bkd_query_value(query_value, r,
visitor.get()));
r->intersect(visitor.get());
break;
@@ -641,25 +639,26 @@ Status BkdIndexReader::try_query(OlapReaderStatistics*
stats, const std::string&
const void* query_value,
InvertedIndexQueryType query_type,
uint32_t* count) {
try {
- auto index_file_path =
_inverted_index_file_reader->get_index_file_path(&_index_meta);
-
std::shared_ptr<lucene::util::bkd::bkd_reader> r;
auto st = get_bkd_reader(r, stats);
if (!st.ok()) {
- LOG(WARNING) << "get bkd reader for " << index_file_path << "
failed: " << st;
+ LOG(WARNING) << "get bkd reader for "
+ <<
_inverted_index_file_reader->get_index_file_path(&_index_meta)
+ << " failed: " << st;
return st;
}
std::string query_str;
_value_key_coder->full_encode_ascending(query_value, &query_str);
- InvertedIndexQueryCache::CacheKey cache_key {index_file_path,
column_name, query_type,
+ auto index_file_key =
_inverted_index_file_reader->get_index_file_key(&_index_meta);
+ InvertedIndexQueryCache::CacheKey cache_key {index_file_key,
column_name, query_type,
query_str};
auto* cache = InvertedIndexQueryCache::instance();
InvertedIndexQueryCacheHandle cache_handler;
- roaring::Roaring bit_map;
- auto cache_status = handle_query_cache(cache, cache_key,
&cache_handler, stats, &bit_map);
+ std::shared_ptr<roaring::Roaring> bit_map;
+ auto cache_status = handle_query_cache(cache, cache_key,
&cache_handler, stats, bit_map);
if (cache_status.ok()) {
- *count = bit_map.cardinality();
+ *count = bit_map->cardinality();
return Status::OK();
}
@@ -675,22 +674,24 @@ Status BkdIndexReader::try_query(OlapReaderStatistics*
stats, const std::string&
Status BkdIndexReader::query(OlapReaderStatistics* stats, RuntimeState*
runtime_state,
const std::string& column_name, const void*
query_value,
- InvertedIndexQueryType query_type,
roaring::Roaring* bit_map) {
+ InvertedIndexQueryType query_type,
+ std::shared_ptr<roaring::Roaring>& bit_map) {
SCOPED_RAW_TIMER(&stats->inverted_index_query_timer);
try {
- auto index_file_path =
_inverted_index_file_reader->get_index_file_path(&_index_meta);
-
std::shared_ptr<lucene::util::bkd::bkd_reader> r;
auto st = get_bkd_reader(r, stats);
if (!st.ok()) {
- LOG(WARNING) << "get bkd reader for " << index_file_path << "
failed: " << st;
+ LOG(WARNING) << "get bkd reader for "
+ <<
_inverted_index_file_reader->get_index_file_path(&_index_meta)
+ << " failed: " << st;
return st;
}
std::string query_str;
_value_key_coder->full_encode_ascending(query_value, &query_str);
- InvertedIndexQueryCache::CacheKey cache_key {index_file_path,
column_name, query_type,
+ auto index_file_key =
_inverted_index_file_reader->get_index_file_key(&_index_meta);
+ InvertedIndexQueryCache::CacheKey cache_key {index_file_key,
column_name, query_type,
query_str};
auto* cache = InvertedIndexQueryCache::instance();
InvertedIndexQueryCacheHandle cache_handler;
@@ -700,16 +701,16 @@ Status BkdIndexReader::query(OlapReaderStatistics* stats,
RuntimeState* runtime_
}
RETURN_IF_ERROR(invoke_bkd_query(query_value, query_type, r, bit_map));
- std::shared_ptr<roaring::Roaring> query_bitmap =
- std::make_shared<roaring::Roaring>(*bit_map);
- query_bitmap->runOptimize();
- cache->insert(cache_key, query_bitmap, &cache_handler);
+ bit_map->runOptimize();
+ cache->insert(cache_key, bit_map, &cache_handler);
VLOG_DEBUG << "BKD index search column: " << column_name
<< " result: " << bit_map->cardinality();
return Status::OK();
} catch (const CLuceneError& e) {
+ LOG(ERROR) << "BKD Query CLuceneError Occurred, error msg: " <<
e.what() << " file_path:"
+ <<
_inverted_index_file_reader->get_index_file_path(&_index_meta);
return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
"BKD Query CLuceneError Occurred, error msg: {}", e.what());
}
@@ -1123,11 +1124,9 @@ lucene::util::bkd::relation
InvertedIndexVisitor<QT>::compare(std::vector<uint8_
}
}
-Status InvertedIndexIterator::read_from_inverted_index(const std::string&
column_name,
- const void* query_value,
- InvertedIndexQueryType
query_type,
- uint32_t
segment_num_rows,
- roaring::Roaring*
bit_map, bool skip_try) {
+Status InvertedIndexIterator::read_from_inverted_index(
+ const std::string& column_name, const void* query_value,
InvertedIndexQueryType query_type,
+ uint32_t segment_num_rows, std::shared_ptr<roaring::Roaring>& bit_map,
bool skip_try) {
if (UNLIKELY(_reader == nullptr)) {
throw CLuceneError(CL_ERR_NullPointer, "bkd index reader is null",
false);
}
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.h
b/be/src/olap/rowset/segment_v2/inverted_index_reader.h
index 2db7e77b43a..63002da5c92 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.h
@@ -86,7 +86,8 @@ public:
std::unique_ptr<InvertedIndexIterator>*
iterator) = 0;
virtual Status query(OlapReaderStatistics* stats, RuntimeState*
runtime_state,
const std::string& column_name, const void*
query_value,
- InvertedIndexQueryType query_type, roaring::Roaring*
bit_map) = 0;
+ InvertedIndexQueryType query_type,
+ std::shared_ptr<roaring::Roaring>& bit_map) = 0;
virtual Status try_query(OlapReaderStatistics* stats, const std::string&
column_name,
const void* query_value, InvertedIndexQueryType
query_type,
uint32_t* count) = 0;
@@ -119,11 +120,12 @@ public:
virtual Status handle_query_cache(InvertedIndexQueryCache* cache,
const InvertedIndexQueryCache::CacheKey&
cache_key,
InvertedIndexQueryCacheHandle*
cache_handler,
- OlapReaderStatistics* stats,
roaring::Roaring* bit_map) {
+ OlapReaderStatistics* stats,
+ std::shared_ptr<roaring::Roaring>&
bit_map) {
if (cache->lookup(cache_key, cache_handler)) {
stats->inverted_index_query_cache_hit++;
SCOPED_RAW_TIMER(&stats->inverted_index_query_bitmap_copy_timer);
- *bit_map = *cache_handler->get_bitmap();
+ bit_map = cache_handler->get_bitmap();
return Status::OK();
}
stats->inverted_index_query_cache_miss++;
@@ -158,7 +160,8 @@ public:
std::unique_ptr<InvertedIndexIterator>* iterator)
override;
Status query(OlapReaderStatistics* stats, RuntimeState* runtime_state,
const std::string& column_name, const void* query_value,
- InvertedIndexQueryType query_type, roaring::Roaring* bit_map)
override;
+ InvertedIndexQueryType query_type,
+ std::shared_ptr<roaring::Roaring>& bit_map) override;
Status try_query(OlapReaderStatistics* stats, const std::string&
column_name,
const void* query_value, InvertedIndexQueryType
query_type,
uint32_t* count) override {
@@ -190,7 +193,8 @@ public:
std::unique_ptr<InvertedIndexIterator>* iterator)
override;
Status query(OlapReaderStatistics* stats, RuntimeState* runtime_state,
const std::string& column_name, const void* query_value,
- InvertedIndexQueryType query_type, roaring::Roaring* bit_map)
override;
+ InvertedIndexQueryType query_type,
+ std::shared_ptr<roaring::Roaring>& bit_map) override;
Status try_query(OlapReaderStatistics* stats, const std::string&
column_name,
const void* query_value, InvertedIndexQueryType
query_type,
uint32_t* count) override {
@@ -249,7 +253,8 @@ public:
Status query(OlapReaderStatistics* stats, RuntimeState* runtime_state,
const std::string& column_name, const void* query_value,
- InvertedIndexQueryType query_type, roaring::Roaring* bit_map)
override;
+ InvertedIndexQueryType query_type,
+ std::shared_ptr<roaring::Roaring>& bit_map) override;
Status try_query(OlapReaderStatistics* stats, const std::string&
column_name,
const void* query_value, InvertedIndexQueryType
query_type,
uint32_t* count) override;
@@ -257,7 +262,7 @@ public:
std::shared_ptr<lucene::util::bkd::bkd_reader>
r, uint32_t* count);
Status invoke_bkd_query(const void* query_value, InvertedIndexQueryType
query_type,
std::shared_ptr<lucene::util::bkd::bkd_reader> r,
- roaring::Roaring* bit_map);
+ std::shared_ptr<roaring::Roaring>& bit_map);
template <InvertedIndexQueryType QT>
Status construct_bkd_query_value(const void* query_value,
std::shared_ptr<lucene::util::bkd::bkd_reader> r,
@@ -281,7 +286,8 @@ public:
Status read_from_inverted_index(const std::string& column_name, const
void* query_value,
InvertedIndexQueryType query_type,
uint32_t segment_num_rows,
- roaring::Roaring* bit_map, bool skip_try =
false);
+ std::shared_ptr<roaring::Roaring>& bit_map,
+ bool skip_try = false);
Status try_read_from_inverted_index(const std::string& column_name, const
void* query_value,
InvertedIndexQueryType query_type,
uint32_t* count);
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index d923b5e9d67..ad5b25f6620 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -864,7 +864,7 @@ Status
SegmentIterator::_apply_inverted_index_except_leafnode_of_andnode(
}
Status SegmentIterator::_apply_index_except_leafnode_of_andnode() {
- for (auto pred : _col_preds_except_leafnode_of_andnode) {
+ for (auto* pred : _col_preds_except_leafnode_of_andnode) {
auto pred_type = pred->type();
bool is_support = pred_type == PredicateType::EQ || pred_type ==
PredicateType::NE ||
pred_type == PredicateType::LT || pred_type ==
PredicateType::LE ||
@@ -899,11 +899,10 @@ Status
SegmentIterator::_apply_index_except_leafnode_of_andnode() {
}
std::string pred_result_sign = _gen_predicate_result_sign(pred);
- _rowid_result_for_index.emplace(
- std::make_pair(pred_result_sign, std::make_pair(true,
bitmap)));
+ _rowid_result_for_index.emplace(pred_result_sign, std::make_pair(true,
std::move(bitmap)));
}
- for (auto pred : _col_preds_except_leafnode_of_andnode) {
+ for (auto* pred : _col_preds_except_leafnode_of_andnode) {
auto column_name = _schema->column(pred->column_id())->name();
if (!_remaining_conjunct_roots.empty() &&
_check_column_pred_all_push_down(column_name, true,
@@ -992,10 +991,9 @@ Status
SegmentIterator::_apply_inverted_index_on_column_predicate(
} else {
bool need_remaining_after_evaluate =
_column_has_fulltext_index(pred->column_id()) &&
PredicateTypeTraits::is_equal_or_list(pred->type());
- roaring::Roaring bitmap = _row_bitmap;
Status res = pred->evaluate(_storage_name_and_type[pred->column_id()],
_inverted_index_iterators[pred->column_id()].get(), num_rows(),
- &bitmap);
+ &_row_bitmap);
if (!res.ok()) {
if (_downgrade_without_index(res, need_remaining_after_evaluate)) {
remaining_predicates.emplace_back(pred);
@@ -1011,11 +1009,9 @@ Status
SegmentIterator::_apply_inverted_index_on_column_predicate(
auto pred_type = pred->type();
if (pred_type == PredicateType::MATCH) {
std::string pred_result_sign = _gen_predicate_result_sign(pred);
- _rowid_result_for_index.emplace(
- std::make_pair(pred_result_sign, std::make_pair(false,
bitmap)));
+ _rowid_result_for_index.emplace(pred_result_sign,
std::make_pair(false, _row_bitmap));
}
- _row_bitmap &= bitmap;
if (_row_bitmap.isEmpty()) {
// all rows have been pruned, no need to process further predicates
*continue_apply = false;
@@ -1060,7 +1056,7 @@ Status
SegmentIterator::_apply_inverted_index_on_block_column_predicate(
std::string column_name = _schema->column(column_id)->name();
auto res = pred->evaluate(column_name,
_inverted_index_iterators[column_id].get(),
- num_rows(), &output_result);
+ num_rows(), &_row_bitmap);
if (res.ok()) {
if (_check_column_pred_all_push_down(column_name) &&
@@ -1070,7 +1066,6 @@ Status
SegmentIterator::_apply_inverted_index_on_block_column_predicate(
}
}
no_need_to_pass_column_predicate_set.insert(predicate_set.begin(),
predicate_set.end());
- _row_bitmap &= output_result;
if (_row_bitmap.isEmpty()) {
// all rows have been pruned, no need to process further
predicates
*continue_apply = false;
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index b3d290e53b8..897dc403ef3 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -464,7 +464,7 @@ Status ExecEnv::_init_mem_env() {
inverted_index_cache_limit = inverted_index_cache_limit / 2;
}
_inverted_index_searcher_cache =
-
InvertedIndexSearcherCache::create_global_instance(inverted_index_cache_limit);
+
InvertedIndexSearcherCache::create_global_instance(inverted_index_cache_limit,
256);
LOG(INFO) << "Inverted index searcher cache memory limit: "
<< PrettyPrinter::print(inverted_index_cache_limit, TUnit::BYTES)
<< ", origin config value: " <<
config::inverted_index_searcher_cache_limit;
@@ -478,7 +478,7 @@ Status ExecEnv::_init_mem_env() {
inverted_index_query_cache_limit = inverted_index_query_cache_limit /
2;
}
_inverted_index_query_cache =
-
InvertedIndexQueryCache::create_global_cache(inverted_index_query_cache_limit);
+
InvertedIndexQueryCache::create_global_cache(inverted_index_query_cache_limit,
256);
LOG(INFO) << "Inverted index query match cache memory limit: "
<< PrettyPrinter::print(inverted_index_cache_limit, TUnit::BYTES)
<< ", origin config value: " <<
config::inverted_index_query_cache_limit;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
