This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f25af662f21 [fix](be) Defer Bloom allocation for IN runtime filters
(#68763)
f25af662f21 is described below
commit f25af662f210fd1f4000ef4d6dc89967565d660f
Author: Mryange <[email protected]>
AuthorDate: Thu Oct 8 17:24:06 2026 +0800
[fix](be) Defer Bloom allocation for IN runtime filters (#68763)
`IN_OR_BLOOM` runtime filters allocate and clear a Bloom buffer even
when they remain IN, adding unnecessary CPU and memory overhead across
parallel join instances. Root cause: initialization checks whether the
Bloom function object exists instead of checking the actual filter type.
Initialize the buffer only when the actual type is Bloom, preserving
threshold conversion and merge behavior.
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
.../exec/runtime_filter/runtime_filter_wrapper.cpp | 2 +-
.../runtime_filter/runtime_filter_wrapper_test.cpp | 42 ++++++++++++++++++++++
2 files changed, 43 insertions(+), 1 deletion(-)
diff --git a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp
b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp
index 02102334404..cd5fc2c04bd 100644
--- a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp
+++ b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp
@@ -76,7 +76,7 @@ Status RuntimeFilterWrapper::init(const size_t real_size) {
if (get_real_type() == RuntimeFilterType::IN_FILTER && real_size >
_max_in_num) {
set_state(RuntimeFilterWrapper::State::DISABLED, "reach max in num");
}
- if (_bloom_filter_func) {
+ if (get_real_type() == RuntimeFilterType::BLOOM_FILTER) {
RETURN_IF_ERROR(_bloom_filter_func->init_with_fixed_length(real_size));
}
return Status::OK();
diff --git a/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
b/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
index 2bac50f3cad..9774ce2f2f1 100644
--- a/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
+++ b/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
@@ -782,6 +782,46 @@ TEST_F(RuntimeFilterWrapperTest, TestMinMax) {
}
}
+TEST_F(RuntimeFilterWrapperTest, TestInOrBloomLazyInitialization) {
+ for (bool build_by_runtime_size : {false, true}) {
+ RuntimeFilterParams params {.filter_id = 0,
+ .filter_type =
RuntimeFilterType::IN_OR_BLOOM_FILTER,
+ .column_return_type = TYPE_INT,
+ .null_aware = false,
+ .max_in_num = 16,
+ .runtime_bloom_filter_min_size = 64,
+ .runtime_bloom_filter_max_size = 128,
+ .bloom_filter_size = 64,
+ .build_bf_by_runtime_size =
build_by_runtime_size,
+ .bloom_filter_size_calculated_by_ndv =
false};
+ for (size_t runtime_size : {0, 1, 16, 17}) {
+ SCOPED_TRACE(runtime_size);
+ SCOPED_TRACE(build_by_runtime_size);
+ RuntimeFilterWrapper wrapper(¶ms);
+ ASSERT_NE(wrapper.bloom_filter_func(), nullptr);
+ ASSERT_EQ(wrapper.bloom_filter_func()->get_size(), 0);
+ ASSERT_TRUE(wrapper.init(runtime_size).ok());
+ if (runtime_size <= static_cast<size_t>(params.max_in_num)) {
+ EXPECT_EQ(wrapper.get_real_type(),
RuntimeFilterType::IN_FILTER);
+ EXPECT_EQ(wrapper.bloom_filter_func()->get_size(), 0);
+ EXPECT_EQ(wrapper.bloom_filter_func()->_bloom_filter_alloced,
0);
+ EXPECT_EQ(wrapper.bloom_filter_func()->_bloom_filter, nullptr);
+ PMergeFilterRequest request;
+
ASSERT_TRUE(wrapper.to_protobuf(request.mutable_in_filter()).ok());
+ } else {
+ EXPECT_EQ(wrapper.get_real_type(),
RuntimeFilterType::BLOOM_FILTER);
+ EXPECT_GT(wrapper.bloom_filter_func()->get_size(), 0);
+ auto col = ColumnHelper::create_column<DataTypeInt32>({1, 2,
3});
+ ASSERT_TRUE(wrapper.insert(col, 0).ok());
+ std::vector<uint8_t> matches(3);
+ wrapper.bloom_filter_func()->find_fixed_len(col,
matches.data());
+ EXPECT_TRUE(std::all_of(matches.begin(), matches.end(),
+ [](uint8_t match) { return match != 0;
}));
+ }
+ }
+ }
+}
+
TEST_F(RuntimeFilterWrapperTest, TestInOrBloom) {
std::vector<int> data_vector(10);
std::iota(data_vector.begin(), data_vector.end(), 0);
@@ -850,6 +890,7 @@ TEST_F(RuntimeFilterWrapperTest, TestInOrBloom) {
// Init (keep in filter)
EXPECT_TRUE(wrapper->init(runtime_size).ok());
EXPECT_EQ(wrapper->get_real_type(), RuntimeFilterType::IN_FILTER);
+ EXPECT_EQ(wrapper->bloom_filter_func()->get_size(), 0);
EXPECT_EQ(wrapper->get_state(), RuntimeFilterWrapper::State::UNINITED);
// Insert
auto col = ColumnHelper::create_column<DataType>(data_vector);
@@ -908,6 +949,7 @@ TEST_F(RuntimeFilterWrapperTest, TestInOrBloom) {
EXPECT_TRUE(wrapper->merge(new_wrapper.get()).ok());
EXPECT_EQ(wrapper->hybrid_set()->size(), col->size() * 2);
EXPECT_EQ(wrapper->get_real_type(), RuntimeFilterType::IN_FILTER);
+ EXPECT_EQ(wrapper->bloom_filter_func()->get_size(), 0);
}
{
// In + In -> Bloom
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
