This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0f7a1c316da [opt](expr) Remove FixedContainer optimization for
StringValueSet (#62243)
0f7a1c316da is described below
commit 0f7a1c316da24e2d7af0332cfb49a486b2402818
Author: Mryange <[email protected]>
AuthorDate: Thu Apr 9 17:52:08 2026 +0800
[opt](expr) Remove FixedContainer optimization for StringValueSet (#62243)
Benchmark results show that `FixedContainer` (SIMD linear scan) is
slower than `DynamicContainer` (hash set) for `std::string`/`StringRef`
when N >= 2. The FixedContainer optimization only benefits integer types
at small sizes, but hurts string lookups due to expensive comparisons
scaling linearly with N.
This PR removes the `FixedContainer` specialization from
`create_string_value_set`, always using the hash-based
`DynamicContainer` instead.
Additionally fixes deprecated `DoNotOptimize` warnings in benchmark
files and adds a new `FixedContainer` vs `DynamicContainer` benchmark.
```
Fixed_Int32_N1
5.28 us 5.25 us 100000
Dynamic_Int32_N1
29.4 us 29.1 us 23538
Fixed_Int32_N2
15.7 us 15.6 us 46383
Dynamic_Int32_N2
41.4 us 41.2 us 16213
Fixed_Int32_N3
25.7 us 25.6 us 27386
Dynamic_Int32_N3
45.9 us 45.5 us 14574
Fixed_Int32_N4
9.45 us 9.40 us 79614
Dynamic_Int32_N4
40.5 us 40.3 us 15873
Fixed_Int32_N5
22.9 us 22.8 us 29587
Dynamic_Int32_N5
46.5 us 46.3 us 14624
Fixed_Int32_N6
31.5 us 31.3 us 22306
Dynamic_Int32_N6
46.5 us 46.3 us 14851
Fixed_Int32_N7
41.6 us 41.4 us 17142
Dynamic_Int32_N7
56.1 us 55.8 us 12542
Fixed_Int32_N8
11.1 us 11.1 us 70730
Dynamic_Int32_N8
43.3 us 42.1 us 12838
Fixed_Int64_N1
5.91 us 5.88 us 121146
Dynamic_Int64_N1
41.5 us 41.3 us 19183
Fixed_Int64_N2
16.0 us 16.0 us 47690
Dynamic_Int64_N2
40.2 us 40.0 us 18290
Fixed_Int64_N3
27.0 us 26.9 us 27097
Dynamic_Int64_N3
46.9 us 46.6 us 15495
Fixed_Int64_N4
24.8 us 24.6 us 28495
Dynamic_Int64_N4
40.5 us 40.2 us 17581
Fixed_Int64_N5
39.9 us 39.6 us 18133
Dynamic_Int64_N5
47.4 us 47.2 us 14943
Fixed_Int64_N6
45.8 us 45.5 us 15396
Dynamic_Int64_N6
49.2 us 48.9 us 14062
Fixed_Int64_N7
55.0 us 54.8 us 12726
Dynamic_Int64_N7
57.6 us 57.3 us 12673
Fixed_Int64_N8
14.5 us 14.4 us 48230
Dynamic_Int64_N8
40.3 us 40.1 us 17560
Fixed_String_N1
243 us 242 us 2884
Dynamic_String_N1
1364 us 1358 us 516
Fixed_String_N2
1195 us 1189 us 591
Dynamic_String_N2
739 us 736 us 946
Fixed_String_N3
163 us 163 us 4433
Dynamic_String_N3
226 us 225 us 3257
Fixed_String_N4
214 us 213 us 3290
Dynamic_String_N4
205 us 204 us 3437
Fixed_String_N5
266 us 265 us 2634
Dynamic_String_N5
220 us 219 us 2900
Fixed_String_N6
313 us 312 us 2246
Dynamic_String_N6
220 us 219 us 3195
Fixed_String_N7
362 us 360 us 1962
Dynamic_String_N7
234 us 233 us 3008
Fixed_String_N8
2586 us 2575 us 274
Dynamic_String_N8
213 us 212 us 3350
```
---
be/benchmark/benchmark_bit_pack.hpp | 12 +-
be/benchmark/benchmark_hybrid_set.hpp | 190 +++++++++++++++++++++++++++++++
be/benchmark/benchmark_main.cpp | 1 +
be/benchmark/benchmark_string.hpp | 34 +++---
be/benchmark/binary_cast_benchmark.hpp | 12 +-
be/src/exprs/create_predicate_function.h | 27 +----
6 files changed, 230 insertions(+), 46 deletions(-)
diff --git a/be/benchmark/benchmark_bit_pack.hpp
b/be/benchmark/benchmark_bit_pack.hpp
index c0e383c6b1c..10b0fc03f8a 100644
--- a/be/benchmark/benchmark_bit_pack.hpp
+++ b/be/benchmark/benchmark_bit_pack.hpp
@@ -65,8 +65,10 @@ static void BM_BitPack(benchmark::State& state) {
std::vector<uint8_t> output(size);
for (auto _ : state) {
- benchmark::DoNotOptimize(test_data.data());
- benchmark::DoNotOptimize(output.data());
+ auto* td = test_data.data();
+ auto* od = output.data();
+ benchmark::DoNotOptimize(td);
+ benchmark::DoNotOptimize(od);
bit_pack(test_data.data(), (uint8_t)n, w, output.data());
benchmark::ClobberMemory();
}
@@ -92,8 +94,10 @@ static void BM_BitPackOptimized(benchmark::State& state) {
ForEncoder<__int128_t> forEncoder(nullptr);
for (auto _ : state) {
- benchmark::DoNotOptimize(test_data.data());
- benchmark::DoNotOptimize(output.data());
+ auto* td = test_data.data();
+ auto* od = output.data();
+ benchmark::DoNotOptimize(td);
+ benchmark::DoNotOptimize(od);
forEncoder.bit_pack(test_data.data(), (uint8_t)n, w, output.data());
benchmark::ClobberMemory();
}
diff --git a/be/benchmark/benchmark_hybrid_set.hpp
b/be/benchmark/benchmark_hybrid_set.hpp
new file mode 100644
index 00000000000..fee58c56013
--- /dev/null
+++ b/be/benchmark/benchmark_hybrid_set.hpp
@@ -0,0 +1,190 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// ============================================================
+// Benchmark: FixedContainer vs DynamicContainer (from hybrid_set.h)
+//
+// Measures find() performance for different container sizes (1-8)
+// and element types (int32_t, int64_t, std::string).
+// ============================================================
+
+#pragma once
+
+#include <benchmark/benchmark.h>
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+#include "exprs/hybrid_set.h"
+
+namespace doris {
+
+// ============================================================
+// Test data generators
+// ============================================================
+
+template <typename T>
+static std::vector<T> generate_values(size_t n);
+
+template <>
+std::vector<int32_t> generate_values<int32_t>(size_t n) {
+ std::vector<int32_t> vals;
+ vals.reserve(n);
+ for (size_t i = 0; i < n; ++i) {
+ vals.push_back(static_cast<int32_t>(i * 7 + 13));
+ }
+ return vals;
+}
+
+template <>
+std::vector<int64_t> generate_values<int64_t>(size_t n) {
+ std::vector<int64_t> vals;
+ vals.reserve(n);
+ for (size_t i = 0; i < n; ++i) {
+ vals.push_back(static_cast<int64_t>(i * 7 + 13));
+ }
+ return vals;
+}
+
+template <>
+std::vector<std::string> generate_values<std::string>(size_t n) {
+ std::vector<std::string> vals;
+ vals.reserve(n);
+ for (size_t i = 0; i < n; ++i) {
+ vals.push_back("key_" + std::to_string(i * 7 + 13));
+ }
+ return vals;
+}
+
+// Number of find() calls per iteration to amortize loop overhead.
+static constexpr size_t FIND_ITERS = 10000;
+
+// ============================================================
+// FixedContainer benchmark: insert N values, then find them
+// ============================================================
+
+template <typename T, size_t N>
+static void BM_FixedContainer_Find(benchmark::State& state) {
+ auto values = generate_values<T>(N);
+
+ // Also prepare a "miss" value for interleaving hit/miss lookups.
+ auto miss_values = generate_values<T>(N + 4);
+
+ for (auto _ : state) {
+ FixedContainer<T, N> container;
+ for (size_t i = 0; i < N; ++i) {
+ container.insert(values[i]);
+ }
+
+ int64_t found = 0;
+ for (size_t iter = 0; iter < FIND_ITERS; ++iter) {
+ // Hit: search for existing value
+ found += container.find(values[iter % N]);
+ // Miss: search for non-existing value
+ found += container.find(miss_values[N + (iter % 4)]);
+ }
+ benchmark::DoNotOptimize(found);
+ }
+}
+
+// ============================================================
+// DynamicContainer benchmark: insert N values, then find them
+// ============================================================
+
+template <typename T, size_t N>
+static void BM_DynamicContainer_Find(benchmark::State& state) {
+ auto values = generate_values<T>(N);
+ auto miss_values = generate_values<T>(N + 4);
+
+ for (auto _ : state) {
+ DynamicContainer<T> container;
+ for (size_t i = 0; i < N; ++i) {
+ container.insert(values[i]);
+ }
+
+ int64_t found = 0;
+ for (size_t iter = 0; iter < FIND_ITERS; ++iter) {
+ found += container.find(values[iter % N]);
+ found += container.find(miss_values[N + (iter % 4)]);
+ }
+ benchmark::DoNotOptimize(found);
+ }
+}
+
+// ============================================================
+// Register benchmarks for int32_t
+// ============================================================
+
+#define REGISTER_FIXED_INT32(N)
\
+ BENCHMARK(BM_FixedContainer_Find<int32_t, N>)->Name("Fixed_Int32_N"
#N)->Unit( \
+ benchmark::kMicrosecond);
\
+ BENCHMARK(BM_DynamicContainer_Find<int32_t, N>)->Name("Dynamic_Int32_N"
#N)->Unit( \
+ benchmark::kMicrosecond);
+
+REGISTER_FIXED_INT32(1)
+REGISTER_FIXED_INT32(2)
+REGISTER_FIXED_INT32(3)
+REGISTER_FIXED_INT32(4)
+REGISTER_FIXED_INT32(5)
+REGISTER_FIXED_INT32(6)
+REGISTER_FIXED_INT32(7)
+REGISTER_FIXED_INT32(8)
+
+// ============================================================
+// Register benchmarks for int64_t
+// ============================================================
+
+#define REGISTER_FIXED_INT64(N)
\
+ BENCHMARK(BM_FixedContainer_Find<int64_t, N>)->Name("Fixed_Int64_N"
#N)->Unit( \
+ benchmark::kMicrosecond);
\
+ BENCHMARK(BM_DynamicContainer_Find<int64_t, N>)->Name("Dynamic_Int64_N"
#N)->Unit( \
+ benchmark::kMicrosecond);
+
+REGISTER_FIXED_INT64(1)
+REGISTER_FIXED_INT64(2)
+REGISTER_FIXED_INT64(3)
+REGISTER_FIXED_INT64(4)
+REGISTER_FIXED_INT64(5)
+REGISTER_FIXED_INT64(6)
+REGISTER_FIXED_INT64(7)
+REGISTER_FIXED_INT64(8)
+
+// ============================================================
+// Register benchmarks for std::string
+// ============================================================
+
+#define REGISTER_FIXED_STRING(N)
\
+ BENCHMARK(BM_FixedContainer_Find<std::string, N>)->Name("Fixed_String_N"
#N)->Unit( \
+ benchmark::kMicrosecond);
\
+ BENCHMARK(BM_DynamicContainer_Find<std::string,
N>)->Name("Dynamic_String_N" #N)->Unit( \
+ benchmark::kMicrosecond);
+
+REGISTER_FIXED_STRING(1)
+REGISTER_FIXED_STRING(2)
+REGISTER_FIXED_STRING(3)
+REGISTER_FIXED_STRING(4)
+REGISTER_FIXED_STRING(5)
+REGISTER_FIXED_STRING(6)
+REGISTER_FIXED_STRING(7)
+REGISTER_FIXED_STRING(8)
+
+#undef REGISTER_FIXED_INT32
+#undef REGISTER_FIXED_INT64
+#undef REGISTER_FIXED_STRING
+
+} // namespace doris
diff --git a/be/benchmark/benchmark_main.cpp b/be/benchmark/benchmark_main.cpp
index f76dbb370e5..21e629e3d6f 100644
--- a/be/benchmark/benchmark_main.cpp
+++ b/be/benchmark/benchmark_main.cpp
@@ -23,6 +23,7 @@
#include "benchmark_column_view.hpp"
#include "benchmark_fastunion.hpp"
#include "benchmark_hll_merge.hpp"
+#include "benchmark_hybrid_set.hpp"
#include "benchmark_string.hpp"
#include "binary_cast_benchmark.hpp"
#include "core/block/block.h"
diff --git a/be/benchmark/benchmark_string.hpp
b/be/benchmark/benchmark_string.hpp
index bab869b693d..fdb1762ed8c 100644
--- a/be/benchmark/benchmark_string.hpp
+++ b/be/benchmark/benchmark_string.hpp
@@ -211,7 +211,8 @@ static void BM_ToBase64Impl_Old(benchmark::State& state) {
for (auto _ : state) {
dst_data.clear();
dst_offsets.clear();
- benchmark::DoNotOptimize(OldToBase64Impl::vector(data, offsets,
dst_data, dst_offsets));
+ auto status = OldToBase64Impl::vector(data, offsets, dst_data,
dst_offsets);
+ benchmark::DoNotOptimize(status);
}
}
@@ -228,7 +229,8 @@ static void BM_ToBase64Impl_New(benchmark::State& state) {
for (auto _ : state) {
dst_data.clear();
dst_offsets.clear();
- benchmark::DoNotOptimize(ToBase64Impl::vector(data, offsets, dst_data,
dst_offsets));
+ auto status = ToBase64Impl::vector(data, offsets, dst_data,
dst_offsets);
+ benchmark::DoNotOptimize(status);
}
}
@@ -258,8 +260,9 @@ static void BM_FromBase64Impl_Old(benchmark::State& state) {
for (auto _ : state) {
dst_data.clear();
dst_offsets.clear();
- benchmark::DoNotOptimize(OldFromBase64Impl::vector(data, offsets,
dst_data, dst_offsets,
-
null_map->get_data()));
+ auto status = OldFromBase64Impl::vector(data, offsets, dst_data,
dst_offsets,
+ null_map->get_data());
+ benchmark::DoNotOptimize(status);
}
}
@@ -277,8 +280,9 @@ static void BM_FromBase64Impl_New(benchmark::State& state) {
for (auto _ : state) {
dst_data.clear();
dst_offsets.clear();
- benchmark::DoNotOptimize(
- FromBase64Impl::vector(data, offsets, dst_data, dst_offsets,
null_map->get_data()));
+ auto status =
+ FromBase64Impl::vector(data, offsets, dst_data, dst_offsets,
null_map->get_data());
+ benchmark::DoNotOptimize(status);
}
}
@@ -307,7 +311,8 @@ static void BM_UnhexImpl_Old(benchmark::State& state) {
for (auto _ : state) {
dst_data.clear();
dst_offsets.clear();
- benchmark::DoNotOptimize(OldUnHexImpl::vector(data, offsets, dst_data,
dst_offsets));
+ auto status = OldUnHexImpl::vector(data, offsets, dst_data,
dst_offsets);
+ benchmark::DoNotOptimize(status);
}
}
@@ -324,8 +329,9 @@ static void BM_UnhexImpl_New(benchmark::State& state) {
for (auto _ : state) {
dst_data.clear();
dst_offsets.clear();
- benchmark::DoNotOptimize(
- UnHexImpl<UnHexImplEmpty>::vector(data, offsets, dst_data,
dst_offsets));
+ auto status =
+ UnHexImpl<UnHexImplEmpty>::vector(data, offsets, dst_data,
dst_offsets);
+ benchmark::DoNotOptimize(status);
}
}
@@ -355,8 +361,9 @@ static void BM_UnhexNullImpl_Old(benchmark::State& state) {
for (auto _ : state) {
dst_data.clear();
dst_offsets.clear();
- benchmark::DoNotOptimize(
- OldUnHexImpl::vector(data, offsets, dst_data, dst_offsets,
&null_map->get_data()));
+ auto status =
+ OldUnHexImpl::vector(data, offsets, dst_data, dst_offsets,
&null_map->get_data());
+ benchmark::DoNotOptimize(status);
}
}
@@ -374,8 +381,9 @@ static void BM_UnhexNullImpl_New(benchmark::State& state) {
for (auto _ : state) {
dst_data.clear();
dst_offsets.clear();
- benchmark::DoNotOptimize(UnHexImpl<UnHexImplNull>::vector(
- data, offsets, dst_data, dst_offsets, &null_map->get_data()));
+ auto status = UnHexImpl<UnHexImplNull>::vector(
+ data, offsets, dst_data, dst_offsets, &null_map->get_data());
+ benchmark::DoNotOptimize(status);
}
}
diff --git a/be/benchmark/binary_cast_benchmark.hpp
b/be/benchmark/binary_cast_benchmark.hpp
index fec9ddb949a..a130b3fc528 100644
--- a/be/benchmark/binary_cast_benchmark.hpp
+++ b/be/benchmark/binary_cast_benchmark.hpp
@@ -138,7 +138,8 @@ static void
BM_BinaryCast_UI64_to_DateTimeV2(benchmark::State& state) {
state.ResumeTiming();
for (auto _ : state) {
- benchmark::DoNotOptimize(test_data.data());
+ auto* td = test_data.data();
+ benchmark::DoNotOptimize(td);
for (size_t i = 0; i < data_size; ++i) {
auto result = binary_cast<uint64_t,
DateV2Value<DateTimeV2ValueType>>(test_data[i]);
@@ -160,7 +161,8 @@ static void
BM_OldBinaryCast_UI64_to_DateTimeV2(benchmark::State& state) {
state.ResumeTiming();
for (auto _ : state) {
- benchmark::DoNotOptimize(test_data.data());
+ auto* td = test_data.data();
+ benchmark::DoNotOptimize(td);
for (size_t i = 0; i < data_size; ++i) {
auto result = old_binary_cast<uint64_t,
DateV2Value<DateTimeV2ValueType>>(test_data[i]);
@@ -183,7 +185,8 @@ static void
BM_BinaryCast_DateTimeV2_to_UI64(benchmark::State& state) {
state.ResumeTiming();
for (auto _ : state) {
- benchmark::DoNotOptimize(test_data.data());
+ auto* td = test_data.data();
+ benchmark::DoNotOptimize(td);
for (size_t i = 0; i < data_size; ++i) {
auto result = binary_cast<DateV2Value<DateTimeV2ValueType>,
uint64_t>(test_data[i]);
@@ -206,7 +209,8 @@ static void
BM_OldBinaryCast_DateTimeV2_to_UI64(benchmark::State& state) {
state.ResumeTiming();
for (auto _ : state) {
- benchmark::DoNotOptimize(test_data.data());
+ auto* td = test_data.data();
+ benchmark::DoNotOptimize(td);
for (size_t i = 0; i < data_size; ++i) {
auto result = old_binary_cast<DateV2Value<DateTimeV2ValueType>,
uint64_t>(test_data[i]);
diff --git a/be/src/exprs/create_predicate_function.h
b/be/src/exprs/create_predicate_function.h
index 94f8802569c..ee8f3a5e927 100644
--- a/be/src/exprs/create_predicate_function.h
+++ b/be/src/exprs/create_predicate_function.h
@@ -188,35 +188,12 @@ inline auto create_set(PrimitiveType type, size_t size,
bool null_aware) {
}
}
-template <size_t N = 0>
inline HybridSetBase* create_string_value_set(bool null_aware) {
- if constexpr (N >= 1 && N <= FIXED_CONTAINER_MAX_SIZE) {
- return new StringValueSet<FixedContainer<StringRef, N>>(null_aware);
- } else {
- return new StringValueSet(null_aware);
- }
+ return new StringValueSet(null_aware);
}
inline HybridSetBase* create_string_value_set(size_t size, bool null_aware) {
- if (size == 1) {
- return create_string_value_set<1>(null_aware);
- } else if (size == 2) {
- return create_string_value_set<2>(null_aware);
- } else if (size == 3) {
- return create_string_value_set<3>(null_aware);
- } else if (size == 4) {
- return create_string_value_set<4>(null_aware);
- } else if (size == 5) {
- return create_string_value_set<5>(null_aware);
- } else if (size == 6) {
- return create_string_value_set<6>(null_aware);
- } else if (size == 7) {
- return create_string_value_set<7>(null_aware);
- } else if (size == FIXED_CONTAINER_MAX_SIZE) {
- return create_string_value_set<FIXED_CONTAINER_MAX_SIZE>(null_aware);
- } else {
- return create_string_value_set(null_aware);
- }
+ return create_string_value_set(null_aware);
}
inline auto create_bloom_filter(PrimitiveType type, bool null_aware) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
