This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 11f7e8485dd branch-4.2: [improvement](be) Run-optimize the SNII NULL
bitmap before serializing it #69727 (#69789)
11f7e8485dd is described below
commit 11f7e8485ddf1e85d3e75715757f99f56ee79d3e
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Oct 29 17:20:07 2026 +0800
branch-4.2: [improvement](be) Run-optimize the SNII NULL bitmap before
serializing it #69727 (#69789)
Cherry-picked from #69727
Co-authored-by: Jack <[email protected]>
---
.../index/inverted/snii/format/null_bitmap.cpp | 6 +++++
.../inverted/snii/format/null_bitmap_test.cpp | 31 ++++++++++++++++++++++
2 files changed, 37 insertions(+)
diff --git a/be/src/storage/index/inverted/snii/format/null_bitmap.cpp
b/be/src/storage/index/inverted/snii/format/null_bitmap.cpp
index 12d35c8b4e4..78eb5c55a6d 100644
--- a/be/src/storage/index/inverted/snii/format/null_bitmap.cpp
+++ b/be/src/storage/index/inverted/snii/format/null_bitmap.cpp
@@ -182,6 +182,12 @@ Status NullBitmapWriter::serialization_sizes(uint32_t
doc_count,
return Status::Error<ErrorCode::INVALID_ARGUMENT, false>(
"null bitmap: null serialization size output");
}
+ // Rewrites the containers into their smallest form, as the V2 writer does
before it serializes.
+ // Runs of NULL rows (a path that is absent from most rows of a sparse
column) are the common
+ // case, and as plain array or bitset containers they cost about 8 KiB per
65536 rows instead
+ // of a few bytes. This is the one place that fixes the serialized size,
and finish() sizes
+ // itself through here, so the sizes reported to callers and the bytes
written always agree.
+ bitmap_->runOptimize();
const size_t roaring_bytes = bitmap_->getSizeInBytes();
const size_t prefix_bytes = varint_len(doc_count) +
varint_len(roaring_bytes);
if (roaring_bytes > std::numeric_limits<size_t>::max() - prefix_bytes) {
diff --git a/be/test/storage/index/inverted/snii/format/null_bitmap_test.cpp
b/be/test/storage/index/inverted/snii/format/null_bitmap_test.cpp
index 9cea5338ee1..6ad1bd78f34 100644
--- a/be/test/storage/index/inverted/snii/format/null_bitmap_test.cpp
+++ b/be/test/storage/index/inverted/snii/format/null_bitmap_test.cpp
@@ -22,6 +22,7 @@
#include <algorithm>
#include <cstdint>
#include <limits>
+#include <numeric>
#include <span>
#include <string>
#include <vector>
@@ -80,6 +81,36 @@ TEST(SniiNullBitmap, RoundTripPerDoc) {
}
}
+// A long run of NULL rows -- the usual shape of a path that is absent from
most rows of a sparse
+// column -- must be written as run containers: a few bytes per 65536 rows
instead of 8 KiB of
+// bitset. The size reported ahead of the write must be the size written.
+TEST(SniiNullBitmap, RunsOfNullsSerializeAsRuns) {
+ constexpr uint32_t kRows = 1'000'000;
+ std::vector<uint32_t> nulls(kRows);
+ std::iota(nulls.begin(), nulls.end(), 0U);
+ NullBitmapWriter writer;
+ writer.add_many(nulls);
+
+ doris::snii::format::NullBitmapSerializationSizes sizes;
+ ASSERT_TRUE(writer.serialization_sizes(kRows, &sizes).ok());
+ // 16 containers of one run each; as bitsets they would take 16 * 8 KiB.
+ EXPECT_LT(sizes.roaring_bytes, 256U);
+
+ ByteSink sink;
+ ASSERT_TRUE(writer.finish(kRows, &sink).ok());
+ EXPECT_EQ(sink.size(), sizes.framed_bytes);
+
+ NullBitmapReader reader;
+ ASSERT_TRUE(NullBitmapReader::open(sink.view(), &reader).ok());
+ EXPECT_EQ(reader.null_count(), kRows);
+ EXPECT_TRUE(reader.is_null(0));
+ EXPECT_TRUE(reader.is_null(kRows - 1));
+ EXPECT_FALSE(reader.is_null(kRows));
+ uint64_t decoded_bytes = 0;
+ ASSERT_TRUE(NullBitmapReader::decoded_memory_bytes(sink.view(),
&decoded_bytes).ok());
+ EXPECT_GT(decoded_bytes, 0U);
+}
+
// Writer null_count reflects the number of distinct null docids added.
TEST(SniiNullBitmap, WriterNullCount) {
NullBitmapWriter writer;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
