This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3029e6ec1e8 [improvement](be) Use plain encoding for V3 floating-point
columns (#62649)
3029e6ec1e8 is described below
commit 3029e6ec1e85a0163c8b86bfc79f19117b238487
Author: bobhan1 <[email protected]>
AuthorDate: Thu Aug 27 11:18:56 2026 +0800
[improvement](be) Use plain encoding for V3 floating-point columns (#62649)
Related PR: #63622
Problem Summary: Segment V3 continued to select `BIT_SHUFFLE` for
`FLOAT` and `DOUBLE` columns. The original implementation added a
tablet-schema flag and propagated it through proto, FE, Cloud, and BE
writers. After #63622 split encoding selection into explicit V2 and V3
maps and persisted `storage_format` in `TabletSchema`, that propagation
chain is obsolete. This PR expresses the behavior directly in the V3
default map while preserving the V2 default for compatibility.
Changes:
- Use `PLAIN_ENCODING` as the V3 default for `FLOAT` and `DOUBLE`.
- Keep the V2 default as `BIT_SHUFFLE`.
- Update the authoritative encoding-map unit tests to lock both
behaviors.
- Regenerate all ten affected SegmentFlusher golden files across the two
scalar and four complex Variant V3 cases.
### Release note
Segment V3 now uses PLAIN encoding by default for FLOAT and DOUBLE
columns. Segment V2 behavior is unchanged.
---
be/src/storage/segment/encoding_info.cpp | 4 ++--
be/test/storage/segment/encoding_info_test.cpp | 20 +++++++++++++-------
.../segment_0.dat | Bin 6800 -> 6719 bytes
.../segment_1.dat | Bin 6795 -> 6714 bytes
.../segment_0.dat | Bin 8451 -> 8359 bytes
.../segment_1.dat | Bin 8239 -> 8149 bytes
.../segment_0.dat | Bin 5680 -> 5656 bytes
.../segment_1.dat | Bin 5680 -> 5656 bytes
.../segment_0.dat | Bin 5496 -> 5472 bytes
.../segment_1.dat | Bin 5496 -> 5472 bytes
.../segment_0.dat | Bin 5478 -> 5454 bytes
.../segment_0.dat | Bin 5294 -> 5270 bytes
12 files changed, 15 insertions(+), 9 deletions(-)
diff --git a/be/src/storage/segment/encoding_info.cpp
b/be/src/storage/segment/encoding_info.cpp
index d2b9d99f114..85d01bf1e0c 100644
--- a/be/src/storage/segment/encoding_info.cpp
+++ b/be/src/storage/segment/encoding_info.cpp
@@ -386,8 +386,8 @@ EncodingInfoResolver::EncodingInfoResolver() {
_set_v3_default<FieldType::OLAP_FIELD_TYPE_LARGEINT, PLAIN_ENCODING>();
_set_v3_default<FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT, BIT_SHUFFLE>();
_set_v3_default<FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT, BIT_SHUFFLE>();
- _set_v3_default<FieldType::OLAP_FIELD_TYPE_FLOAT, BIT_SHUFFLE>();
- _set_v3_default<FieldType::OLAP_FIELD_TYPE_DOUBLE, BIT_SHUFFLE>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_FLOAT, PLAIN_ENCODING>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_DOUBLE, PLAIN_ENCODING>();
_set_v3_default<FieldType::OLAP_FIELD_TYPE_CHAR, DICT_ENCODING>();
_set_v3_default<FieldType::OLAP_FIELD_TYPE_VARCHAR, DICT_ENCODING>();
_set_v3_default<FieldType::OLAP_FIELD_TYPE_STRING, DICT_ENCODING>();
diff --git a/be/test/storage/segment/encoding_info_test.cpp
b/be/test/storage/segment/encoding_info_test.cpp
index 6a3b6e76e32..d32c72e4200 100644
--- a/be/test/storage/segment/encoding_info_test.cpp
+++ b/be/test/storage/segment/encoding_info_test.cpp
@@ -91,10 +91,15 @@ TEST_F(EncodingInfoTest, v2_vs_v3_defaults) {
check_split(FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE, "QUANTILE_STATE");
check_split(FieldType::OLAP_FIELD_TYPE_AGG_STATE, "AGG_STATE");
- // Signed integers: V2=BIT_SHUFFLE, V3=PLAIN.
+ // Signed integers and floating-point types: V2=BIT_SHUFFLE, V3=PLAIN.
+ auto check_plain_v3 = [](FieldType type, const std::string& name) {
+ EXPECT_EQ(BIT_SHUFFLE, get_v2_default_encoding(type)) << name << " v2
default";
+ EXPECT_EQ(PLAIN_ENCODING, get_v3_default_encoding(type)) << name << "
v3 default";
+ };
constexpr FieldType bigint_type = FieldType::OLAP_FIELD_TYPE_BIGINT;
- EXPECT_EQ(BIT_SHUFFLE, get_v2_default_encoding(bigint_type));
- EXPECT_EQ(PLAIN_ENCODING, get_v3_default_encoding(bigint_type));
+ check_plain_v3(bigint_type, "BIGINT");
+ check_plain_v3(FieldType::OLAP_FIELD_TYPE_FLOAT, "FLOAT");
+ check_plain_v3(FieldType::OLAP_FIELD_TYPE_DOUBLE, "DOUBLE");
// Value-seek default is only registered for VARCHAR (the only production
caller).
EXPECT_EQ(PREFIX_ENCODING,
@@ -345,9 +350,10 @@ struct EncodingMapEntry {
const char* name;
};
-// Expected V3 default per type. Differs from V2 default in two type families:
-// - binary blobs (HLL/BITMAP/QUANTILE_STATE/AGG_STATE): PLAIN_ENCODING_V2
(vs V2's PLAIN)
+// Expected V3 default per type. Differs from V2 default in three type
families:
+// - binary blobs (HLL/BITMAP/QUANTILE_STATE/AGG_STATE): PLAIN_ENCODING_V3
(vs V2's PLAIN)
// - signed integers (TINYINT..LARGEINT): PLAIN_ENCODING (vs V2's
BIT_SHUFFLE)
+// - floating-point types (FLOAT/DOUBLE): PLAIN_ENCODING (vs V2's
BIT_SHUFFLE)
const std::vector<DefaultExpectation> kV3DefaultExpect = {
{FieldType::OLAP_FIELD_TYPE_TINYINT, PLAIN_ENCODING, "TINYINT"},
{FieldType::OLAP_FIELD_TYPE_SMALLINT, PLAIN_ENCODING, "SMALLINT"},
@@ -356,8 +362,8 @@ const std::vector<DefaultExpectation> kV3DefaultExpect = {
{FieldType::OLAP_FIELD_TYPE_LARGEINT, PLAIN_ENCODING, "LARGEINT"},
{FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT, BIT_SHUFFLE,
"UNSIGNED_BIGINT"},
{FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT, BIT_SHUFFLE, "UNSIGNED_INT"},
- {FieldType::OLAP_FIELD_TYPE_FLOAT, BIT_SHUFFLE, "FLOAT"},
- {FieldType::OLAP_FIELD_TYPE_DOUBLE, BIT_SHUFFLE, "DOUBLE"},
+ {FieldType::OLAP_FIELD_TYPE_FLOAT, PLAIN_ENCODING, "FLOAT"},
+ {FieldType::OLAP_FIELD_TYPE_DOUBLE, PLAIN_ENCODING, "DOUBLE"},
{FieldType::OLAP_FIELD_TYPE_CHAR, DICT_ENCODING, "CHAR"},
{FieldType::OLAP_FIELD_TYPE_VARCHAR, DICT_ENCODING, "VARCHAR"},
{FieldType::OLAP_FIELD_TYPE_STRING, DICT_ENCODING, "STRING"},
diff --git
a/be/test/storage/test_data/segment_flusher_format/all_scalar_values_v3_not_nullable_without_bloom_vertical/segment_0.dat
b/be/test/storage/test_data/segment_flusher_format/all_scalar_values_v3_not_nullable_without_bloom_vertical/segment_0.dat
index 3b552960641..493a0bad583 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/all_scalar_values_v3_not_nullable_without_bloom_vertical/segment_0.dat
and
b/be/test/storage/test_data/segment_flusher_format/all_scalar_values_v3_not_nullable_without_bloom_vertical/segment_0.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/all_scalar_values_v3_not_nullable_without_bloom_vertical/segment_1.dat
b/be/test/storage/test_data/segment_flusher_format/all_scalar_values_v3_not_nullable_without_bloom_vertical/segment_1.dat
index 88ab18cdf7f..edcd401c259 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/all_scalar_values_v3_not_nullable_without_bloom_vertical/segment_1.dat
and
b/be/test/storage/test_data/segment_flusher_format/all_scalar_values_v3_not_nullable_without_bloom_vertical/segment_1.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/all_scalar_values_v3_nullable_with_bloom_horizontal/segment_0.dat
b/be/test/storage/test_data/segment_flusher_format/all_scalar_values_v3_nullable_with_bloom_horizontal/segment_0.dat
index 101e4969349..4fa704f79cb 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/all_scalar_values_v3_nullable_with_bloom_horizontal/segment_0.dat
and
b/be/test/storage/test_data/segment_flusher_format/all_scalar_values_v3_nullable_with_bloom_horizontal/segment_0.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/all_scalar_values_v3_nullable_with_bloom_horizontal/segment_1.dat
b/be/test/storage/test_data/segment_flusher_format/all_scalar_values_v3_nullable_with_bloom_horizontal/segment_1.dat
index ada6c601fef..ef87c6f1273 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/all_scalar_values_v3_nullable_with_bloom_horizontal/segment_1.dat
and
b/be/test/storage/test_data/segment_flusher_format/all_scalar_values_v3_nullable_with_bloom_horizontal/segment_1.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_0.dat
b/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_0.dat
index b607d5d4309..151025699d9 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_0.dat
and
b/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_0.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_1.dat
b/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_1.dat
index f8b08ffaed0..db4be9934a7 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_1.dat
and
b/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_with_variant_bloom_vertical_compressed/segment_1.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_without_variant_bloom_vertical_uncompressed/segment_0.dat
b/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_without_variant_bloom_vertical_uncompressed/segment_0.dat
index 3a201138c91..f180609ba78 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_without_variant_bloom_vertical_uncompressed/segment_0.dat
and
b/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_without_variant_bloom_vertical_uncompressed/segment_0.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_without_variant_bloom_vertical_uncompressed/segment_1.dat
b/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_without_variant_bloom_vertical_uncompressed/segment_1.dat
index 4223b2b44c0..f396653185c 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_without_variant_bloom_vertical_uncompressed/segment_1.dat
and
b/be/test/storage/test_data/segment_flusher_format/complex_v3_not_nullable_without_variant_bloom_vertical_uncompressed/segment_1.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_with_variant_bloom_horizontal_compressed/segment_0.dat
b/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_with_variant_bloom_horizontal_compressed/segment_0.dat
index d472a56a9d4..ee745aa9bd7 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_with_variant_bloom_horizontal_compressed/segment_0.dat
and
b/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_with_variant_bloom_horizontal_compressed/segment_0.dat
differ
diff --git
a/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_without_variant_bloom_horizontal_uncompressed/segment_0.dat
b/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_without_variant_bloom_horizontal_uncompressed/segment_0.dat
index db20be8790f..d7c8fcb0a5e 100644
Binary files
a/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_without_variant_bloom_horizontal_uncompressed/segment_0.dat
and
b/be/test/storage/test_data/segment_flusher_format/complex_v3_nullable_without_variant_bloom_horizontal_uncompressed/segment_0.dat
differ
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
