This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new a35707da74f [fix](variant) Write Variant V2 sparse keys in byte order
(#70001)
a35707da74f is described below
commit a35707da74f8df0d6502df95feaf734dd9d20d9d
Author: lihangyu <[email protected]>
AuthorDate: Tue Nov 3 16:37:26 2026 +0800
[fix](variant) Write Variant V2 sparse keys in byte order (#70001)
---
.../segment/variant/v2/variant_shredder.cpp | 5 +
.../variant/variant_column_writer_reader_test.cpp | 87 ++++++++++++++
.../data/variant_p0/sparse_key_byte_order.out | 129 +++++++++++++++++++++
.../suites/variant_p0/sparse_key_byte_order.groovy | 69 +++++++++++
4 files changed, 290 insertions(+)
diff --git a/be/src/storage/segment/variant/v2/variant_shredder.cpp
b/be/src/storage/segment/variant/v2/variant_shredder.cpp
index e9164a513d7..6175cbaef4c 100644
--- a/be/src/storage/segment/variant/v2/variant_shredder.cpp
+++ b/be/src/storage/segment/variant/v2/variant_shredder.cpp
@@ -509,6 +509,11 @@ struct VariantShredder::Impl {
DorisVector<SparsePlan> sparse_plan = build_sparse_plan(selection,
candidate_builders);
select_sparse_statistics(&sparse_plan);
+ // Readers search full path bytes: a-b must precede a.c, unlike
component order.
+ // Select statistics first to retain the existing encounter-order
budget tie-break.
+ std::ranges::sort(sparse_plan, [](const SparsePlan& left, const
SparsePlan& right) {
+ return *left.path < *right.path;
+ });
RETURN_IF_ERROR(append_binary_rows(sparse_plan, sparse_maps));
DorisVector<VariantStatistics>
bucket_statistics(options.sparse_bucket_count);
publish_sparse_statistics(sparse_plan, &bucket_statistics, result);
diff --git a/be/test/storage/variant/variant_column_writer_reader_test.cpp
b/be/test/storage/variant/variant_column_writer_reader_test.cpp
index 0e8ebdaf46e..4b63f624480 100644
--- a/be/test/storage/variant/variant_column_writer_reader_test.cpp
+++ b/be/test/storage/variant/variant_column_writer_reader_test.cpp
@@ -74,6 +74,7 @@
#include "storage/segment/variant/nested_group_provider.h"
#include "storage/segment/variant/nested_group_streaming_write_plan.h"
#include "storage/segment/variant/sparse_column_merge_iterator.h"
+#include "storage/segment/variant/v2/variant_assembler.h"
#include "storage/segment/variant/v2/variant_column_writer.h"
#include "storage/segment/variant/v2/variant_path_builder.h"
#include "storage/segment/variant/v2/variant_shredder.h"
@@ -968,6 +969,92 @@ TEST(VariantPathBuilderTest,
ShredderReusesCanonicalPathsAcrossAppends) {
EXPECT_EQ(selected.type->to_string(*selected.column, 1), "2");
}
+TEST(VariantShredderTest,
SparseRowsUseByteOrderWithoutChangingStatisticsSelection) {
+ ColumnPtr source;
+ DataTypePtr source_type;
+ ASSERT_TRUE(create_variant_writer_source(
+ VariantWriterInput::V2,
+ {R"({"z":1,"a-b":1,"a
b":3,"a$b":4,"a%2Eb":5,"a":{"c":2}})", R"({"z":2})",
+ R"({"z":3})"},
+ 1, false, {}, &source, &source_type)
+ .ok());
+ const auto& values = assert_cast<const ColumnVariantV2&>(*source);
+ for (const auto layout :
{segment_v2::VariantShredderPhysicalLayout::ORDINARY,
+ segment_v2::VariantShredderPhysicalLayout::DOC})
{
+ for (const uint32_t buckets : {1, 3}) {
+ SCOPED_TRACE(testing::Message()
+ << "layout=" << static_cast<int>(layout) << "
buckets=" << buckets);
+ segment_v2::VariantShredderOptions options;
+ options.physical_layout = layout;
+ options.max_subcolumns_count = 1;
+ options.sparse_bucket_count = buckets;
+ options.max_sparse_column_statistics_size = 1;
+ options.doc_bucket_count = buckets;
+ options.doc_materialization_min_rows = 4;
+ segment_v2::VariantShredder shredder(options);
+
segment_v2::VariantShredder::TestAccess::set_binary_cells_per_chunk(shredder,
2);
+ ASSERT_TRUE(shredder.append(values.read_view(), 0,
values.size()).ok());
+ segment_v2::VariantShreddedColumns output;
+ ASSERT_TRUE(shredder.finish(&output).ok());
+ ASSERT_EQ(output.binary_buckets.size(), buckets);
+ for (const auto& bucket : output.binary_buckets) {
+ const auto& map = assert_cast<const
ColumnMap&>(*bucket.column);
+ const auto& keys = assert_cast<const
ColumnString&>(map.get_keys());
+ for (size_t row = 0; row < map.size(); ++row) {
+ std::vector<std::string> actual;
+ for (size_t cell = map.offset_at(row);
+ cell < map.offset_at(row) + map.size_at(row); ++cell)
{
+ actual.push_back(keys.get_data_at(cell).to_string());
+ }
+ auto expected = actual;
+ std::ranges::sort(expected);
+ EXPECT_EQ(actual, expected) << "row=" << row;
+ }
+ }
+ if (layout == segment_v2::VariantShredderPhysicalLayout::ORDINARY)
{
+ ASSERT_EQ(output.materialized.size(), 1);
+ EXPECT_EQ(output.materialized.front().path.get_path(), "z");
+ const std::vector<std::pair<std::string, std::string>> cases {
+ {"a.c", "2"}, {"a-b", "1"}, {"a b", "3"}, {"a$b",
"4"}, {"a%2Eb", "5"}};
+ for (const auto& [path, expected] : cases) {
+ SCOPED_TRACE(path);
+ const uint32_t bucket =
variant_util::variant_binary_shard_of(
+ {path.data(), path.size()}, buckets);
+ const auto& map =
+ assert_cast<const
ColumnMap&>(*output.binary_buckets[bucket].column);
+ segment_v2::variant_v2::VariantAssemblerOptions
read_options;
+ read_options.requested_path = PathInData(path);
+ read_options.storage_map_kind =
segment_v2::variant_v2::StorageMapKind::SPARSE;
+ auto assembler =
segment_v2::variant_v2::VariantAssembler::create(
+ std::move(read_options));
+ ASSERT_TRUE(assembler.has_value()) << assembler.error();
+ ColumnNullable::MutablePtr result;
+ ASSERT_TRUE((*assembler)
+ ->assemble({.num_rows = values.size(),
+ .materialized_columns = {},
+ .storage_map = &map},
+ &result)
+ .ok());
+ const auto& read_values =
+ assert_cast<const
ColumnVariantV2&>(result->get_nested_column());
+ EXPECT_EQ(variant_v2_json_at(read_values, 0), expected);
+ EXPECT_FALSE(result->is_null_at(0));
+ EXPECT_TRUE(result->is_null_at(1));
+ EXPECT_TRUE(result->is_null_at(2));
+ }
+ if (buckets == 1) {
+ // Component order encounters a.c first; physical byte
order must not
+ // change which path wins the one-entry statistics budget.
+
EXPECT_EQ(output.statistics.sparse_column_non_null_size.size(), 1);
+
EXPECT_EQ(output.statistics.sparse_column_non_null_size.at("a.c"), 1);
+
EXPECT_EQ(output.binary_buckets.front().statistics.sparse_column_non_null_size,
+ output.statistics.sparse_column_non_null_size);
+ }
+ }
+ }
+ }
+}
+
TEST(VariantShredderTest, ReusesRootAndNestedPathsForSharedMetadata) {
DataTypeVariantV2SerDe serde;
DataTypeSerDe::FormatOptions format_options;
diff --git a/regression-test/data/variant_p0/sparse_key_byte_order.out
b/regression-test/data/variant_p0/sparse_key_byte_order.out
new file mode 100644
index 00000000000..785f7cbcb32
--- /dev/null
+++ b/regression-test/data/variant_p0/sparse_key_byte_order.out
@@ -0,0 +1,129 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !before_compaction --
+1 11 12 13 14 15 {"c":15}
{"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
+2 \N \N \N \N \N \N {"z":2}
+3 \N \N \N \N \N \N {"z":3}
+4 21 22 23 24 25 {"c":25}
{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5 \N \N \N \N \N \N {"z":5}
+6 \N \N \N \N \N \N {"z":6}
+
+-- !after_compaction --
+1 11 12 13 14 15 {"c":15}
{"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
+2 \N \N \N \N \N \N {"z":2}
+3 \N \N \N \N \N \N {"z":3}
+4 21 22 23 24 25 {"c":25}
{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5 \N \N \N \N \N \N {"z":5}
+6 \N \N \N \N \N \N {"z":6}
+
+-- !before_compaction --
+1 11 12 13 14 15 {"c":15}
{"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
+2 \N \N \N \N \N \N {"z":2}
+3 \N \N \N \N \N \N {"z":3}
+4 21 22 23 24 25 {"c":25}
{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5 \N \N \N \N \N \N {"z":5}
+6 \N \N \N \N \N \N {"z":6}
+
+-- !after_compaction --
+1 11 12 13 14 15 {"c":15}
{"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
+2 \N \N \N \N \N \N {"z":2}
+3 \N \N \N \N \N \N {"z":3}
+4 21 22 23 24 25 {"c":25}
{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5 \N \N \N \N \N \N {"z":5}
+6 \N \N \N \N \N \N {"z":6}
+
+-- !before_compaction --
+1 11 12 13 14 15 {"c":15}
{"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
+2 \N \N \N \N \N \N {"z":2}
+3 \N \N \N \N \N \N {"z":3}
+4 21 22 23 24 25 {"c":25}
{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5 \N \N \N \N \N \N {"z":5}
+6 \N \N \N \N \N \N {"z":6}
+
+-- !after_compaction --
+1 11 12 13 14 15 {"c":15}
{"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
+2 \N \N \N \N \N \N {"z":2}
+3 \N \N \N \N \N \N {"z":3}
+4 21 22 23 24 25 {"c":25}
{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5 \N \N \N \N \N \N {"z":5}
+6 \N \N \N \N \N \N {"z":6}
+
+-- !before_compaction --
+1 11 12 13 14 15 {"c":15}
{"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
+2 \N \N \N \N \N \N {"z":2}
+3 \N \N \N \N \N \N {"z":3}
+4 21 22 23 24 25 {"c":25}
{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5 \N \N \N \N \N \N {"z":5}
+6 \N \N \N \N \N \N {"z":6}
+
+-- !after_compaction --
+1 11 12 13 14 15 {"c":15}
{"a":{"c":15},"a b":12,"a$b":13,"a%2Eb":14,"a-b":11,"z":1}
+2 \N \N \N \N \N \N {"z":2}
+3 \N \N \N \N \N \N {"z":3}
+4 21 22 23 24 25 {"c":25}
{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5 \N \N \N \N \N \N {"z":5}
+6 \N \N \N \N \N \N {"z":6}
+
+-- !before_compaction --
+1 31 32 33 34 35 {"c":35}
{"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
+2 \N \N \N \N \N \N {"z":2}
+3 \N \N \N \N \N \N {"z":3}
+4 21 22 23 24 25 {"c":25}
{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5 \N \N \N \N \N \N {"z":5}
+6 \N \N \N \N \N \N {"z":6}
+
+-- !after_compaction --
+1 31 32 33 34 35 {"c":35}
{"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
+2 \N \N \N \N \N \N {"z":2}
+3 \N \N \N \N \N \N {"z":3}
+4 21 22 23 24 25 {"c":25}
{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5 \N \N \N \N \N \N {"z":5}
+6 \N \N \N \N \N \N {"z":6}
+
+-- !before_compaction --
+1 31 32 33 34 35 {"c":35}
{"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
+2 \N \N \N \N \N \N {"z":2}
+3 \N \N \N \N \N \N {"z":3}
+4 21 22 23 24 25 {"c":25}
{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5 \N \N \N \N \N \N {"z":5}
+6 \N \N \N \N \N \N {"z":6}
+
+-- !after_compaction --
+1 31 32 33 34 35 {"c":35}
{"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
+2 \N \N \N \N \N \N {"z":2}
+3 \N \N \N \N \N \N {"z":3}
+4 21 22 23 24 25 {"c":25}
{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5 \N \N \N \N \N \N {"z":5}
+6 \N \N \N \N \N \N {"z":6}
+
+-- !before_compaction --
+1 31 32 33 34 35 {"c":35}
{"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
+2 \N \N \N \N \N \N {"z":2}
+3 \N \N \N \N \N \N {"z":3}
+4 21 22 23 24 25 {"c":25}
{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5 \N \N \N \N \N \N {"z":5}
+6 \N \N \N \N \N \N {"z":6}
+
+-- !after_compaction --
+1 31 32 33 34 35 {"c":35}
{"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
+2 \N \N \N \N \N \N {"z":2}
+3 \N \N \N \N \N \N {"z":3}
+4 21 22 23 24 25 {"c":25}
{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5 \N \N \N \N \N \N {"z":5}
+6 \N \N \N \N \N \N {"z":6}
+
+-- !before_compaction --
+1 31 32 33 34 35 {"c":35}
{"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
+2 \N \N \N \N \N \N {"z":2}
+3 \N \N \N \N \N \N {"z":3}
+4 21 22 23 24 25 {"c":25}
{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5 \N \N \N \N \N \N {"z":5}
+6 \N \N \N \N \N \N {"z":6}
+
+-- !after_compaction --
+1 31 32 33 34 35 {"c":35}
{"a":{"c":35},"a b":32,"a$b":33,"a%2Eb":34,"a-b":31,"z":1}
+2 \N \N \N \N \N \N {"z":2}
+3 \N \N \N \N \N \N {"z":3}
+4 21 22 23 24 25 {"c":25}
{"a":{"c":25},"a b":22,"a$b":23,"a%2Eb":24,"a-b":21,"z":4}
+5 \N \N \N \N \N \N {"z":5}
+6 \N \N \N \N \N \N {"z":6}
+
diff --git a/regression-test/suites/variant_p0/sparse_key_byte_order.groovy
b/regression-test/suites/variant_p0/sparse_key_byte_order.groovy
new file mode 100644
index 00000000000..654d80ca3ce
--- /dev/null
+++ b/regression-test/suites/variant_p0/sparse_key_byte_order.groovy
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("sparse_key_byte_order", "p0") {
+ // The parser rejects an explicit doc-mode property beside sparse
properties.
+ sql "SET default_variant_enable_doc_mode = false"
+ for (def keyModel : ["DUPLICATE", "UNIQUE"]) {
+ for (def shardCount : [1, 3]) {
+ for (def docMode : [false, true]) {
+ def tableName =
"sparse_key_byte_order_${keyModel.toLowerCase()}_${shardCount}_${docMode}"
+ def mowProperty = keyModel == "UNIQUE" ? ',
"enable_unique_key_merge_on_write"="true"' : ''
+ def variantProperties = docMode ?
+ '"variant_enable_doc_mode"="true",
"variant_enable_nested_group"="false", ' +
+ '"variant_doc_materialization_min_rows"="1000000", ' +
+ "\"variant_doc_hash_shard_count\"=\"${shardCount}\"" :
+ '"variant_max_subcolumns_count"="1",
"variant_enable_typed_paths_to_sparse"="false", ' +
+ '"variant_enable_nested_group"="false", ' +
+ "\"variant_sparse_hash_shard_count\"=\"${shardCount}\""
+ sql "DROP TABLE IF EXISTS ${tableName}"
+ sql """
+ CREATE TABLE ${tableName} (
+ id INT,
+ v VARIANT<PROPERTIES(${variantProperties})>
+ ) ${keyModel} KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num"="1",
"disable_auto_compaction"="true"${mowProperty})
+ """
+ // z is hotter than every other leaf, leaving the punctuation
keys sparse.
+ sql """INSERT INTO ${tableName} VALUES
+ (1, parse_to_variant('{"z":1,"a-b":11,"a
b":12,"a\$b":13,"a%2Eb":14,"a":{"c":15}}')),
+ (2, parse_to_variant('{"z":2}')),
+ (3, parse_to_variant('{"z":3}'))
+ """
+ sql """INSERT INTO ${tableName} VALUES
+ (4, parse_to_variant('{"z":4,"a-b":21,"a
b":22,"a\$b":23,"a%2Eb":24,"a":{"c":25}}')),
+ (5, parse_to_variant('{"z":5}')),
+ (6, parse_to_variant('{"z":6}'))
+ """
+ if (keyModel == "UNIQUE") {
+ sql """INSERT INTO ${tableName} VALUES
+ (1, parse_to_variant('{"z":1,"a-b":31,"a
b":32,"a\$b":33,"a%2Eb":34,"a":{"c":35}}')),
+ (2, parse_to_variant('{"z":2}')),
+ (3, parse_to_variant('{"z":3}'))
+ """
+ }
+ def projection = """SELECT id, v['a-b'], v['a b'], v['a\$b'],
v['a%2Eb'],
+ v['a']['c'], CAST(v['a'] AS STRING), CAST(v AS STRING)
+ FROM ${tableName} ORDER BY id"""
+ qt_before_compaction projection
+ trigger_and_wait_compaction(tableName, "full", 1800)
+ qt_after_compaction projection
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
