This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 7f81c9847f6 branch-4.1: [fix](compaction) Fix sparse vertical
compaction destination offset #66306 (#66429)
7f81c9847f6 is described below
commit 7f81c9847f631f21493e3f1c6a3370322f0a1129
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Aug 4 21:41:51 2026 +0800
branch-4.1: [fix](compaction) Fix sparse vertical compaction destination
offset #66306 (#66429)
Cherry-picked from #66306
Co-authored-by: Jamie <[email protected]>
---
be/src/storage/iterator/vertical_block_reader.cpp | 4 +-
.../compaction/vertical_compaction_test.cpp | 112 ++++++++++++++-------
2 files changed, 79 insertions(+), 37 deletions(-)
diff --git a/be/src/storage/iterator/vertical_block_reader.cpp
b/be/src/storage/iterator/vertical_block_reader.cpp
index d761387050d..dbc4d7f55ba 100644
--- a/be/src/storage/iterator/vertical_block_reader.cpp
+++ b/be/src/storage/iterator/vertical_block_reader.cpp
@@ -575,12 +575,12 @@ Status VerticalBlockReader::_unique_key_next_block(Block*
block, bool* eof) {
// Step 2: Prepare columns - pre-fill NULL for fixed-width,
reserve for others
std::vector<ColumnNullable*> nullable_dst_cols;
std::vector<bool> supports_replace;
+ const size_t old_rows = target_columns.empty() ? 0 :
target_columns.front()->size();
_prepare_sparse_columns(target_columns, actual_rows,
nullable_dst_cols,
supports_replace);
// Step 3: Process each batch
- size_t dst_offset =
- target_columns.empty() ? 0 : target_columns[0]->size() -
actual_rows;
+ size_t dst_offset = old_rows;
for (const auto& batch : batches) {
Block* src_block = batch.block.get();
DCHECK(src_block != nullptr);
diff --git a/be/test/storage/compaction/vertical_compaction_test.cpp
b/be/test/storage/compaction/vertical_compaction_test.cpp
index 2b12e34e41e..079bbb3f9af 100644
--- a/be/test/storage/compaction/vertical_compaction_test.cpp
+++ b/be/test/storage/compaction/vertical_compaction_test.cpp
@@ -38,9 +38,11 @@
#include <vector>
#include "common/status.h"
+#include "core/assert_cast.h"
#include "core/block/block.h"
#include "core/block/column_with_type_and_name.h"
#include "core/column/column.h"
+#include "core/column/column_nullable.h"
#include "core/data_type/data_type.h"
#include "gtest/gtest_pred_impl.h"
#include "io/cache/block_file_cache_factory.h"
@@ -1470,29 +1472,34 @@ TEST_F(VerticalCompactionTest, TestAggKeyVerticalMerge)
{
}
}
-// Test to cover _sample_info->null_count logic in vertical_block_reader.cpp
-// This test creates a UNIQUE_KEYS table with nullable columns and sparse data
+// Test sparse compaction when a value group starts with a reserve-only column.
TEST_F(VerticalCompactionTest,
TestUniqueKeyVerticalMergeWithNullableSparseColumn) {
- // Save original threshold and set to 1 to always enable sparse
optimization
- double original_threshold =
config::sparse_column_compaction_threshold_percent;
+ const auto original_threshold =
config::sparse_column_compaction_threshold_percent;
+ const auto original_columns_per_group =
config::vertical_compaction_num_columns_per_group;
+ Defer restore_config {[original_threshold, original_columns_per_group]() {
+ config::sparse_column_compaction_threshold_percent =
original_threshold;
+ config::vertical_compaction_num_columns_per_group =
original_columns_per_group;
+ }};
config::sparse_column_compaction_threshold_percent = 1.0;
+ config::vertical_compaction_num_columns_per_group = 2;
auto num_input_rowset = 2;
auto num_segments = 1;
auto rows_per_segment = 100;
- // Create schema with nullable column (c2 is nullable)
+ // The first value column only reserves capacity, while the nullable
BIGINT column
+ // pre-fills actual_rows slots for in-place replacement.
TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
TabletSchemaPB tablet_schema_pb;
tablet_schema_pb.set_keys_type(UNIQUE_KEYS);
tablet_schema_pb.set_num_short_key_columns(1);
tablet_schema_pb.set_num_rows_per_row_block(1024);
tablet_schema_pb.set_compress_kind(COMPRESS_NONE);
- tablet_schema_pb.set_next_column_unique_id(4);
+ tablet_schema_pb.set_next_column_unique_id(5);
ColumnPB* column_1 = tablet_schema_pb.add_column();
column_1->set_unique_id(1);
- column_1->set_name("c1");
+ column_1->set_name("k1");
column_1->set_type("INT");
column_1->set_is_key(true);
column_1->set_length(4);
@@ -1500,31 +1507,40 @@ TEST_F(VerticalCompactionTest,
TestUniqueKeyVerticalMergeWithNullableSparseColum
column_1->set_is_nullable(false);
column_1->set_is_bf_column(false);
- // c2 is nullable - this is key for testing _sample_info->null_count
ColumnPB* column_2 = tablet_schema_pb.add_column();
column_2->set_unique_id(2);
- column_2->set_name("c2");
- column_2->set_type("INT");
- column_2->set_length(4);
- column_2->set_index_length(4);
+ column_2->set_name("v0");
+ column_2->set_type("BOOLEAN");
+ column_2->set_length(1);
+ column_2->set_index_length(1);
column_2->set_is_key(false);
- column_2->set_is_nullable(true); // nullable column
+ column_2->set_is_nullable(false);
column_2->set_is_bf_column(false);
- // DELETE_SIGN column required for unique keys
ColumnPB* column_3 = tablet_schema_pb.add_column();
column_3->set_unique_id(3);
- column_3->set_name(DELETE_SIGN);
- column_3->set_type("TINYINT");
- column_3->set_length(1);
- column_3->set_index_length(1);
- column_3->set_is_nullable(false);
+ column_3->set_name("v1");
+ column_3->set_type("BIGINT");
+ column_3->set_length(8);
+ column_3->set_index_length(8);
column_3->set_is_key(false);
+ column_3->set_is_nullable(true);
column_3->set_is_bf_column(false);
+ // DELETE_SIGN column required for unique keys
+ ColumnPB* column_4 = tablet_schema_pb.add_column();
+ column_4->set_unique_id(4);
+ column_4->set_name(DELETE_SIGN);
+ column_4->set_type("TINYINT");
+ column_4->set_length(1);
+ column_4->set_index_length(1);
+ column_4->set_is_nullable(false);
+ column_4->set_is_key(false);
+ column_4->set_is_bf_column(false);
+
tablet_schema->init_from_pb(tablet_schema_pb);
- // Create input rowsets with NULL values in c2
+ // Create input rowsets with mixed NULL values in v1.
std::vector<RowsetSharedPtr> input_rowsets;
for (auto i = 0; i < num_input_rowset; i++) {
RowsetWriterContext rowset_writer_context;
@@ -1546,26 +1562,26 @@ TEST_F(VerticalCompactionTest,
TestUniqueKeyVerticalMergeWithNullableSparseColum
ASSERT_TRUE(res.has_value()) << res.error();
auto rowset_writer = std::move(res).value();
- // Create block with nullable c2 column
Block block = tablet_schema->create_block();
auto columns = std::move(block).mutate_columns();
for (int rid = 0; rid < rows_per_segment; ++rid) {
- int32_t c1 = i * rows_per_segment + rid;
- columns[0]->insert_data((const char*)&c1, sizeof(c1));
+ int32_t k1 = i * rows_per_segment + rid;
+ columns[0]->insert_data((const char*)&k1, sizeof(k1));
+
+ uint8_t v0 = rid % 2;
+ columns[1]->insert_data((const char*)&v0, sizeof(v0));
- // Insert NULL for most rows (sparse pattern: 90% NULL)
+ // The first row is non-NULL, so the first sparse batch must
execute replace.
if (rid % 10 == 0) {
- // non-NULL value
- int32_t c2 = c1 * 10;
- columns[1]->insert_data((const char*)&c2, sizeof(c2));
+ int64_t v1 = static_cast<int64_t>(k1) * 10;
+ columns[2]->insert_data((const char*)&v1, sizeof(v1));
} else {
- // NULL value
- columns[1]->insert_default();
+ columns[2]->insert_default();
}
uint8_t delete_sign = 0;
- columns[2]->insert_data((const char*)&delete_sign,
sizeof(delete_sign));
+ columns[3]->insert_data((const char*)&delete_sign,
sizeof(delete_sign));
}
auto s = add_block_with_columns(rowset_writer.get(), &block, &columns);
@@ -1599,7 +1615,6 @@ TEST_F(VerticalCompactionTest,
TestUniqueKeyVerticalMergeWithNullableSparseColum
RowIdConversion rowid_conversion;
stats.rowid_conversion = &rowid_conversion;
- // This will trigger the _sample_info->null_count logic in
vertical_block_reader.cpp
auto s = Merger::vertical_merge_rowsets(tablet,
ReaderType::READER_BASE_COMPACTION,
*tablet_schema, input_rs_readers,
output_rs_writer.get(), 10000,
num_segments, &stats);
@@ -1608,11 +1623,38 @@ TEST_F(VerticalCompactionTest,
TestUniqueKeyVerticalMergeWithNullableSparseColum
RowsetSharedPtr out_rowset;
ASSERT_EQ(Status::OK(), output_rs_writer->build(out_rowset));
- // Verify output
- EXPECT_EQ(out_rowset->rowset_meta()->num_rows(), num_input_rowset *
rows_per_segment);
+ RowsetReaderContext reader_context;
+ reader_context.tablet_schema = tablet_schema;
+ reader_context.need_ordered_result = false;
+ std::vector<uint32_t> return_columns = {0, 1, 2};
+ reader_context.return_columns = &return_columns;
+ RowsetReaderSharedPtr output_rs_reader;
+ create_and_init_rowset_reader(out_rowset.get(), reader_context,
&output_rs_reader);
- // Restore original threshold
- config::sparse_column_compaction_threshold_percent = original_threshold;
+ Block output_block;
+ size_t output_rows = 0;
+ do {
+ block_create(tablet_schema, &output_block);
+ s = output_rs_reader->next_batch(&output_block);
+ auto columns = output_block.get_columns_with_type_and_name();
+ ASSERT_EQ(columns.size(), 3);
+ const auto& nullable_v1 = assert_cast<const
ColumnNullable&>(*columns[2].column);
+ for (size_t row = 0; row < output_block.rows(); ++row) {
+ int64_t k1 = columns[0].column->get_int(row);
+ EXPECT_EQ(k1, output_rows);
+ EXPECT_EQ(columns[1].column->get_bool(row), k1 % 2 != 0);
+ if (k1 % 10 == 0) {
+ EXPECT_FALSE(nullable_v1.is_null_at(row));
+ EXPECT_EQ(nullable_v1.get_nested_column().get_int(row), k1 *
10);
+ } else {
+ EXPECT_TRUE(nullable_v1.is_null_at(row));
+ }
+ ++output_rows;
+ }
+ } while (s.ok());
+ EXPECT_TRUE(s.is<END_OF_FILE>()) << s;
+ EXPECT_EQ(output_rows, num_input_rowset * rows_per_segment);
+ EXPECT_EQ(out_rowset->rowset_meta()->num_rows(), output_rows);
}
// Test that first-time compaction (no historical sampling) uses footer
raw_data_bytes
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
