This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new ddecccd2a6c branch-4.2: [fix](be) Preserve hidden VERSION values
during ordered compaction (#70240)
ddecccd2a6c is described below
commit ddecccd2a6ce67c0fdbb84be027daadce0910fe5
Author: Jerry Hu <[email protected]>
AuthorDate: Thu Nov 5 17:21:27 2026 +0800
branch-4.2: [fix](be) Preserve hidden VERSION values during ordered
compaction (#70240)
### What problem does this PR solve?
A non-empty single-version rowset may store a placeholder for the hidden
VERSION column. Its reader synthesizes the published value, but ordered
file-link compaction can combine several such rowsets into a range whose
reader exposes the stored placeholder.
This backport of #69395 to branch-4.2 rejects an ordered link only when
the selected tidy prefix spans multiple versions and contains a
non-empty single-version rowset with a hidden VERSION column. The
existing merge path then materializes the published value.
Already-materialized range rowsets and safe tidy prefixes remain
eligible for linking. The branch-4.2 change is limited to its VERSION
path; it does not import the separate changelog or TSO paths.
### Release note
Prevent ordered file-link compaction from exposing incorrect hidden
VERSION values.
### Validation
- [x] ASAN BE and FE build, including FE Checkstyle.
- [x] `OrderedDataCompactionTest.*`: 6 tests passed.
- [x] Both hidden-VERSION regression suites passed with generated
expected output and a normal verification run.
- [x] `clang-format`, `check-format`, and `git diff --check` passed.
---
be/src/storage/compaction/compaction.cpp | 15 ++
.../compaction/ordered_data_compaction_test.cpp | 157 ++++++++++++++++++++-
...st_ordered_compaction_hidden_version_column.out | 12 ++
...ction_hidden_version_no_light_schema_change.out | 12 ++
...ordered_compaction_hidden_version_column.groovy | 103 ++++++++++++++
...on_hidden_version_no_light_schema_change.groovy | 104 ++++++++++++++
6 files changed, 400 insertions(+), 3 deletions(-)
diff --git a/be/src/storage/compaction/compaction.cpp
b/be/src/storage/compaction/compaction.cpp
index 2fcb22f62db..1c8dff950e9 100644
--- a/be/src/storage/compaction/compaction.cpp
+++ b/be/src/storage/compaction/compaction.cpp
@@ -578,6 +578,18 @@ bool CompactionMixin::handle_ordered_data_compaction() {
}
}
+ // Non-empty singleton rowsets store a placeholder for the publish-time
VERSION column.
+ // A linked range rowset would expose it instead of the version
synthesized on read.
+ // Check only the tidy prefix actually selected for linking.
+ auto links_unmaterialized_version = [this] {
+ return _input_rowsets.front()->start_version() !=
_input_rowsets.back()->end_version() &&
+ std::ranges::any_of(_input_rowsets, [](const RowsetSharedPtr&
rowset) {
+ return rowset->num_rows() > 0 &&
+ rowset->start_version() == rowset->end_version() &&
+ rowset->tablet_schema()->field_index(VERSION_COL) >=
0;
+ });
+ };
+
// check if rowsets are tidy so we can just modify meta and do link
// files to handle compaction
auto input_size = _input_rowsets.size();
@@ -593,6 +605,9 @@ bool CompactionMixin::handle_ordered_data_compaction() {
}
}
}
+ if (links_unmaterialized_version()) {
+ return false;
+ }
// most rowset of current compaction is nonoverlapping
// just handle nonoverlappint rowsets
auto st = do_compact_ordered_rowsets();
diff --git a/be/test/storage/compaction/ordered_data_compaction_test.cpp
b/be/test/storage/compaction/ordered_data_compaction_test.cpp
index 9cc991de654..4c4588336d4 100644
--- a/be/test/storage/compaction/ordered_data_compaction_test.cpp
+++ b/be/test/storage/compaction/ordered_data_compaction_test.cpp
@@ -113,14 +113,15 @@ protected:
ExecEnv::GetInstance()->set_storage_engine(nullptr);
}
- TabletSchemaSPtr create_schema(KeysType keys_type = DUP_KEYS) {
+ TabletSchemaSPtr create_schema(KeysType keys_type = DUP_KEYS, bool
with_version_column = false,
+ bool unset_version_col_idx = false) {
TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
TabletSchemaPB tablet_schema_pb;
tablet_schema_pb.set_keys_type(keys_type);
tablet_schema_pb.set_num_short_key_columns(1);
tablet_schema_pb.set_num_rows_per_row_block(1024);
tablet_schema_pb.set_compress_kind(COMPRESS_NONE);
- tablet_schema_pb.set_next_column_unique_id(4);
+ tablet_schema_pb.set_next_column_unique_id(4 + with_version_column);
ColumnPB* column_1 = tablet_schema_pb.add_column();
column_1->set_unique_id(1);
@@ -157,6 +158,23 @@ protected:
column_3->set_is_bf_column(false);
}
+ if (with_version_column) {
+ ColumnPB* version_column = tablet_schema_pb.add_column();
+ version_column->set_unique_id(4);
+ version_column->set_name(VERSION_COL);
+ version_column->set_type("BIGINT");
+ version_column->set_is_key(false);
+ version_column->set_is_nullable(false);
+ version_column->set_visible(false);
+ version_column->set_default_value("0");
+ version_column->set_length(8);
+ version_column->set_index_length(8);
+ version_column->set_aggregation("REPLACE");
+ if (!unset_version_col_idx) {
+ tablet_schema_pb.set_version_col_idx(3);
+ }
+ }
+
tablet_schema->init_from_pb(tablet_schema_pb);
return tablet_schema;
}
@@ -284,7 +302,8 @@ protected:
RowsetSharedPtr create_rowset(
TabletSchemaSPtr tablet_schema, TabletSharedPtr tablet,
const SegmentsOverlapPB& overlap,
- std::vector<std::vector<std::tuple<int64_t, int64_t>>>
rowset_data) {
+ std::vector<std::vector<std::tuple<int64_t, int64_t>>> rowset_data,
+ Version version = Version(-1, -1), int64_t stored_version = 0) {
RowsetWriterContext writer_context;
if (overlap == NONOVERLAPPING) {
for (auto i = 1; i < rowset_data.size(); i++) {
@@ -297,6 +316,9 @@ protected:
}
create_rowset_writer_context(tablet_schema, tablet->tablet_path(),
overlap, UINT32_MAX,
&writer_context);
+ if (version.first >= 0) {
+ writer_context.version = version;
+ }
auto res = RowsetFactory::create_rowset_writer(*engine_ref,
writer_context, true);
EXPECT_TRUE(res.has_value()) << res.error();
@@ -316,6 +338,10 @@ protected:
uint8_t num = 0;
columns[2]->insert_data((const char*)&num, sizeof(num));
}
+ if (auto version_idx =
tablet_schema->field_index(VERSION_COL); version_idx >= 0) {
+ columns[version_idx]->insert_data((const
char*)&stored_version,
+ sizeof(stored_version));
+ }
num_rows++;
}
block.set_columns(std::move(columns));
@@ -332,6 +358,31 @@ protected:
return rowset;
}
+ std::vector<int64_t> read_hidden_values(Rowset* rowset, const
TabletSchemaSPtr& tablet_schema,
+ ColumnId hidden_column_id) {
+ std::vector<uint32_t> return_columns
{static_cast<uint32_t>(hidden_column_id)};
+ RowsetReaderContext reader_context;
+ reader_context.tablet_schema = tablet_schema;
+ reader_context.return_columns = &return_columns;
+ RowsetReaderSharedPtr reader;
+ create_and_init_rowset_reader(rowset, reader_context, &reader);
+ std::vector<int64_t> values;
+ Status status = Status::OK();
+ do {
+ Block block = tablet_schema->create_block(return_columns);
+ status = reader->next_batch(&block);
+ for (size_t i = 0; i < block.rows(); ++i) {
+ values.push_back(block.get_by_position(0).column->get_int(i));
+ }
+ } while (status.ok());
+ EXPECT_EQ(status, Status::Error<END_OF_FILE>(""));
+ return values;
+ }
+
+ std::vector<int64_t> read_versions(Rowset* rowset, const TabletSchemaSPtr&
tablet_schema) {
+ return read_hidden_values(rowset, tablet_schema,
tablet_schema->field_index(VERSION_COL));
+ }
+
void init_rs_meta(RowsetMetaSharedPtr& pb1, int64_t start, int64_t end) {
std::string json_rowset_meta = R"({
"rowset_id": 540085,
@@ -558,6 +609,106 @@ TEST_F(OrderedDataCompactionTest, test_01) {
}
}
+TEST_F(OrderedDataCompactionTest,
test_version_col_singleton_rowset_not_linked) {
+ auto tablet_schema = create_schema(UNIQUE_KEYS, true);
+ auto tablet = create_tablet(*tablet_schema, false, 10000, false);
+
ASSERT_TRUE(io::global_local_filesystem()->create_directory(tablet->tablet_path()).ok());
+
+ std::vector<std::vector<std::vector<std::tuple<int64_t, int64_t>>>>
input_data;
+ generate_input_data(2, 1, 2, input_data);
+ std::vector<RowsetSharedPtr> input_rowsets;
+ for (const auto& rowset_data : input_data) {
+ auto rowset = create_rowset(tablet_schema, tablet, NONOVERLAPPING,
rowset_data);
+ ASSERT_EQ(rowset->start_version(), rowset->end_version());
+ ASSERT_EQ(rowset->num_rows(), 2);
+ EXPECT_EQ(read_versions(rowset.get(), tablet_schema),
+ (std::vector<int64_t> {rowset->end_version(),
rowset->end_version()}));
+ input_rowsets.push_back(std::move(rowset));
+ }
+
+ CumulativeCompaction compaction(*engine_ref, tablet);
+ compaction._input_rowsets = std::move(input_rowsets);
+ ASSERT_FALSE(compaction.handle_ordered_data_compaction());
+}
+
+TEST_F(OrderedDataCompactionTest,
test_version_col_with_unset_index_singleton_not_linked) {
+ // A schema created without light schema change has the VERSION column but
no index in PB.
+ auto tablet_schema = create_schema(UNIQUE_KEYS, true, true);
+ ASSERT_EQ(tablet_schema->version_col_idx(), -1);
+ ASSERT_EQ(tablet_schema->field_index(VERSION_COL), 3);
+ auto tablet = create_tablet(*tablet_schema, false, 10000, false);
+
ASSERT_TRUE(io::global_local_filesystem()->create_directory(tablet->tablet_path()).ok());
+
+ std::vector<std::vector<std::vector<std::tuple<int64_t, int64_t>>>>
input_data;
+ generate_input_data(2, 1, 2, input_data);
+ std::vector<RowsetSharedPtr> input_rowsets;
+ for (const auto& rowset_data : input_data) {
+ auto rowset = create_rowset(tablet_schema, tablet, NONOVERLAPPING,
rowset_data);
+ ASSERT_EQ(rowset->start_version(), rowset->end_version());
+ ASSERT_EQ(rowset->num_rows(), 2);
+ EXPECT_EQ(read_versions(rowset.get(), tablet_schema),
+ (std::vector<int64_t> {rowset->end_version(),
rowset->end_version()}));
+ input_rowsets.push_back(std::move(rowset));
+ }
+
+ CumulativeCompaction compaction(*engine_ref, tablet);
+ compaction._input_rowsets = std::move(input_rowsets);
+ ASSERT_FALSE(compaction.handle_ordered_data_compaction());
+}
+
+TEST_F(OrderedDataCompactionTest,
test_version_col_materialized_range_rowsets_can_link) {
+ auto tablet_schema = create_schema(UNIQUE_KEYS, true);
+ auto tablet = create_tablet(*tablet_schema, false, 10000, false);
+
ASSERT_TRUE(io::global_local_filesystem()->create_directory(tablet->tablet_path()).ok());
+
+ std::vector<std::vector<std::vector<std::tuple<int64_t, int64_t>>>>
input_data;
+ generate_input_data(2, 1, 2, input_data);
+ auto first = create_rowset(tablet_schema, tablet, NONOVERLAPPING,
input_data[0],
+ Version(1000, 1001), 1001);
+ auto second = create_rowset(tablet_schema, tablet, NONOVERLAPPING,
input_data[1],
+ Version(1002, 1003), 1003);
+ EXPECT_EQ(read_versions(first.get(), tablet_schema), (std::vector<int64_t>
{1001, 1001}));
+ EXPECT_EQ(read_versions(second.get(), tablet_schema),
(std::vector<int64_t> {1003, 1003}));
+
+ CumulativeCompaction compaction(*engine_ref, tablet);
+ compaction._input_rowsets = {first, second};
+ ASSERT_TRUE(compaction.handle_ordered_data_compaction());
+ EXPECT_EQ(compaction._output_rowset->start_version(), 1000);
+ EXPECT_EQ(compaction._output_rowset->end_version(), 1003);
+ EXPECT_EQ(read_versions(compaction._output_rowset.get(), tablet_schema),
+ (std::vector<int64_t> {1001, 1001, 1003, 1003}));
+}
+
+TEST_F(OrderedDataCompactionTest, test_unsafe_tail_does_not_block_safe_prefix)
{
+ auto tablet_schema = create_schema(UNIQUE_KEYS, true);
+ auto tablet = create_tablet(*tablet_schema, false, 10000, false);
+
ASSERT_TRUE(io::global_local_filesystem()->create_directory(tablet->tablet_path()).ok());
+
+ std::vector<std::vector<std::vector<std::tuple<int64_t, int64_t>>>>
input_data;
+ generate_input_data(5, 1, 2, input_data);
+ std::vector<RowsetSharedPtr> input_rowsets;
+ for (size_t i = 0; i < input_data.size() - 1; ++i) {
+ input_rowsets.push_back(create_rowset(tablet_schema, tablet,
NONOVERLAPPING, input_data[i],
+ Version(1000 + 2 * i, 1001 + 2 *
i), 1001 + 2 * i));
+ }
+ // Mark a two-segment singleton tail untidy; it must not veto the tidy
range prefix.
+ auto tail_data = std::vector<std::vector<std::tuple<int64_t, int64_t>>> {
+ {input_data.back()[0][0]}, {input_data.back()[0][0]}};
+ auto tail = create_rowset(tablet_schema, tablet, OVERLAPPING, tail_data,
Version(1008, 1008));
+ ASSERT_EQ(tail->num_segments(), 2);
+ ASSERT_TRUE(tail->is_segments_overlapping());
+ input_rowsets.push_back(tail);
+
+ CumulativeCompaction compaction(*engine_ref, tablet);
+ compaction._input_rowsets = std::move(input_rowsets);
+ ASSERT_TRUE(compaction.handle_ordered_data_compaction());
+ EXPECT_EQ(compaction._input_rowsets.size(), 4);
+ EXPECT_EQ(compaction._output_rowset->start_version(), 1000);
+ EXPECT_EQ(compaction._output_rowset->end_version(), 1007);
+ EXPECT_EQ(read_versions(compaction._output_rowset.get(), tablet_schema),
+ (std::vector<int64_t> {1001, 1001, 1003, 1003, 1005, 1005, 1007,
1007}));
+}
+
TEST_F(OrderedDataCompactionTest, test_index_disk_size) {
auto num_input_rowset = 3;
auto num_segments = 2;
diff --git
a/regression-test/data/fault_injection_p0/test_ordered_compaction_hidden_version_column.out
b/regression-test/data/fault_injection_p0/test_ordered_compaction_hidden_version_column.out
new file mode 100644
index 00000000000..db1ee8131fa
--- /dev/null
+++
b/regression-test/data/fault_injection_p0/test_ordered_compaction_hidden_version_column.out
@@ -0,0 +1,12 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !versions --
+10 true
+11 true
+20 true
+21 true
+
+-- !min_max --
+true true
+
+-- !version_filters --
+2 2
diff --git
a/regression-test/data/fault_injection_p0/test_ordered_compaction_hidden_version_no_light_schema_change.out
b/regression-test/data/fault_injection_p0/test_ordered_compaction_hidden_version_no_light_schema_change.out
new file mode 100644
index 00000000000..db1ee8131fa
--- /dev/null
+++
b/regression-test/data/fault_injection_p0/test_ordered_compaction_hidden_version_no_light_schema_change.out
@@ -0,0 +1,12 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !versions --
+10 true
+11 true
+20 true
+21 true
+
+-- !min_max --
+true true
+
+-- !version_filters --
+2 2
diff --git
a/regression-test/suites/fault_injection_p0/test_ordered_compaction_hidden_version_column.groovy
b/regression-test/suites/fault_injection_p0/test_ordered_compaction_hidden_version_column.groovy
new file mode 100644
index 00000000000..8a8e662aa4b
--- /dev/null
+++
b/regression-test/suites/fault_injection_p0/test_ordered_compaction_hidden_version_column.groovy
@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_ordered_compaction_hidden_version_column", "nonConcurrent") {
+ if (isCloudMode()) {
+ return
+ }
+
+ setBeConfigTemporary([
+ enable_ordered_data_compaction: true,
+ ordered_data_compaction_min_segment_size: 1,
+ segments_key_bounds_truncation_threshold: -1
+ ]) {
+ sql "DROP TABLE IF EXISTS
test_ordered_compaction_hidden_version_column"
+ sql """
+ CREATE TABLE test_ordered_compaction_hidden_version_column (
+ k INT NOT NULL,
+ v INT NOT NULL
+ ) UNIQUE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "false",
+ "disable_auto_compaction" = "true")
+ """
+
+ sql "INSERT INTO test_ordered_compaction_hidden_version_column VALUES
(10, 10), (11, 11)"
+ sql "INSERT INTO test_ordered_compaction_hidden_version_column VALUES
(20, 20), (21, 21)"
+ sql "SET show_hidden_columns = true"
+ def firstVersions = sql """SELECT __DORIS_VERSION_COL__
+ FROM test_ordered_compaction_hidden_version_column WHERE k < 20
ORDER BY k"""
+ def secondVersions = sql """SELECT __DORIS_VERSION_COL__
+ FROM test_ordered_compaction_hidden_version_column WHERE k >= 20
ORDER BY k"""
+ assert firstVersions.size() == 2 && secondVersions.size() == 2
+ long v1 = firstVersions[0][0].toString().toLong()
+ long v2 = secondVersions[0][0].toString().toLong()
+ assert firstVersions.every { it[0].toString().toLong() == v1 }
+ assert secondVersions.every { it[0].toString().toLong() == v2 }
+ assert v1 > 0 && v2 == v1 + 1
+
+ def tabletInfo = sql_return_maparray(
+ "SHOW TABLETS FROM
test_ordered_compaction_hidden_version_column").get(0)
+ def tabletId = tabletInfo.TabletId
+ def (inputCode, inputBody, inputError) = curl("GET",
tabletInfo.MetaUrl)
+ assert inputCode == 0 : inputError
+ def inputRowsets = parseJson(inputBody.trim()).rs_metas
+ for (def version : [v1, v2]) {
+ assert inputRowsets.count {
+ it.start_version.toString().toLong() == version &&
+ it.end_version.toString().toLong() == version &&
+ it.num_rows.toString().toLong() == 2 &&
+ it.num_segments.toString().toLong() == 1
+ } == 1
+ }
+ try {
+ GetDebugPoint().enableDebugPointForAllBEs(
+
"SizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets",
+ [tablet_id: "${tabletId}", start_version: "${v1}",
end_version: "${v2}"])
+
trigger_and_wait_compaction("test_ordered_compaction_hidden_version_column",
"cumulative")
+ } finally {
+ GetDebugPoint().disableDebugPointForAllBEs(
+
"SizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets")
+ }
+
+ def metaUrl = sql_return_maparray(
+ "SHOW TABLETS FROM
test_ordered_compaction_hidden_version_column").get(0).MetaUrl
+ def (code, body, error) = curl("GET", metaUrl)
+ assert code == 0 : error
+ def rowsets = parseJson(body.trim()).rs_metas
+ assert rowsets.count {
+ it.start_version.toString().toLong() == v1 &&
+ it.end_version.toString().toLong() == v2 &&
+ it.num_rows.toString().toLong() == 4
+ } == 1
+
+ order_qt_versions """SELECT k,
+ (__DORIS_VERSION_COL__ = CASE WHEN k < 20 THEN ${v1} ELSE ${v2}
END) AS correct_version
+ FROM test_ordered_compaction_hidden_version_column ORDER BY k"""
+ qt_min_max """SELECT MIN(__DORIS_VERSION_COL__) = ${v1},
+ MAX(__DORIS_VERSION_COL__) = ${v2}
+ FROM test_ordered_compaction_hidden_version_column"""
+ qt_version_filters """SELECT
+ (SELECT COUNT(*) FROM test_ordered_compaction_hidden_version_column
+ WHERE __DORIS_VERSION_COL__ = ${v1}),
+ (SELECT COUNT(*) FROM test_ordered_compaction_hidden_version_column
+ WHERE __DORIS_VERSION_COL__ = ${v2})"""
+
+ }
+}
diff --git
a/regression-test/suites/fault_injection_p0/test_ordered_compaction_hidden_version_no_light_schema_change.groovy
b/regression-test/suites/fault_injection_p0/test_ordered_compaction_hidden_version_no_light_schema_change.groovy
new file mode 100644
index 00000000000..5b245cd885c
--- /dev/null
+++
b/regression-test/suites/fault_injection_p0/test_ordered_compaction_hidden_version_no_light_schema_change.groovy
@@ -0,0 +1,104 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_ordered_compaction_hidden_version_no_light_schema_change",
"nonConcurrent") {
+ if (isCloudMode()) {
+ return
+ }
+
+ setBeConfigTemporary([
+ enable_ordered_data_compaction: true,
+ ordered_data_compaction_min_segment_size: 1,
+ segments_key_bounds_truncation_threshold: -1
+ ]) {
+ sql "DROP TABLE IF EXISTS
test_ordered_compaction_hidden_version_no_light_schema_change"
+ sql """
+ CREATE TABLE
test_ordered_compaction_hidden_version_no_light_schema_change (
+ k INT NOT NULL,
+ v INT NOT NULL
+ ) UNIQUE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "false",
+ "light_schema_change" = "false",
+ "disable_auto_compaction" = "true")
+ """
+
+ sql "INSERT INTO
test_ordered_compaction_hidden_version_no_light_schema_change VALUES (10, 10),
(11, 11)"
+ sql "INSERT INTO
test_ordered_compaction_hidden_version_no_light_schema_change VALUES (20, 20),
(21, 21)"
+ sql "SET show_hidden_columns = true"
+ def firstVersions = sql """SELECT __DORIS_VERSION_COL__
+ FROM test_ordered_compaction_hidden_version_no_light_schema_change
WHERE k < 20 ORDER BY k"""
+ def secondVersions = sql """SELECT __DORIS_VERSION_COL__
+ FROM test_ordered_compaction_hidden_version_no_light_schema_change
WHERE k >= 20 ORDER BY k"""
+ assert firstVersions.size() == 2 && secondVersions.size() == 2
+ long v1 = firstVersions[0][0].toString().toLong()
+ long v2 = secondVersions[0][0].toString().toLong()
+ assert firstVersions.every { it[0].toString().toLong() == v1 }
+ assert secondVersions.every { it[0].toString().toLong() == v2 }
+ assert v1 > 0 && v2 == v1 + 1
+
+ def tabletInfo = sql_return_maparray(
+ "SHOW TABLETS FROM
test_ordered_compaction_hidden_version_no_light_schema_change").get(0)
+ def tabletId = tabletInfo.TabletId
+ def (inputCode, inputBody, inputError) = curl("GET",
tabletInfo.MetaUrl)
+ assert inputCode == 0 : inputError
+ def inputRowsets = parseJson(inputBody.trim()).rs_metas
+ for (def version : [v1, v2]) {
+ assert inputRowsets.count {
+ it.start_version.toString().toLong() == version &&
+ it.end_version.toString().toLong() == version &&
+ it.num_rows.toString().toLong() == 2 &&
+ it.num_segments.toString().toLong() == 1
+ } == 1
+ }
+ try {
+ GetDebugPoint().enableDebugPointForAllBEs(
+
"SizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets",
+ [tablet_id: "${tabletId}", start_version: "${v1}",
end_version: "${v2}"])
+
trigger_and_wait_compaction("test_ordered_compaction_hidden_version_no_light_schema_change",
"cumulative")
+ } finally {
+ GetDebugPoint().disableDebugPointForAllBEs(
+
"SizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets")
+ }
+
+ def metaUrl = sql_return_maparray(
+ "SHOW TABLETS FROM
test_ordered_compaction_hidden_version_no_light_schema_change").get(0).MetaUrl
+ def (code, body, error) = curl("GET", metaUrl)
+ assert code == 0 : error
+ def rowsets = parseJson(body.trim()).rs_metas
+ assert rowsets.count {
+ it.start_version.toString().toLong() == v1 &&
+ it.end_version.toString().toLong() == v2 &&
+ it.num_rows.toString().toLong() == 4
+ } == 1
+
+ order_qt_versions """SELECT k,
+ (__DORIS_VERSION_COL__ = CASE WHEN k < 20 THEN ${v1} ELSE ${v2}
END) AS correct_version
+ FROM test_ordered_compaction_hidden_version_no_light_schema_change
ORDER BY k"""
+ qt_min_max """SELECT MIN(__DORIS_VERSION_COL__) = ${v1},
+ MAX(__DORIS_VERSION_COL__) = ${v2}
+ FROM
test_ordered_compaction_hidden_version_no_light_schema_change"""
+ qt_version_filters """SELECT
+ (SELECT COUNT(*) FROM
test_ordered_compaction_hidden_version_no_light_schema_change
+ WHERE __DORIS_VERSION_COL__ = ${v1}),
+ (SELECT COUNT(*) FROM
test_ordered_compaction_hidden_version_no_light_schema_change
+ WHERE __DORIS_VERSION_COL__ = ${v2})"""
+
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
