This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2f760a8fe17 [refactor](be) Use concrete columns for VariantV2 encoded
storage (#66427)
2f760a8fe17 is described below
commit 2f760a8fe17fa7bc124df5ed11e1b4e5f03abcd9
Author: Mryange <[email protected]>
AuthorDate: Mon Aug 10 16:49:05 2026 +0800
[refactor](be) Use concrete columns for VariantV2 encoded storage (#66427)
### What problem does this PR solve?
Problem Summary: ColumnVariantV2 stored its fixed encoded metadata,
metadata-id, and value columns through IColumn pointers. This obscured
their invariant types and required repeated runtime casts throughout
encoded-state operations. Store these subcolumns through their concrete
COW pointer types, update the read view and SerDe boundaries
accordingly, and keep checked conversions only at generic IColumn
interface boundaries.
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
.../core/column/variant_v2/column_variant_v2.cpp | 185 ++++++++++-----------
be/src/core/column/variant_v2/column_variant_v2.h | 24 ++-
.../variant_v2/column_variant_v2_read_view.cpp | 24 +--
.../data_type_serde/data_type_variant_v2_serde.cpp | 9 +-
be/test/core/column/column_variant_v2_test.cpp | 8 +-
5 files changed, 122 insertions(+), 128 deletions(-)
diff --git a/be/src/core/column/variant_v2/column_variant_v2.cpp
b/be/src/core/column/variant_v2/column_variant_v2.cpp
index 6a74ac091ee..6b37eb64792 100644
--- a/be/src/core/column/variant_v2/column_variant_v2.cpp
+++ b/be/src/core/column/variant_v2/column_variant_v2.cpp
@@ -46,7 +46,7 @@
namespace doris {
namespace {
-using MetaIdsColumn = ColumnVector<TYPE_UINT32>;
+using MetaIdsColumn = ColumnVariantV2::MetadataIdsColumn;
constexpr uint32_t UNMAPPED_METADATA_ID = std::numeric_limits<uint32_t>::max();
constexpr std::array<char, 3> EMPTY_OBJECT_VALUE {static_cast<char>(0x02), 0,
0};
@@ -107,12 +107,17 @@ void validate_offsets(StringRef bytes, std::span<const
uint32_t> offsets,
}
}
-void require_exclusive(const IColumn::WrappedPtr& column, std::string_view
description) {
- const auto& immutable = static_cast<const IColumn::Ptr&>(column);
- DORIS_CHECK(immutable->is_exclusive())
+template <typename WrappedPtr>
+void require_exclusive(const WrappedPtr& column, std::string_view description)
{
+ DORIS_CHECK(column->is_exclusive())
<< "ColumnVariantV2 " << description << " must be COW-detached
before mutation";
}
+template <typename ColumnType, typename ColumnPtrType>
+typename ColumnType::Ptr cast_column_ptr(ColumnPtrType column) {
+ return ColumnType::cast_to_column_ptr(assert_cast<const
ColumnType*>(column.get()));
+}
+
void reserve_rows(ColumnString& values, MetaIdsColumn& metadata_ids, size_t
value_bytes,
size_t rows) {
const size_t final_value_bytes = values.get_chars().size() + value_bytes;
@@ -316,22 +321,16 @@ ValidatedTypedInput validate_typed_input(ColumnPtr
column, DataTypePtr scalar_ty
} // namespace
#ifdef BE_TEST
-void ColumnVariantV2::TestAccess::replace_encoded_subcolumn(ColumnVariantV2&
column, size_t index,
- ColumnPtr
replacement) {
+void ColumnVariantV2::TestAccess::replace_metadata_ids(ColumnVariantV2& column,
+ MetadataIdsColumn::Ptr
replacement) {
DORIS_CHECK(!column._typed);
- switch (index) {
- case 0:
- static_cast<IColumn::Ptr&>(column._metadatas) = std::move(replacement);
- break;
- case 1:
- static_cast<IColumn::Ptr&>(column._meta_ids) = std::move(replacement);
- break;
- case 2:
- static_cast<IColumn::Ptr&>(column._values) = std::move(replacement);
- break;
- default:
- DORIS_CHECK(false) << "ColumnVariantV2 test subcolumn index is out of
range";
- }
+ static_cast<MetadataIdsColumn::Ptr&>(column._meta_ids) =
std::move(replacement);
+}
+
+void ColumnVariantV2::TestAccess::replace_values(ColumnVariantV2& column,
+ ColumnString::Ptr
replacement) {
+ DORIS_CHECK(!column._typed);
+ static_cast<ColumnString::Ptr&>(column._values) = std::move(replacement);
}
#endif
@@ -385,9 +384,9 @@ void ColumnVariantV2::ensure_encoded() {
encoded =
encode_typed_column<Type>(nullable, column, scale);
});
- static_cast<IColumn::Ptr&>(_metadatas) = std::move(encoded.metadatas);
- static_cast<IColumn::Ptr&>(_meta_ids) = std::move(encoded.metadata_ids);
- static_cast<IColumn::Ptr&>(_values) = std::move(encoded.values);
+ static_cast<ColumnString::Ptr&>(_metadatas) = std::move(encoded.metadatas);
+ static_cast<MetadataIdsColumn::Ptr&>(_meta_ids) =
std::move(encoded.metadata_ids);
+ static_cast<ColumnString::Ptr&>(_values) = std::move(encoded.values);
static_cast<IColumn::Ptr&>(_typed).reset();
_typed_type.reset();
_check_invariants();
@@ -469,9 +468,9 @@ void ColumnVariantV2::sanity_check() const {
}
_check_invariants();
if (!_typed) {
- const auto& metadatas = assert_cast<const ColumnString&>(*_metadatas);
- const auto& metadata_ids = assert_cast<const
MetaIdsColumn&>(*_meta_ids).get_data();
- const auto& values = assert_cast<const ColumnString&>(*_values);
+ const auto& metadatas = *_metadatas;
+ const auto& metadata_ids = _meta_ids->get_data();
+ const auto& values = *_values;
for (size_t id = 0; id < metadatas.size(); ++id) {
const StringRef metadata = metadatas.get_data_at(id);
validate_variant_metadata({metadata.data, metadata.size});
@@ -490,9 +489,9 @@ void ColumnVariantV2::for_each_subcolumn(ColumnCallback
callback) const {
if (_typed) {
callback(*static_cast<const IColumn::Ptr&>(_typed));
} else {
- callback(*static_cast<const IColumn::Ptr&>(_metadatas));
- callback(*static_cast<const IColumn::Ptr&>(_meta_ids));
- callback(*static_cast<const IColumn::Ptr&>(_values));
+ callback(*_metadatas);
+ callback(*_meta_ids);
+ callback(*_values);
}
}
@@ -500,9 +499,9 @@ void ColumnVariantV2::mutate_subcolumns() {
if (_typed) {
mutate_subcolumn(_typed);
} else {
- mutate_subcolumn(_metadatas);
- mutate_subcolumn(_meta_ids);
- mutate_subcolumn(_values);
+ mutate_subcolumn<ColumnString>(_metadatas);
+ mutate_subcolumn<MetadataIdsColumn>(_meta_ids);
+ mutate_subcolumn<ColumnString>(_values);
}
}
@@ -511,7 +510,7 @@ void ColumnVariantV2::clear() {
mutate_subcolumn(_typed);
_typed->clear();
} else {
- auto& metadata_ptr = static_cast<IColumn::Ptr&>(_metadatas);
+ auto& metadata_ptr = static_cast<ColumnString::Ptr&>(_metadatas);
if (metadata_ptr->is_exclusive()) {
_metadatas->clear();
} else {
@@ -563,8 +562,8 @@ void ColumnVariantV2::insert_encoded_rows( //
NOLINT(readability-function-size)
require_exclusive(_meta_ids, "metadata ids");
require_exclusive(_values, "values");
- auto& values = assert_cast<ColumnString&>(*_values);
- auto& metadata_ids = assert_cast<MetaIdsColumn&>(*_meta_ids);
+ auto& values = *_values;
+ auto& metadata_ids = *_meta_ids;
reserve_rows(values, metadata_ids, data.value_bytes.size, rows);
if (data.meta_ids.empty()) {
@@ -626,8 +625,8 @@ void ColumnVariantV2::insert_encoded_batch(const
VariantBatchBuilder& block) {
require_exclusive(_meta_ids, "metadata ids");
require_exclusive(_values, "values");
- auto& values = assert_cast<ColumnString&>(*_values);
- auto& metadata_ids = assert_cast<MetaIdsColumn&>(*_meta_ids);
+ auto& values = *_values;
+ auto& metadata_ids = *_meta_ids;
reserve_rows(values, metadata_ids, value_bytes.size, rows);
const uint32_t id = _find_or_insert_metadata({metadata.data,
metadata.size});
@@ -642,12 +641,11 @@ VariantRef ColumnVariantV2::get_value_ref(size_t row)
const {
DCHECK(!_typed);
DCHECK(_typed_type == nullptr);
DCHECK_LT(row, size());
- const auto& metadata_ids = assert_cast<const
MetaIdsColumn&>(*_meta_ids).get_data();
+ const auto& metadata_ids = _meta_ids->get_data();
const uint32_t metadata_id = metadata_ids[row];
DCHECK_LT(metadata_id, _metadatas->size());
- const StringRef metadata =
- assert_cast<const
ColumnString&>(*_metadatas).get_data_at(metadata_id);
- const StringRef value = assert_cast<const
ColumnString&>(*_values).get_data_at(row);
+ const StringRef metadata = _metadatas->get_data_at(metadata_id);
+ const StringRef value = _values->get_data_at(row);
return {.metadata = {.data = metadata.data, .size = metadata.size}, .value
= value};
}
@@ -686,8 +684,8 @@ void ColumnVariantV2::insert_many_defaults(size_t length) {
require_exclusive(_values, "values");
const size_t value_bytes = length * EMPTY_OBJECT_VALUE.size();
- auto& values = assert_cast<ColumnString&>(*_values);
- auto& metadata_ids = assert_cast<MetaIdsColumn&>(*_meta_ids);
+ auto& values = *_values;
+ auto& metadata_ids = *_meta_ids;
DORIS_CHECK_LE(value_bytes, std::numeric_limits<size_t>::max() -
values.get_chars().size())
<< "default value bytes overflow the destination";
reserve_rows(values, metadata_ids, value_bytes, length);
@@ -756,30 +754,29 @@ void ColumnVariantV2::insert_range_from( //
NOLINT(readability-function-size)
require_exclusive(_meta_ids, "metadata ids");
require_exclusive(_values, "values");
- const auto& source_metadatas = assert_cast<const
ColumnString&>(*source._metadatas);
- const auto& source_metadata_ids =
- assert_cast<const MetaIdsColumn&>(*source._meta_ids).get_data();
-
- const auto& source_values = assert_cast<const
ColumnString&>(*source._values);
+ const auto& source_metadatas = *source._metadatas;
+ const auto& source_metadata_ids = source._meta_ids->get_data();
+ const auto& source_values = *source._values;
const auto& source_offsets = source_values.get_offsets();
const size_t value_begin = source_offsets[static_cast<ssize_t>(start) - 1];
const size_t value_end = source_offsets[start + length - 1];
- const bool destination_has_no_metadata = static_cast<const
IColumn::Ptr&>(_metadatas)->empty();
+ const bool destination_has_no_metadata =
+ static_cast<const ColumnString::Ptr&>(_metadatas)->empty();
const bool adopt_metadata = empty() && destination_has_no_metadata;
- const bool already_shared = static_cast<const
IColumn::Ptr&>(_metadatas).get() ==
- static_cast<const
IColumn::Ptr&>(source._metadatas).get();
+ const bool already_shared =
+ static_cast<const ColumnString::Ptr&>(_metadatas).get() ==
source._metadatas.get();
DorisVector<uint32_t> remap;
if (!adopt_metadata && !already_shared) {
remap.assign(source_metadatas.size(), UNMAPPED_METADATA_ID);
}
- auto& values = assert_cast<ColumnString&>(*_values);
- auto& metadata_ids = assert_cast<MetaIdsColumn&>(*_meta_ids);
+ auto& values = *_values;
+ auto& metadata_ids = *_meta_ids;
reserve_rows(values, metadata_ids, value_end - value_begin, length);
if (adopt_metadata) {
_metadatas = source._metadatas;
}
- const bool shared_metadata = static_cast<const
IColumn::Ptr&>(_metadatas).get() ==
- static_cast<const
IColumn::Ptr&>(source._metadatas).get();
+ const bool shared_metadata =
+ static_cast<const ColumnString::Ptr&>(_metadatas).get() ==
source._metadatas.get();
values.insert_range_from(source_values, start, length);
if (shared_metadata) {
metadata_ids.insert_range_from(*source._meta_ids, start, length);
@@ -840,27 +837,27 @@ void ColumnVariantV2::insert_indices_from( //
NOLINT(readability-function-size)
require_exclusive(_meta_ids, "metadata ids");
require_exclusive(_values, "values");
- const auto& source_metadatas = assert_cast<const
ColumnString&>(*source._metadatas);
- const auto& source_metadata_ids =
- assert_cast<const MetaIdsColumn&>(*source._meta_ids).get_data();
- const auto& source_values = assert_cast<const
ColumnString&>(*source._values);
+ const auto& source_metadatas = *source._metadatas;
+ const auto& source_metadata_ids = source._meta_ids->get_data();
+ const auto& source_values = *source._values;
- const bool destination_has_no_metadata = static_cast<const
IColumn::Ptr&>(_metadatas)->empty();
+ const bool destination_has_no_metadata =
+ static_cast<const ColumnString::Ptr&>(_metadatas)->empty();
const bool adopt_metadata = empty() && destination_has_no_metadata;
- const bool already_shared = static_cast<const
IColumn::Ptr&>(_metadatas).get() ==
- static_cast<const
IColumn::Ptr&>(source._metadatas).get();
+ const bool already_shared =
+ static_cast<const ColumnString::Ptr&>(_metadatas).get() ==
source._metadatas.get();
DorisVector<uint32_t> remap;
if (!adopt_metadata && !already_shared) {
remap.assign(source_metadatas.size(), UNMAPPED_METADATA_ID);
}
- auto& values = assert_cast<ColumnString&>(*_values);
- auto& metadata_ids = assert_cast<MetaIdsColumn&>(*_meta_ids);
+ auto& values = *_values;
+ auto& metadata_ids = *_meta_ids;
metadata_ids.get_data().reserve(metadata_ids.size() + rows);
if (adopt_metadata) {
_metadatas = source._metadatas;
}
- const bool shared_metadata = static_cast<const
IColumn::Ptr&>(_metadatas).get() ==
- static_cast<const
IColumn::Ptr&>(source._metadatas).get();
+ const bool shared_metadata =
+ static_cast<const ColumnString::Ptr&>(_metadatas).get() ==
source._metadatas.get();
values.insert_indices_from(source_values, indices_begin, indices_end);
if (shared_metadata) {
metadata_ids.insert_indices_from(*source._meta_ids, indices_begin,
indices_end);
@@ -1184,17 +1181,18 @@ ColumnPtr ColumnVariantV2::filter(const Filter& filter,
ssize_t result_size_hint
result->_check_invariants();
return result;
}
- ColumnPtr filtered_values =
- static_cast<const IColumn::Ptr&>(_values)->filter(filter,
result_size_hint);
- ColumnPtr filtered_metadata_ids =
- static_cast<const IColumn::Ptr&>(_meta_ids)->filter(filter,
result_size_hint);
+ ColumnPtr filtered_values = _values->filter(filter, result_size_hint);
+ ColumnPtr filtered_metadata_ids = _meta_ids->filter(filter,
result_size_hint);
DORIS_CHECK_EQ(filtered_values->size(), filtered_metadata_ids->size())
<< "filtered encoded row counts differ";
auto result = ColumnVariantV2::create();
- static_cast<IColumn::Ptr&>(result->_metadatas) = static_cast<const
IColumn::Ptr&>(_metadatas);
- static_cast<IColumn::Ptr&>(result->_meta_ids) =
std::move(filtered_metadata_ids);
- static_cast<IColumn::Ptr&>(result->_values) = std::move(filtered_values);
+ static_cast<ColumnString::Ptr&>(result->_metadatas) =
+ static_cast<const ColumnString::Ptr&>(_metadatas);
+ static_cast<MetadataIdsColumn::Ptr&>(result->_meta_ids) =
+
cast_column_ptr<MetadataIdsColumn>(std::move(filtered_metadata_ids));
+ static_cast<ColumnString::Ptr&>(result->_values) =
+ cast_column_ptr<ColumnString>(std::move(filtered_values));
result->_check_invariants();
return result;
}
@@ -1241,17 +1239,18 @@ MutableColumnPtr ColumnVariantV2::permute(const
Permutation& permutation, size_t
return result;
}
- MutableColumnPtr permuted_values =
- static_cast<const IColumn::Ptr&>(_values)->permute(permutation,
result_size);
- MutableColumnPtr permuted_metadata_ids =
- static_cast<const IColumn::Ptr&>(_meta_ids)->permute(permutation,
result_size);
+ MutableColumnPtr permuted_values = _values->permute(permutation,
result_size);
+ MutableColumnPtr permuted_metadata_ids = _meta_ids->permute(permutation,
result_size);
DORIS_CHECK_EQ(permuted_values->size(), permuted_metadata_ids->size())
<< "permuted encoded row counts differ";
auto result = ColumnVariantV2::create();
- static_cast<IColumn::Ptr&>(result->_metadatas) = static_cast<const
IColumn::Ptr&>(_metadatas);
- static_cast<IColumn::Ptr&>(result->_meta_ids) =
std::move(permuted_metadata_ids);
- static_cast<IColumn::Ptr&>(result->_values) = std::move(permuted_values);
+ static_cast<ColumnString::Ptr&>(result->_metadatas) =
+ static_cast<const ColumnString::Ptr&>(_metadatas);
+ static_cast<MetadataIdsColumn::Ptr&>(result->_meta_ids) =
+
cast_column_ptr<MetadataIdsColumn>(std::move(permuted_metadata_ids));
+ static_cast<ColumnString::Ptr&>(result->_values) =
+ cast_column_ptr<ColumnString>(std::move(permuted_values));
result->_check_invariants();
return result;
}
@@ -1278,14 +1277,15 @@ MutableColumnPtr ColumnVariantV2::clone_resized(size_t
new_size) const {
}
const size_t copied_rows = std::min(size(), new_size);
- MutableColumnPtr copied_values =
- static_cast<const
IColumn::Ptr&>(_values)->clone_resized(copied_rows);
- MutableColumnPtr copied_metadata_ids =
- static_cast<const
IColumn::Ptr&>(_meta_ids)->clone_resized(copied_rows);
+ MutableColumnPtr copied_values = _values->clone_resized(copied_rows);
+ MutableColumnPtr copied_metadata_ids =
_meta_ids->clone_resized(copied_rows);
auto result = ColumnVariantV2::create();
- static_cast<IColumn::Ptr&>(result->_metadatas) = static_cast<const
IColumn::Ptr&>(_metadatas);
- static_cast<IColumn::Ptr&>(result->_meta_ids) =
std::move(copied_metadata_ids);
- static_cast<IColumn::Ptr&>(result->_values) = std::move(copied_values);
+ static_cast<ColumnString::Ptr&>(result->_metadatas) =
+ static_cast<const ColumnString::Ptr&>(_metadatas);
+ static_cast<MetadataIdsColumn::Ptr&>(result->_meta_ids) =
+ cast_column_ptr<MetadataIdsColumn>(std::move(copied_metadata_ids));
+ static_cast<ColumnString::Ptr&>(result->_values) =
+ cast_column_ptr<ColumnString>(std::move(copied_values));
if (new_size > copied_rows) {
result->insert_many_defaults(new_size - copied_rows);
}
@@ -1327,8 +1327,7 @@ void ColumnVariantV2::replace_column_data(const IColumn&,
size_t, size_t) {
uint32_t ColumnVariantV2::_find_or_insert_metadata(StringRef metadata) {
DORIS_CHECK(metadata.data != nullptr || metadata.size == 0)
<< "metadata bytes have a null pointer";
- const auto& current_metadatas =
- assert_cast<const ColumnString&>(*static_cast<const
IColumn::Ptr&>(_metadatas));
+ const auto& current_metadatas = *static_cast<const
ColumnString::Ptr&>(_metadatas);
for (uint32_t id = 0; id < current_metadatas.size(); ++id) {
if (current_metadatas.get_data_at(id) == metadata) {
return id;
@@ -1340,7 +1339,7 @@ uint32_t
ColumnVariantV2::_find_or_insert_metadata(StringRef metadata) {
}
_detach_metadata_for_write();
- auto& metadatas = assert_cast<ColumnString&>(*_metadatas);
+ auto& metadatas = *_metadatas;
const size_t new_chars_size = metadatas.get_chars().size() + metadata.size;
ColumnString::check_chars_length(new_chars_size, metadatas.size() + 1,
_meta_ids->size());
metadatas.get_chars().reserve(new_chars_size);
@@ -1361,9 +1360,9 @@ void ColumnVariantV2::_adopt_state_from(ColumnVariantV2&
replacement) {
}
void ColumnVariantV2::_detach_metadata_for_write() {
- auto& metadata_ptr = static_cast<IColumn::Ptr&>(_metadatas);
+ auto& metadata_ptr = static_cast<ColumnString::Ptr&>(_metadatas);
if (!metadata_ptr->is_exclusive()) {
- metadata_ptr = std::move(*metadata_ptr).mutate();
+ metadata_ptr =
cast_column_ptr<ColumnString>(std::move(*metadata_ptr).mutate());
}
}
@@ -1383,8 +1382,8 @@ void ColumnVariantV2::_check_invariants() const {
}
DORIS_CHECK(_typed_type == nullptr) << "encoded state cannot retain a
typed data type";
- const auto& metadata_ids = assert_cast<const
MetaIdsColumn&>(*_meta_ids).get_data();
- const auto& values = assert_cast<const ColumnString&>(*_values);
+ const auto& metadata_ids = _meta_ids->get_data();
+ const auto& values = *_values;
DORIS_CHECK_EQ(metadata_ids.size(), values.size())
<< "ColumnVariantV2 encoded row counts differ";
}
diff --git a/be/src/core/column/variant_v2/column_variant_v2.h
b/be/src/core/column/variant_v2/column_variant_v2.h
index e39bef34547..4bf2ae4470d 100644
--- a/be/src/core/column/variant_v2/column_variant_v2.h
+++ b/be/src/core/column/variant_v2/column_variant_v2.h
@@ -26,6 +26,8 @@
#include "core/assert_cast.h"
#include "core/column/column.h"
#include "core/column/column_const.h"
+#include "core/column/column_string.h"
+#include "core/column/column_vector.h"
#include "core/column/variant_v2/column_variant_v2_typed_column.h"
#include "core/custom_allocator.h"
#include "core/data_type/data_type.h"
@@ -41,6 +43,8 @@ class VariantBatchBuilder;
// typed scalar column. Mixed operations materialize the typed state as
encoded bytes on demand.
class ColumnVariantV2 final : public COWHelper<IColumn, ColumnVariantV2> {
public:
+ using MetadataIdsColumn = ColumnVector<TYPE_UINT32>;
+
struct EncodedDataView {
StringRef metadata_bytes;
std::span<const uint32_t> metadata_offsets;
@@ -65,13 +69,14 @@ public:
private:
friend class ColumnVariantV2;
- ReadView(const IColumn* metadatas, const IColumn* metadata_ids, const
IColumn* values);
+ ReadView(const ColumnString* metadatas, const MetadataIdsColumn*
metadata_ids,
+ const ColumnString* values);
ReadView(const IColumn* typed, const DataTypePtr* typed_type);
bool _typed_state = false;
- const IColumn* _metadatas = nullptr;
- const IColumn* _metadata_ids = nullptr;
- const IColumn* _values = nullptr;
+ const ColumnString* _metadatas = nullptr;
+ const MetadataIdsColumn* _metadata_ids = nullptr;
+ const ColumnString* _values = nullptr;
const IColumn* _typed = nullptr;
const DataTypePtr* _typed_type = nullptr;
};
@@ -79,8 +84,9 @@ public:
#ifdef BE_TEST
// Narrow unit-test seam for encoded-state invariant coverage.
struct TestAccess {
- static void replace_encoded_subcolumn(ColumnVariantV2& column, size_t
index,
- ColumnPtr replacement);
+ static void replace_metadata_ids(ColumnVariantV2& column,
+ MetadataIdsColumn::Ptr replacement);
+ static void replace_values(ColumnVariantV2& column, ColumnString::Ptr
replacement);
};
#endif
@@ -189,9 +195,9 @@ private:
// uint32 id costs four bytes per encoded row, but avoids repeating
object-key metadata and
// gives canonical comparison, hashing, subpath lookup, and binary SerDe
O(1) schema access.
// It is required because valid external Variant rows may use different
metadata dictionaries.
- IColumn::WrappedPtr _metadatas;
- IColumn::WrappedPtr _meta_ids;
- IColumn::WrappedPtr _values;
+ ColumnString::WrappedPtr _metadatas;
+ MetadataIdsColumn::WrappedPtr _meta_ids;
+ ColumnString::WrappedPtr _values;
// A non-null _typed always means all encoded buffers are empty and the
entire column has the
// single type described by _typed_type.
diff --git a/be/src/core/column/variant_v2/column_variant_v2_read_view.cpp
b/be/src/core/column/variant_v2/column_variant_v2_read_view.cpp
index 3ce60aed8dd..dd64e49c2de 100644
--- a/be/src/core/column/variant_v2/column_variant_v2_read_view.cpp
+++ b/be/src/core/column/variant_v2/column_variant_v2_read_view.cpp
@@ -15,21 +15,13 @@
// specific language governing permissions and limitations
// under the License.
-#include "core/assert_cast.h"
-#include "core/column/column_string.h"
-#include "core/column/column_vector.h"
#include "core/column/variant_v2/column_variant_v2.h"
namespace doris {
-namespace {
-
-using MetadataIdsColumn = ColumnVector<TYPE_UINT32>;
-
-} // namespace
-
-ColumnVariantV2::ReadView::ReadView(const IColumn* metadatas, const IColumn*
metadata_ids,
- const IColumn* values)
+ColumnVariantV2::ReadView::ReadView(const ColumnString* metadatas,
+ const MetadataIdsColumn* metadata_ids,
+ const ColumnString* values)
: _metadatas(metadatas), _metadata_ids(metadata_ids), _values(values) {
DORIS_CHECK(_metadatas != nullptr);
DORIS_CHECK(_metadata_ids != nullptr);
@@ -57,7 +49,7 @@ size_t ColumnVariantV2::ReadView::metadata_count() const
noexcept {
uint32_t ColumnVariantV2::ReadView::metadata_id_at(size_t row) const {
DORIS_CHECK(!_typed_state) << "metadata_id_at requires ColumnVariantV2
encoded state";
DORIS_CHECK_LT(row, size()) << "ColumnVariantV2 encoded read row is out of
range";
- const auto& ids = assert_cast<const
MetadataIdsColumn&>(*_metadata_ids).get_data();
+ const auto& ids = _metadata_ids->get_data();
const uint32_t id = ids[row];
DORIS_CHECK_LT(id, metadata_count())
<< "ColumnVariantV2 encoded read metadata id is out of range";
@@ -67,14 +59,14 @@ uint32_t ColumnVariantV2::ReadView::metadata_id_at(size_t
row) const {
VariantMetadataRef ColumnVariantV2::ReadView::metadata_at(uint32_t id) const {
DORIS_CHECK(!_typed_state) << "metadata_at requires ColumnVariantV2
encoded state";
DORIS_CHECK_LT(id, metadata_count()) << "ColumnVariantV2 encoded read
metadata is out of range";
- const StringRef metadata = assert_cast<const
ColumnString&>(*_metadatas).get_data_at(id);
+ const StringRef metadata = _metadatas->get_data_at(id);
return {.data = metadata.data, .size = metadata.size};
}
VariantRef ColumnVariantV2::ReadView::value_at(size_t row) const {
DORIS_CHECK(!_typed_state) << "value_at requires ColumnVariantV2 encoded
state";
const uint32_t metadata_id = metadata_id_at(row);
- const StringRef value = assert_cast<const
ColumnString&>(*_values).get_data_at(row);
+ const StringRef value = _values->get_data_at(row);
return {.metadata = metadata_at(metadata_id), .value = value};
}
@@ -94,9 +86,7 @@ ColumnVariantV2::ReadView ColumnVariantV2::read_view() const {
return {static_cast<const IColumn::Ptr&>(_typed).get(), &_typed_type};
}
DORIS_CHECK(_typed_type == nullptr) << "encoded state cannot retain a
typed data type";
- return {static_cast<const IColumn::Ptr&>(_metadatas).get(),
- static_cast<const IColumn::Ptr&>(_meta_ids).get(),
- static_cast<const IColumn::Ptr&>(_values).get()};
+ return {_metadatas.get(), _meta_ids.get(), _values.get()};
}
} // namespace doris
diff --git a/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp
b/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp
index a9a137e4fea..f08509e6769 100644
--- a/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_variant_v2_serde.cpp
@@ -251,9 +251,12 @@ const char* DataTypeVariantV2SerDe::deserialize(const
char* buf, MutableColumnPt
}
}
decoded = ColumnVariantV2::create();
- static_cast<IColumn::Ptr&>(decoded->_metadatas) = std::move(metadatas);
- static_cast<IColumn::Ptr&>(decoded->_meta_ids) = std::move(meta_ids);
- static_cast<IColumn::Ptr&>(decoded->_values) = std::move(values);
+ static_cast<ColumnString::Ptr&>(decoded->_metadatas) =
+ ColumnString::cast_to_column_ptr(assert_cast<const
ColumnString*>(metadatas.get()));
+ static_cast<MetaIdsColumn::Ptr&>(decoded->_meta_ids) =
MetaIdsColumn::cast_to_column_ptr(
+ assert_cast<const MetaIdsColumn*>(meta_ids.get()));
+ static_cast<ColumnString::Ptr&>(decoded->_values) =
+ ColumnString::cast_to_column_ptr(assert_cast<const
ColumnString*>(values.get()));
decoded->sanity_check();
}
if (decoded->size() != saved_rows) {
diff --git a/be/test/core/column/column_variant_v2_test.cpp
b/be/test/core/column/column_variant_v2_test.cpp
index 2665304cfa0..1a3b9ff7d45 100644
--- a/be/test/core/column/column_variant_v2_test.cpp
+++ b/be/test/core/column/column_variant_v2_test.cpp
@@ -319,10 +319,6 @@ uint32_t canonical_crc32c_hash(VariantRef value, uint32_t
seed) {
return sink.digest();
}
-void replace_subcolumn(ColumnVariantV2& column, size_t target, ColumnPtr
replacement) {
- ColumnVariantV2::TestAccess::replace_encoded_subcolumn(column, target,
std::move(replacement));
-}
-
ColumnPtr nullable_int32(std::span<const int32_t> values, std::span<const
uint8_t> null_map) {
EXPECT_EQ(values.size(), null_map.size());
auto nested = ColumnInt32::create();
@@ -1793,7 +1789,7 @@ TEST(ColumnVariantV2Test, EncodedRowCountInvariant) {
{
auto column = ColumnVariantV2::create();
insert_encoded_field(*column, encode_json("1"));
- replace_subcolumn(*column, 2, ColumnString::create());
+ ColumnVariantV2::TestAccess::replace_values(*column,
ColumnString::create());
column->sanity_check();
},
"encoded row counts differ");
@@ -1806,7 +1802,7 @@ TEST(ColumnVariantV2Test, MetadataIdInvariant) {
insert_encoded_field(*column, encode_json("1"));
auto invalid_ids = MetaIdsColumn::create();
invalid_ids->insert_value(9);
- replace_subcolumn(*column, 1, invalid_ids->get_ptr());
+ ColumnVariantV2::TestAccess::replace_metadata_ids(*column,
std::move(invalid_ids));
column->sanity_check();
},
"metadata id is out of range");
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
