This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch variant-sparse
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/variant-sparse by this push:
new d89ac056382 [test](variant) add ut (#45947)
d89ac056382 is described below
commit d89ac0563823e1fc00b49e947b69bdc916d4043f
Author: Sun Chenyang <[email protected]>
AuthorDate: Mon Jan 6 21:56:10 2025 +0800
[test](variant) add ut (#45947)
---
.../rowset/segment_v2/hierarchical_data_reader.cpp | 24 +-
.../segment_v2/variant_column_writer_impl.cpp | 39 +-
be/src/vec/columns/column_object.cpp | 196 +++--
be/src/vec/columns/column_object.h | 19 +-
be/src/vec/common/schema_util.cpp | 5 +-
be/src/vec/functions/function_variant_element.cpp | 9 +-
be/test/vec/columns/column_object_test.cpp | 795 +++++++++++++++++++++
.../data/variant_p0/variant_with_rowstore.out | 2 +-
8 files changed, 987 insertions(+), 102 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/hierarchical_data_reader.cpp
b/be/src/olap/rowset/segment_v2/hierarchical_data_reader.cpp
index a584cda2e3d..8df56dc05e8 100644
--- a/be/src/olap/rowset/segment_v2/hierarchical_data_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/hierarchical_data_reader.cpp
@@ -29,6 +29,7 @@
#include "vec/common/assert_cast.h"
#include "vec/common/schema_util.h"
#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_factory.hpp"
#include "vec/data_types/data_type_nullable.h"
#include "vec/json/path_in_data.h"
@@ -193,7 +194,7 @@ Status HierarchicalDataReader::_process_nested_columns(
// into a new object column and wrap it with array column using the first
element offsets.The wrapped array column
// will type the type of ColumnObject::NESTED_TYPE, whih is
Nullable<ColumnArray<NULLABLE(ColumnObject)>>.
for (const auto& entry : nested_subcolumns) {
- MutableColumnPtr nested_object = ColumnObject::create(true, false);
+ MutableColumnPtr nested_object = ColumnObject::create(true);
const auto* base_array =
check_and_get_column<ColumnArray>(remove_nullable(entry.second[0].column));
MutableColumnPtr offset =
base_array->get_offsets_ptr()->assume_mutable();
@@ -245,10 +246,8 @@ Status HierarchicalDataReader::_process_nested_columns(
Status HierarchicalDataReader::_init_container(vectorized::MutableColumnPtr&
container,
size_t nrows) {
using namespace vectorized;
- // build variant as container
- container = ColumnObject::create(true, false);
- auto& container_variant = assert_cast<ColumnObject&>(*container);
+ // build variant as container
// add root first
if (_path.get_parts().empty() && _root_reader) {
// auto& root_var =
@@ -259,16 +258,28 @@ Status
HierarchicalDataReader::_init_container(vectorized::MutableColumnPtr& con
// :
assert_cast<vectorized::ColumnObject&>(*_root_reader->column);
// auto column = root_var.get_root();
// auto type = root_var.get_root_type();
+
MutableColumnPtr column = _root_reader->column->get_ptr();
- container_variant.add_sub_column({}, std::move(column),
_root_reader->type);
+ // container_variant.add_sub_column({}, std::move(column),
_root_reader->type);
+ DCHECK(column->size() == nrows);
+ container = ColumnObject::create(_root_reader->type,
std::move(column));
+ } else {
+ auto root_type =
+
vectorized::DataTypeFactory::instance().create_data_type(TypeIndex::Nothing,
true);
+ MutableColumnPtr column = root_type->create_column();
+ column->insert_many_defaults(nrows);
+ container = ColumnObject::create(root_type, std::move(column));
}
+
+ auto& container_variant = assert_cast<ColumnObject&>(*container);
+
// parent path -> subcolumns
std::map<PathInData, PathsWithColumnAndType> nested_subcolumns;
PathsWithColumnAndType non_nested_subcolumns;
RETURN_IF_ERROR(tranverse([&](SubstreamReaderTree::Node& node) {
MutableColumnPtr column = node.data.column->get_ptr();
PathInData relative_path =
node.path.copy_pop_nfront(_path.get_parts().size());
-
+ DCHECK(column->size() == nrows);
if (node.path.has_nested_part()) {
CHECK_EQ(getTypeName(remove_nullable(node.data.type)->get_type_id()),
getTypeName(TypeIndex::Array));
@@ -301,6 +312,7 @@ static std::string_view get_sub_path(const
std::string_view& path, const std::st
Status
HierarchicalDataReader::_process_sparse_column(vectorized::ColumnObject&
container_variant,
size_t nrows) {
using namespace vectorized;
+ container_variant.clear_sparse_column();
if (!_sparse_column_reader) {
container_variant.get_sparse_column()->assume_mutable()->insert_many_defaults(nrows);
return Status::OK();
diff --git a/be/src/olap/rowset/segment_v2/variant_column_writer_impl.cpp
b/be/src/olap/rowset/segment_v2/variant_column_writer_impl.cpp
index 7f35364b172..a26d3d4eab8 100644
--- a/be/src/olap/rowset/segment_v2/variant_column_writer_impl.cpp
+++ b/be/src/olap/rowset/segment_v2/variant_column_writer_impl.cpp
@@ -47,16 +47,12 @@ Status VariantColumnWriterImpl::init() {
// caculate stats info
std::set<std::string> subcolumn_paths;
RETURN_IF_ERROR(_get_subcolumn_paths_from_stats(subcolumn_paths));
- if (subcolumn_paths.empty()) {
- _column = vectorized::ColumnObject::create(true, false);
- } else {
- // create root
- auto col = vectorized::ColumnObject::create(true, true);
- for (const auto& str_path : subcolumn_paths) {
- DCHECK(col->add_sub_column(vectorized::PathInData(str_path), 0));
- }
- _column = std::move(col);
+
+ auto col = vectorized::ColumnObject::create(true);
+ for (const auto& str_path : subcolumn_paths) {
+ DCHECK(col->add_sub_column(vectorized::PathInData(str_path), 0));
}
+ _column = std::move(col);
if (_tablet_column->is_nullable()) {
_null_column = vectorized::ColumnUInt8::create(0);
}
@@ -91,6 +87,9 @@ Status
VariantColumnWriterImpl::_get_subcolumn_paths_from_stats(std::set<std::st
it->second += size;
}
for (const auto& [path, size] :
source_statistics->sparse_column_non_null_size) {
+ if (path.empty()) {
+ CHECK(false);
+ }
auto it = path_to_total_number_of_non_null_values.find(path);
if (it == path_to_total_number_of_non_null_values.end()) {
it = path_to_total_number_of_non_null_values.emplace(path,
0).first;
@@ -112,7 +111,7 @@ Status
VariantColumnWriterImpl::_get_subcolumn_paths_from_stats(std::set<std::st
// Fill subcolumn_paths with first subcolumn paths in sorted list.
// reserve 1 for root column
for (const auto& [size, path] : paths_with_sizes) {
- if (paths.size() < config::variant_max_subcolumns_count - 1) {
+ if (paths.size() < config::variant_max_subcolumns_count) {
VLOG_DEBUG << "pick " << path << " as subcolumn";
paths.emplace(path);
}
@@ -215,6 +214,11 @@ Status
VariantColumnWriterImpl::_process_subcolumns(vectorized::ColumnObject* pt
// convert sub column data from engine format to storage layer format
for (const auto& entry :
vectorized::schema_util::get_sorted_subcolumns(ptr->get_subcolumns())) {
+ const auto& least_common_type = entry->data.get_least_common_type();
+ if (is_nothing(remove_nullable(
+
vectorized::schema_util::get_base_type_of_array(least_common_type)))) {
+ continue;
+ }
if (entry->path.empty()) {
// already handled
continue;
@@ -328,13 +332,14 @@ Status VariantColumnWriterImpl::finalize() {
DCHECK(ptr->is_finalized());
- if (ptr->is_null_root()) {
- auto root_type = vectorized::make_nullable(
- std::make_shared<vectorized::ColumnObject::MostCommonType>());
- auto root_col = root_type->create_column();
- root_col->insert_many_defaults(ptr->rows());
- ptr->create_root(root_type, std::move(root_col));
- }
+ // if (ptr->is_null_root()) {
+ // CHECK(false);
+ // // auto root_type = vectorized::make_nullable(
+ // //
std::make_shared<vectorized::ColumnObject::MostCommonType>());
+ // // auto root_col = root_type->create_column();
+ // // root_col->insert_many_defaults(ptr->rows());
+ // // ptr->create_root(root_type, std::move(root_col));
+ // }
#ifndef NDEBUG
ptr->check_consistency();
diff --git a/be/src/vec/columns/column_object.cpp
b/be/src/vec/columns/column_object.cpp
index d6d5b5c44cc..9582f57aa41 100644
--- a/be/src/vec/columns/column_object.cpp
+++ b/be/src/vec/columns/column_object.cpp
@@ -352,7 +352,7 @@ void get_field_info(const Field& field, FieldInfo* info) {
}
#ifdef NDEBUG
-#define ENABLE_CHECK_CONSISTENCY (void) /* Nothing */
+#define ENABLE_CHECK_CONSISTENCY (void)/* Nothing */
#else
#define ENABLE_CHECK_CONSISTENCY(this) (this)->check_consistency()
#endif
@@ -360,12 +360,21 @@ void get_field_info(const Field& field, FieldInfo* info) {
// current nested level is 2, inside column object
constexpr int CURRENT_SERIALIZE_NESTING_LEVEL = 2;
+DataTypeSerDeSPtr ColumnObject::Subcolumn::generate_data_serdes(DataTypePtr
type, bool is_root) {
+ // For the root column, there is no path, so there is no need to add extra
'"'
+ if (is_root) {
+ return type->get_serde(CURRENT_SERIALIZE_NESTING_LEVEL - 1);
+ } else {
+ return type->get_serde(CURRENT_SERIALIZE_NESTING_LEVEL);
+ }
+}
+
ColumnObject::Subcolumn::Subcolumn(MutableColumnPtr&& data_, DataTypePtr type,
bool is_nullable_,
bool is_root_)
: least_common_type(type), is_nullable(is_nullable_),
is_root(is_root_) {
data.push_back(std::move(data_));
data_types.push_back(type);
- data_serdes.push_back(type->get_serde(CURRENT_SERIALIZE_NESTING_LEVEL));
+ data_serdes.push_back(generate_data_serdes(type, is_root));
}
ColumnObject::Subcolumn::Subcolumn(size_t size_, bool is_nullable_, bool
is_root_)
@@ -406,9 +415,9 @@ void ColumnObject::Subcolumn::insert(Field field) {
void ColumnObject::Subcolumn::add_new_column_part(DataTypePtr type) {
data.push_back(type->create_column());
- least_common_type = LeastCommonType {type};
+ least_common_type = LeastCommonType {type, is_root};
data_types.push_back(type);
- data_serdes.push_back(type->get_serde(CURRENT_SERIALIZE_NESTING_LEVEL));
+ data_serdes.push_back(generate_data_serdes(type, is_root));
}
void ColumnObject::Subcolumn::insert(Field field, FieldInfo info) {
@@ -512,8 +521,8 @@ static ColumnPtr recreate_column_with_default_values(const
ColumnPtr& column, Ty
ColumnObject::Subcolumn ColumnObject::Subcolumn::clone_with_default_values(
const FieldInfo& field_info) const {
Subcolumn new_subcolumn(*this);
- new_subcolumn.least_common_type =
- LeastCommonType {create_array(field_info.scalar_type_id,
field_info.num_dimensions)};
+ new_subcolumn.least_common_type = LeastCommonType {
+ create_array(field_info.scalar_type_id,
field_info.num_dimensions), is_root};
for (int i = 0; i < new_subcolumn.data.size(); ++i) {
new_subcolumn.data[i] = recreate_column_with_default_values(
@@ -633,8 +642,12 @@ MutableColumnPtr ColumnObject::apply_for_columns(Func&&
func) const {
auto& finalized_object = assert_cast<ColumnObject&>(*finalized);
return finalized_object.apply_for_columns(std::forward<Func>(func));
}
- auto res = ColumnObject::create(is_nullable, false);
+ auto new_root = func(get_root())->assume_mutable();
+ auto res = ColumnObject::create(get_root_type(), std::move(new_root));
for (const auto& subcolumn : subcolumns) {
+ if (subcolumn->data.is_root) {
+ continue;
+ }
auto new_subcolumn = func(subcolumn->data.get_finalized_column_ptr());
res->add_sub_column(subcolumn->path, new_subcolumn->assume_mutable(),
subcolumn->data.get_least_common_type());
@@ -663,10 +676,7 @@ void ColumnObject::resize(size_t n) {
}
void ColumnObject::Subcolumn::finalize(FinalizeMode mode) {
- if (is_finalized()) {
- return;
- }
- if (data.size() == 1 && num_of_defaults_in_prefix == 0) {
+ if (!is_root && data.size() == 1 && num_of_defaults_in_prefix == 0) {
data[0] = data[0]->convert_to_full_column_if_const();
return;
}
@@ -675,7 +685,7 @@ void ColumnObject::Subcolumn::finalize(FinalizeMode mode) {
// Root always JSONB type in write mode
to_type = is_nullable ?
make_nullable(std::make_shared<MostCommonType>())
: std::make_shared<MostCommonType>();
- least_common_type = LeastCommonType {to_type};
+ least_common_type = LeastCommonType {to_type, is_root};
}
auto result_column = to_type->create_column();
if (num_of_defaults_in_prefix) {
@@ -698,7 +708,8 @@ void ColumnObject::Subcolumn::finalize(FinalizeMode mode) {
}
data = {std::move(result_column)};
data_types = {std::move(to_type)};
- data_serdes = {data_types[0]->get_serde(CURRENT_SERIALIZE_NESTING_LEVEL)};
+ data_serdes = {(generate_data_serdes(data_types[0], is_root))};
+
num_of_defaults_in_prefix = 0;
}
@@ -779,11 +790,12 @@ void ColumnObject::Subcolumn::remove_nullable() {
least_common_type.remove_nullable();
}
-ColumnObject::Subcolumn::LeastCommonType::LeastCommonType(DataTypePtr type_)
+ColumnObject::Subcolumn::LeastCommonType::LeastCommonType(DataTypePtr type_,
bool is_root)
: type(std::move(type_)),
base_type(get_base_type_of_array(type)),
num_dimensions(get_number_of_dimensions(*type)) {
- least_common_type_serder =
type->get_serde(CURRENT_SERIALIZE_NESTING_LEVEL);
+ least_common_type_serder = Subcolumn::generate_data_serdes(type, is_root);
+
type_id = type->is_nullable() ? assert_cast<const
DataTypeNullable*>(type.get())
->get_nested_type()
->get_type_id()
@@ -794,34 +806,35 @@
ColumnObject::Subcolumn::LeastCommonType::LeastCommonType(DataTypePtr type_)
: base_type->get_type_id();
}
-ColumnObject::ColumnObject(bool is_nullable_, bool create_root_)
- : is_nullable(is_nullable_), num_rows(0) {
- if (create_root_) {
- subcolumns.create_root(Subcolumn(0, is_nullable, true /*root*/));
- }
+ColumnObject::ColumnObject(bool is_nullable_) : is_nullable(is_nullable_),
num_rows(0) {
+ subcolumns.create_root(Subcolumn(0, is_nullable, true /*root*/));
ENABLE_CHECK_CONSISTENCY(this);
}
-ColumnObject::ColumnObject(bool is_nullable_, DataTypePtr type,
MutableColumnPtr&& column)
- : is_nullable(is_nullable_), num_rows(0) {
- add_sub_column({}, std::move(column), type);
+ColumnObject::ColumnObject(DataTypePtr root_type, MutableColumnPtr&&
root_column)
+ : is_nullable(true), num_rows(root_column->size()) {
+ subcolumns.create_root(
+ Subcolumn(std::move(root_column), root_type, is_nullable, true
/*root*/));
serialized_sparse_column->insert_many_defaults(num_rows);
ENABLE_CHECK_CONSISTENCY(this);
}
-ColumnObject::ColumnObject(Subcolumns&& subcolumns_, bool is_nullable_)
- : is_nullable(is_nullable_),
+ColumnObject::ColumnObject(Subcolumns&& subcolumns_)
+ : is_nullable(true),
subcolumns(std::move(subcolumns_)),
num_rows(subcolumns.empty() ? 0 :
(*subcolumns.begin())->data.size()) {
+ serialized_sparse_column->insert_many_defaults(num_rows);
ENABLE_CHECK_CONSISTENCY(this);
}
ColumnObject::ColumnObject(size_t size) : is_nullable(true), num_rows(0) {
+ subcolumns.create_root(Subcolumn(0, is_nullable, true /*root*/));
insert_many_defaults(size);
ENABLE_CHECK_CONSISTENCY(this);
}
void ColumnObject::check_consistency() const {
+ CHECK(subcolumns.get_root() != nullptr);
for (const auto& leaf : subcolumns) {
if (num_rows != leaf->data.size()) {
throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR,
@@ -834,6 +847,32 @@ void ColumnObject::check_consistency() const {
"unmatched sparse column:, expeted rows: {},
but meet: {}", num_rows,
serialized_sparse_column->size());
}
+
+#ifdef NDEBUG
+ bool error = false;
+ auto [path, value] = get_sparse_data_paths_and_values();
+
+ auto& offsets = serialized_sparse_column_offsets();
+ for (size_t row = 0; row != num_rows; ++row) {
+ size_t offset = offsets[row - 1];
+ size_t end = offsets[row];
+ // Iterator over [path, binary value]
+ for (size_t i = offset; i != end; ++i) {
+ const StringRef sparse_path_string = path->get_data_at(i);
+ const std::string_view sparse_path(sparse_path_string);
+
+ const PathInData column_path(sparse_path);
+ if (auto* subcolumn = get_subcolumn(column_path); subcolumn !=
nullptr) {
+ LOG(WARNING) << "err path: " << sparse_path;
+ error = true;
+ }
+ }
+ }
+ if (error) {
+ throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR,
+ "path {} both exists in subcolumn and sparse
columns");
+ }
+#endif
}
size_t ColumnObject::size() const {
@@ -895,6 +934,7 @@ void ColumnObject::insert_from(const IColumn& src, size_t
n) {
}
void ColumnObject::try_insert(const Field& field) {
+ size_t old_size = size();
if (field.get_type() != Field::Types::VariantMap) {
if (field.is_null()) {
insert_default();
@@ -902,20 +942,12 @@ void ColumnObject::try_insert(const Field& field) {
return;
}
auto* root = get_subcolumn({});
- // Insert to an emtpy ColumnObject may result root null,
- // so create a root column of Variant is expected.
if (root == nullptr) {
- bool succ = add_sub_column({}, num_rows);
- if (!succ) {
- throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT,
- "Failed to add root sub column {}");
- }
- root = get_subcolumn({});
+ CHECK(false) << "column object has no root column";
}
root->insert(field);
} else {
const auto& object = field.get<const VariantMap&>();
- size_t old_size = size();
for (const auto& [key_str, value] : object) {
PathInData key;
if (!key_str.empty()) {
@@ -935,12 +967,12 @@ void ColumnObject::try_insert(const Field& field) {
}
subcolumn->insert(value);
}
- for (auto& entry : subcolumns) {
- if (old_size == entry->data.size()) {
- bool inserted = try_insert_default_from_nested(entry);
- if (!inserted) {
- entry->data.insert_default();
- }
+ }
+ for (auto& entry : subcolumns) {
+ if (old_size == entry->data.size()) {
+ bool inserted = try_insert_default_from_nested(entry);
+ if (!inserted) {
+ entry->data.insert_default();
}
}
}
@@ -1053,8 +1085,8 @@ void
ColumnObject::Subcolumn::serialize_to_sparse_column(ColumnString* key, std:
key->insert_data(path.data(), path.size());
// every subcolumn is always Nullable
- auto nullable_serde =
std::static_pointer_cast<DataTypeNullableSerDe>(
-
data_types[i]->get_serde(CURRENT_SERIALIZE_NESTING_LEVEL));
+ auto nullable_serde =
+
std::static_pointer_cast<DataTypeNullableSerDe>(data_serdes[i]);
auto& nullable_col =
assert_cast<const ColumnNullable&,
TypeCheckOnRelease::DISABLE>(*part);
@@ -1262,10 +1294,19 @@ void ColumnObject::add_nested_subcolumn(const
PathInData& key, const FieldInfo&
ENABLE_CHECK_CONSISTENCY(this);
}
+void ColumnObject::set_num_rows(size_t n) {
+ num_rows = n;
+}
+
bool ColumnObject::try_add_new_subcolumn(const PathInData& path) {
- if (subcolumns.size() == config::variant_max_subcolumns_count) return
false;
+ if (subcolumns.get_root() == nullptr || path.empty()) {
+ throw Exception(ErrorCode::INTERNAL_ERROR, "column object has no root
or path is empty");
+ }
+ if (subcolumns.size() < config::variant_max_subcolumns_count + 1) {
+ return add_sub_column(path, num_rows);
+ }
- return add_sub_column(path, num_rows);
+ return false;
}
void ColumnObject::insert_range_from(const IColumn& src, size_t start, size_t
length) {
@@ -1296,7 +1337,6 @@ void ColumnObject::insert_range_from(const IColumn& src,
size_t start, size_t le
auto it = src_path_and_subcoumn_for_sparse_column.begin();
auto end = src_path_and_subcoumn_for_sparse_column.end();
while (it != end) {
- VLOG_DEBUG << "pick " << it->first << " as sparse column";
sorted_src_subcolumn_for_sparse_column.emplace_back(it->first,
it->second);
++it;
}
@@ -1524,7 +1564,8 @@ bool ColumnObject::add_sub_column(const PathInData& key,
MutableColumnPtr&& subc
}
if (key.empty() &&
(!subcolumns.get_root()->is_scalar() ||
- (is_null_root() ||
is_nothing(subcolumns.get_root()->data.get_least_common_type())))) {
+ (is_null_root() ||
+
is_nothing(remove_nullable(subcolumns.get_root()->data.get_least_common_type())))))
{
bool root_it_scalar = subcolumns.get_root()->is_scalar();
// update root to scalar
subcolumns.get_mutable_root()->modify_to_scalar(
@@ -1600,7 +1641,7 @@ void ColumnObject::Subcolumn::wrapp_array_nullable() {
}
result_column = ColumnNullable::create(std::move(result_column),
std::move(new_null_map));
data_types[0] = make_nullable(data_types[0]);
- least_common_type = LeastCommonType {data_types[0]};
+ least_common_type = LeastCommonType {data_types[0], is_root};
}
}
@@ -1609,7 +1650,6 @@ Status ColumnObject::serialize_one_row_to_string(int64_t
row, std::string* outpu
VectorBufferWriter write_buffer(*tmp_col.get());
if (is_scalar_variant()) {
subcolumns.get_root()->data.serialize_text_json(row, write_buffer);
- return Status::OK();
} else {
// TODO preallocate memory
RETURN_IF_ERROR(serialize_one_row_to_json_format(row, write_buffer,
nullptr));
@@ -1894,24 +1934,41 @@ void ColumnObject::unnest(Subcolumns::NodePtr& entry,
Subcolumns& subcolumns) co
}
}
+void ColumnObject::clear_sparse_column() {
+#ifndef NDEBUG
+ auto& sparse_column_offsets = serialized_sparse_column_offsets();
+ if (sparse_column_offsets[num_rows - 1] != 0) {
+ throw Exception(ErrorCode::INTERNAL_ERROR, "sprase column has nonnull
value");
+ }
+#endif
+
+ serialized_sparse_column->clear();
+}
+
Status ColumnObject::finalize(FinalizeMode mode) {
Subcolumns new_subcolumns;
- // finalize root first
- if (!is_null_root()) {
- new_subcolumns.create_root(subcolumns.get_root()->data);
- new_subcolumns.get_mutable_root()->data.finalize(mode);
- } else if (mode == FinalizeMode::WRITE_MODE) {
- new_subcolumns.create_root(Subcolumn(num_rows, is_nullable, true));
- new_subcolumns.get_mutable_root()->data.finalize(mode);
+ if (auto root = subcolumns.get_mutable_root(); root == nullptr) {
+ CHECK(false);
+ } else {
+ root->data.finalize(mode);
+ new_subcolumns.create_root(root->data);
}
- const bool need_pick_subcolumn_to_sparse_column =
+ // 1. only write mode need to pick subcolumns to sparse column
+ // 2. root column must be exsit in subcolumns
+ bool need_pick_subcolumn_to_sparse_column =
mode == FinalizeMode::WRITE_MODE &&
- subcolumns.size() > config::variant_max_subcolumns_count;
+ subcolumns.size() > config::variant_max_subcolumns_count + 1;
+
// finalize all subcolumns
for (auto&& entry : subcolumns) {
+ if (entry->data.is_root) {
+ continue;
+ }
+
const auto& least_common_type = entry->data.get_least_common_type();
+
// unnest all nested columns, add them to new_subcolumns
if (mode == FinalizeMode::WRITE_MODE &&
least_common_type->equals(*ColumnObject::NESTED_TYPE)) {
@@ -1919,9 +1976,6 @@ Status ColumnObject::finalize(FinalizeMode mode) {
continue;
}
- if (entry->data.is_root) {
- continue;
- }
entry->data.finalize(mode);
entry->data.wrapp_array_nullable();
@@ -1938,10 +1992,14 @@ Status ColumnObject::finalize(FinalizeMode mode) {
std::unordered_map<std::string_view, size_t> none_null_value_sizes;
// 1. get the none null value sizes
for (auto&& entry : subcolumns) {
+ // root column is already in the subcolumns
if (entry->data.is_root) {
continue;
}
size_t size = entry->data.get_non_null_value_size();
+ if (size == 0) {
+ continue;
+ }
none_null_value_sizes[entry->path.get_path()] = size;
}
// 2. sort by the size
@@ -1965,12 +2023,16 @@ Status ColumnObject::finalize(FinalizeMode mode) {
for (auto&& entry : subcolumns) {
if (selected_path.find(entry->path.get_path()) !=
selected_path.end()) {
new_subcolumns.add(entry->path, entry->data);
- } else {
+ } else if (none_null_value_sizes.find(entry->path.get_path()) !=
+ none_null_value_sizes.end()) {
VLOG_DEBUG << "pick " << entry->path.get_path() << " as sparse
column";
remaing_subcolumns.emplace(entry->path.get_path(),
entry->data);
}
}
- serialized_sparse_column->clear();
+
+ ENABLE_CHECK_CONSISTENCY(this);
+ clear_sparse_column();
+
RETURN_IF_ERROR(serialize_sparse_columns(std::move(remaing_subcolumns)));
}
@@ -1996,7 +2058,7 @@ void ColumnObject::ensure_root_node_type(const
DataTypePtr& expected_root_type)
expected_root_type, &casted_column));
root.data[0] = casted_column;
root.data_types[0] = expected_root_type;
- root.least_common_type = Subcolumn::LeastCommonType
{expected_root_type};
+ root.least_common_type = Subcolumn::LeastCommonType
{expected_root_type, true};
}
}
@@ -2022,8 +2084,12 @@ ColumnPtr ColumnObject::filter(const Filter& filter,
ssize_t count) const {
ENABLE_CHECK_CONSISTENCY(res.get());
return res;
}
- auto new_column = ColumnObject::create(true, false);
+ auto new_root = get_root()->filter(filter, count)->assume_mutable();
+ auto new_column = ColumnObject::create(get_root_type(),
std::move(new_root));
for (auto& entry : subcolumns) {
+ if (entry->data.is_root) {
+ continue;
+ }
auto subcolumn = entry->data.get_finalized_column().filter(filter, -1);
new_column->add_sub_column(entry->path, subcolumn->assume_mutable(),
entry->data.get_least_common_type());
@@ -2083,6 +2149,8 @@ void ColumnObject::clear_column_data() {
void ColumnObject::clear() {
Subcolumns empty;
+ // we must keep root column exist
+ empty.create_root(Subcolumn(0, is_nullable, true));
std::swap(empty, subcolumns);
serialized_sparse_column->clear();
num_rows = 0;
diff --git a/be/src/vec/columns/column_object.h
b/be/src/vec/columns/column_object.h
index b3f2679dbe6..05e2693ed71 100644
--- a/be/src/vec/columns/column_object.h
+++ b/be/src/vec/columns/column_object.h
@@ -188,6 +188,8 @@ public:
void serialize_to_sparse_column(ColumnString* key, std::string_view
path,
ColumnString* value, size_t row);
+ static DataTypeSerDeSPtr generate_data_serdes(DataTypePtr type, bool
is_root = false);
+
friend class ColumnObject;
private:
@@ -195,7 +197,7 @@ public:
public:
LeastCommonType() = default;
- explicit LeastCommonType(DataTypePtr type_);
+ explicit LeastCommonType(DataTypePtr type_, bool is_root = false);
const DataTypePtr& get() const { return type; }
@@ -269,14 +271,15 @@ private:
public:
static constexpr auto COLUMN_NAME_DUMMY = "_dummy";
- explicit ColumnObject(bool is_nullable_, bool create_root = true);
-
- explicit ColumnObject(bool is_nullable_, DataTypePtr type,
MutableColumnPtr&& column);
+ // always create root: data type nothing
+ explicit ColumnObject(bool is_nullable_);
- // create without root, num_rows = size
+ // always create root: data type nothing
explicit ColumnObject(size_t size = 0);
- ColumnObject(Subcolumns&& subcolumns_, bool is_nullable_);
+ explicit ColumnObject(DataTypePtr root_type, MutableColumnPtr&&
root_column);
+
+ explicit ColumnObject(Subcolumns&& subcolumns_);
~ColumnObject() override = default;
@@ -309,6 +312,8 @@ public:
static const DataTypePtr& get_most_common_type();
+ void clear_sparse_column();
+
// root is null or type nothing
bool is_null_root() const;
@@ -337,7 +342,7 @@ public:
void incr_num_rows(size_t n) { num_rows += n; }
- void set_num_rows(size_t n) { num_rows = n; }
+ void set_num_rows(size_t n);
size_t rows() const { return num_rows; }
diff --git a/be/src/vec/common/schema_util.cpp
b/be/src/vec/common/schema_util.cpp
index 74298d2c838..fdae135cc81 100644
--- a/be/src/vec/common/schema_util.cpp
+++ b/be/src/vec/common/schema_util.cpp
@@ -231,8 +231,9 @@ void get_column_by_type(const vectorized::DataTypePtr&
data_type, const std::str
return;
}
// TODO handle more types like struct/date/datetime/decimal...
- LOG(FATAL) << "__builtin_unreachable";
- __builtin_unreachable();
+ throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR,
+ "unexcepted data column type: {}, column name is:
{}",
+ data_type->get_name(), name);
}
TabletColumn get_column_by_type(const vectorized::DataTypePtr& data_type,
const std::string& name,
diff --git a/be/src/vec/functions/function_variant_element.cpp
b/be/src/vec/functions/function_variant_element.cpp
index e04ff54dbe9..d549c36595f 100644
--- a/be/src/vec/functions/function_variant_element.cpp
+++ b/be/src/vec/functions/function_variant_element.cpp
@@ -151,7 +151,7 @@ private:
result_column->insert_default();
}
}
- *result = ColumnObject::create(true, type,
std::move(result_column));
+ *result = ColumnObject::create(type, std::move(result_column));
(*result)->assume_mutable()->finalize();
return Status::OK();
} else {
@@ -162,8 +162,7 @@ private:
const auto* node = subcolumns.find_exact(path);
MutableColumnPtr result_col;
if (node != nullptr) {
- // Create without root, since root will be added
- result_col = ColumnObject::create(true, false /*should not
create root*/);
+ result_col = ColumnObject::create(true);
std::vector<decltype(node)> nodes;
PathsInData paths;
ColumnObject::Subcolumns::get_leaves_of_node(node, nodes,
paths);
@@ -186,10 +185,10 @@ private:
nodes[0]->data.get_finalized_column_ptr()->assume_mutable(),
nodes[0]->data.get_least_common_type(), true,
true});
}
- auto container =
ColumnObject::create(std::move(new_subcolumns), true);
+ auto container =
ColumnObject::create(std::move(new_subcolumns));
result_col->insert_range_from(*container, 0,
container->size());
} else {
- // Create with root, otherwise the root type maybe type Nothing
+ // Create with root, otherwise the root type maybe type
Nothing ?
result_col = ColumnObject::create(true);
result_col->insert_many_defaults(src.size());
}
diff --git a/be/test/vec/columns/column_object_test.cpp
b/be/test/vec/columns/column_object_test.cpp
new file mode 100644
index 00000000000..e59219827db
--- /dev/null
+++ b/be/test/vec/columns/column_object_test.cpp
@@ -0,0 +1,795 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/columns/column_object.h"
+
+#include <gmock/gmock-more-matchers.h>
+#include <gtest/gtest.h>
+#include <rapidjson/prettywriter.h>
+#include <stdio.h>
+
+#include "runtime/jsonb_value.h"
+#include "vec/common/string_ref.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_factory.hpp"
+
+using namespace doris::vectorized;
+
+using namespace doris;
+// #define ADD_SUB_COLUMN(key) \
+// varaint->add_sub_column(PathInData(std::string_view(key)), 0);
+
+doris::vectorized::Field construct_variant_map(
+ const std::vector<std::pair<std::string, doris::vectorized::Field>>&
key_and_values) {
+ doris::vectorized::Field res = VariantMap();
+ auto& object = res.get<VariantMap&>();
+ for (const auto& [k, v] : key_and_values) {
+ object.try_emplace(k, v);
+ }
+ return res;
+}
+
+auto construct_basic_varint_column() {
+ // 1. create an empty variant column
+ auto variant = ColumnObject::create();
+
+ std::vector<std::pair<std::string, doris::vectorized::Field>> data;
+
+ // 2. subcolumn path
+ data.emplace_back("v.a", 20);
+ data.emplace_back("v.b", "20");
+ data.emplace_back("v.c", 20);
+ data.emplace_back("v.f", 20);
+ data.emplace_back("v.e", "50");
+ for (int i = 0; i < 5; ++i) {
+ auto field = construct_variant_map(data);
+ variant->try_insert(field);
+ }
+
+ // 3. sparse column path
+ data.emplace_back("v.d.d", "50");
+ data.emplace_back("v.c.d", 30);
+ data.emplace_back("v.b.d", 30);
+ for (int i = 0; i < 5; ++i) {
+ auto field = construct_variant_map(data);
+ variant->try_insert(field);
+ }
+ return variant;
+}
+
+auto construct_dst_varint_column() {
+ // 1. create an empty variant column
+ vectorized::ColumnObject::Subcolumns dynamic_subcolumns;
+ dynamic_subcolumns.create_root(vectorized::ColumnObject::Subcolumn(0,
true, true /*root*/));
+ dynamic_subcolumns.add(vectorized::PathInData("v.f"),
+ vectorized::ColumnObject::Subcolumn {0, true});
+ dynamic_subcolumns.add(vectorized::PathInData("v.e"),
+ vectorized::ColumnObject::Subcolumn {0, true});
+ dynamic_subcolumns.add(vectorized::PathInData("v.b"),
+ vectorized::ColumnObject::Subcolumn {0, true});
+ dynamic_subcolumns.add(vectorized::PathInData("v.b.d"),
+ vectorized::ColumnObject::Subcolumn {0, true});
+ dynamic_subcolumns.add(vectorized::PathInData("v.c.d"),
+ vectorized::ColumnObject::Subcolumn {0, true});
+ return ColumnObject::create(std::move(dynamic_subcolumns));
+}
+
+TEST(ColumnVariantTest, basic_finalize) {
+ auto variant = construct_basic_varint_column();
+ // 4. finalize
+
EXPECT_TRUE(variant->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
+ EXPECT_EQ(variant->size(), 10);
+
+ // check finalized subcolumn
+ // 5 subcolumn + 1 root
+ EXPECT_EQ(variant->subcolumns.size(), 6);
+ for (const auto& column : variant->subcolumns) {
+ if (column->data.is_root) {
+ continue;
+ }
+ EXPECT_EQ(column->data.data.size(), 1);
+ }
+
+ // check sparse column
+ const auto& offsets = variant->serialized_sparse_column_offsets();
+ for (int row = 0; row < 5; ++row) {
+ EXPECT_EQ(offsets[row], 0);
+ }
+ for (int row = 5; row < 10; ++row) {
+ EXPECT_EQ(offsets[row] - offsets[row - 1], 3);
+ }
+}
+
+TEST(ColumnVariantTest, basic_deserialize) {
+ auto variant = construct_basic_varint_column();
+
+ // 4. finalize
+
EXPECT_TRUE(variant->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
+ EXPECT_EQ(variant->size(), 10);
+
+ const auto& [path, value] = variant->get_sparse_data_paths_and_values();
+ const auto& offsets = variant->serialized_sparse_column_offsets();
+ for (size_t row = 5; row < 10; ++row) {
+ size_t start = offsets[row - 1];
+ size_t end = offsets[row];
+
+ auto data = path->get_data_at(start);
+ EXPECT_EQ(data, StringRef("v.b.d", 5));
+ auto pair = variant->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(pair.first.get<Int64>(), 30);
+
+ auto data2 = path->get_data_at(start);
+ auto pair2 = variant->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(data2, StringRef("v.c.d", 5));
+ EXPECT_EQ(pair2.first.get<Int64>(), 30);
+
+ auto data3 = path->get_data_at(start);
+ auto pair3 = variant->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(data3, StringRef("v.d.d", 5));
+ EXPECT_EQ(pair3.first.get<String>(), "50");
+ EXPECT_EQ(start, end);
+ }
+}
+
+TEST(ColumnVariantTest, basic_inset_range_from) {
+ auto src = construct_basic_varint_column();
+ EXPECT_TRUE(src->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
+ EXPECT_EQ(src->size(), 10);
+
+ // dst is an empty column, has 5 subcolumn + 1 root
+ auto dst = construct_dst_varint_column();
+
+ // subcolumn->subcolumn v.b v.f v.e
+ // subcolumn->sparse_column v.a v.c
+ // sparse_column->subcolumn v.b.d v.c.d
+ // sparse_column->sparse_column v.d.d
+ dst->insert_range_from(*src, 0, 10);
+ EXPECT_EQ(dst->size(), 10);
+
+ // 5 subcolumn
+ EXPECT_EQ(dst->subcolumns.size(), 6);
+ ColumnObject::Subcolumns dst_subcolumns = dst->subcolumns;
+ std::sort(
+ dst_subcolumns.begin(), dst_subcolumns.end(),
+ [](const auto& lhsItem, const auto& rhsItem) { return
lhsItem->path < rhsItem->path; });
+
+ for (const auto& column : dst_subcolumns) {
+ if (column->data.is_root) {
+ continue;
+ }
+ EXPECT_EQ(column->data.data.size(), 1);
+ EXPECT_EQ(column->data.data[0]->size(), 10);
+ if (column->path.get_path().size() == 3) {
+ EXPECT_EQ(column->data.get_non_null_value_size(), 10);
+ } else {
+ EXPECT_EQ(column->path.get_path().size(), 5);
+ EXPECT_EQ(column->data.get_non_null_value_size(), 5);
+ for (size_t row = 0; row != 5; ++row) {
+ EXPECT_TRUE(column->data.data[0]->is_null_at(row));
+ }
+ for (size_t row = 5; row != 10; ++row) {
+ EXPECT_EQ((*column->data.data[0])[row].get<Int64>(), 30);
+ }
+ }
+ }
+
+ // check sparse column
+ const auto& [path, value] = dst->get_sparse_data_paths_and_values();
+ const auto& offsets = dst->serialized_sparse_column_offsets();
+
+ // v.a v.c
+ for (int row = 0; row < 5; ++row) {
+ size_t start = offsets[row - 1];
+ size_t end = offsets[row];
+
+ auto data = path->get_data_at(start);
+ EXPECT_EQ(data, StringRef("v.a", 3));
+ auto pair = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(pair.first.get<Int64>(), 20);
+
+ auto data2 = path->get_data_at(start);
+ EXPECT_EQ(data2, StringRef("v.c", 3));
+ auto pair2 = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(pair2.first.get<Int64>(), 20);
+
+ EXPECT_EQ(start, end);
+ }
+
+ // v.a v.c v.d.d
+ for (int row = 5; row < 10; ++row) {
+ size_t start = offsets[row - 1];
+ size_t end = offsets[row];
+
+ auto data = path->get_data_at(start);
+ EXPECT_EQ(data, StringRef("v.a", 3));
+ auto pair = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(pair.first.get<Int64>(), 20);
+
+ auto data2 = path->get_data_at(start);
+ EXPECT_EQ(data2, StringRef("v.c", 3));
+ auto pair2 = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(pair2.first.get<Int64>(), 20);
+
+ auto data3 = path->get_data_at(start);
+ EXPECT_EQ(data3, StringRef("v.d.d", 5));
+ auto pair3 = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(pair3.first.get<String>(), "50");
+
+ EXPECT_EQ(start, end);
+ }
+}
+
+doris::vectorized::Field get_field(std::string_view type) {
+ static std::unordered_map<std::string_view, doris::vectorized::Field>
field_map;
+ if (field_map.empty()) {
+ doris::vectorized::Field int_field = 20;
+ doris::vectorized::Field str_field(String("str", 3));
+ doris::vectorized::Field arr_int_field = Array();
+ auto& array1 = arr_int_field.get<Array>();
+ array1.emplace_back(int_field);
+ array1.emplace_back(int_field);
+ doris::vectorized::Field arr_str_field = Array();
+ auto& array2 = arr_str_field.get<Array>();
+ array2.emplace_back(str_field);
+ array2.emplace_back(str_field);
+ field_map["int"] = int_field;
+ field_map["string"] = str_field;
+ field_map["array_int"] = arr_int_field;
+ field_map["array_str"] = arr_str_field;
+ }
+ return field_map[type];
+}
+
+auto convert_to_jsonb_field(auto serde, auto& column) {
+ vectorized::DataTypeSerDe::FormatOptions options;
+ options.escape_char = '\\';
+ auto tmp_col = ColumnString::create();
+ VectorBufferWriter write_buffer(*tmp_col.get());
+ EXPECT_TRUE(serde->serialize_column_to_json(column, 0, 1, write_buffer,
options).ok());
+
+ write_buffer.commit();
+ auto str_ref = tmp_col->get_data_at(0);
+ Slice data((char*)(str_ref.data), str_ref.size);
+
+ auto jsonb_type =
doris::vectorized::DataTypeFactory::instance().create_data_type(
+ TypeIndex::JSONB, false);
+ auto jsonb_serde = jsonb_type->get_serde();
+ auto jsonb_column = jsonb_type->create_column();
+
+ DataTypeSerDe::FormatOptions format_options;
+ format_options.converted_from_string = true;
+ EXPECT_TRUE(
+ jsonb_serde->deserialize_one_cell_from_json(*jsonb_column, data,
format_options).ok());
+ auto res = jsonb_column->get_data_at(0);
+ return JsonbField(res.data, res.size);
+}
+
+auto convert_string_to_jsonb_field(auto& column) {
+ auto str_ref = column.get_data_at(0);
+ Slice data((char*)(str_ref.data), str_ref.size);
+
+ auto jsonb_type =
doris::vectorized::DataTypeFactory::instance().create_data_type(
+ TypeIndex::JSONB, false);
+ auto jsonb_serde = jsonb_type->get_serde();
+ auto jsonb_column = jsonb_type->create_column();
+ DataTypeSerDe::FormatOptions format_options;
+ format_options.converted_from_string = true;
+ format_options.escape_char = '\\';
+
+ EXPECT_TRUE(
+ jsonb_serde->deserialize_one_cell_from_json(*jsonb_column, data,
format_options).ok());
+ auto res = jsonb_column->get_data_at(0);
+ return JsonbField(res.data, res.size);
+}
+
+doris::vectorized::Field get_jsonb_field(std::string_view type) {
+ static std::unordered_map<std::string_view, doris::vectorized::Field>
field_map;
+ if (field_map.empty()) {
+ DataTypePtr data_type_int =
doris::vectorized::DataTypeFactory::instance().create_data_type(
+ TypeIndex::Int8, false);
+ DataTypePtr data_type_array_int =
+
std::make_shared<doris::vectorized::DataTypeArray>(data_type_int);
+ auto array_column_int = data_type_array_int->create_column();
+ array_column_int->insert(get_field("array_int"));
+ auto array_serde_int = data_type_array_int->get_serde();
+ field_map["array_int"] = convert_to_jsonb_field(array_serde_int,
*array_column_int);
+
+ DataTypePtr data_type_str =
doris::vectorized::DataTypeFactory::instance().create_data_type(
+ TypeIndex::String, false);
+ DataTypePtr data_type_array_str =
+
std::make_shared<doris::vectorized::DataTypeArray>(data_type_str);
+ auto array_column_str = data_type_array_str->create_column();
+ array_column_str->insert(get_field("array_str"));
+ auto array_serde_str = data_type_array_str->get_serde();
+ field_map["array_str"] = convert_to_jsonb_field(array_serde_str,
*array_column_str);
+
+ auto column_int = data_type_int->create_column();
+ column_int->insert(get_field("int"));
+ auto serde_int = data_type_int->get_serde();
+ field_map["int"] = convert_to_jsonb_field(serde_int, *column_int);
+
+ // auto column_str = data_type_str->create_column();
+ // column_str->insert(get_field("string"));
+ // field_map["string"] = convert_string_to_jsonb_field(*column_str);
+ }
+ return field_map[type];
+}
+
+auto construct_advanced_varint_column() {
+ // 1. create an empty variant column
+ auto variant = ColumnObject::create();
+
+ std::vector<std::pair<std::string, doris::vectorized::Field>> data;
+
+ // 2. subcolumn path
+ data.emplace_back("v.a", get_field("int"));
+ data.emplace_back("v.b", get_field("string"));
+ data.emplace_back("v.c", get_field("array_int"));
+ data.emplace_back("v.f", get_field("array_str"));
+ data.emplace_back("v.e", get_field("string"));
+
+ for (int i = 0; i < 5; ++i) {
+ auto field = construct_variant_map(data);
+ variant->try_insert(field);
+ }
+
+ // 3. sparse column path
+ data.emplace_back("v.d.d", get_field("array_int"));
+ data.emplace_back("v.c.d", get_field("string"));
+ data.emplace_back("v.b.d", get_field("array_int"));
+ for (int i = 0; i < 5; ++i) {
+ auto field = construct_variant_map(data);
+ variant->try_insert(field);
+ }
+
+ data.clear();
+ data.emplace_back("v.a", get_field("int"));
+ data.emplace_back("v.b", get_field("int"));
+ data.emplace_back("v.c", get_field("array_int"));
+ data.emplace_back("v.f", get_field("array_str"));
+ data.emplace_back("v.e", get_field("string"));
+ data.emplace_back("v.d.d", get_field("array_str"));
+ data.emplace_back("v.c.d", get_field("int"));
+ data.emplace_back("v.b.d", get_field("array_str"));
+ for (int i = 0; i < 5; ++i) {
+ auto field = construct_variant_map(data);
+ variant->try_insert(field);
+ }
+ return variant;
+}
+
+// std::string convert_jsonb_field_to_string(doris::vectorized::Field jsonb) {
+// const auto& val = jsonb.get<JsonbField>();
+// const JsonbValue* json_val =
JsonbDocument::createValue(val.get_value(), val.get_size());
+
+// rapidjson::Document doc;
+// doc.SetObject();
+// rapidjson::Document::AllocatorType& allocator = doc.GetAllocator();
+// rapidjson::Value json_value;
+// convert_jsonb_to_rapidjson(*json_val, json_value, allocator);
+// doc.AddMember("value", json_value, allocator);
+// rapidjson::StringBuffer buffer;
+// rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer);
+// doc.Accept(writer);
+// return std::string(buffer.GetString());
+// }
+
+std::string convert_field_to_string(doris::vectorized::Field array) {
+ rapidjson::Document doc;
+ doc.SetObject();
+ rapidjson::Document::AllocatorType& allocator = doc.GetAllocator();
+ rapidjson::Value json_value;
+ DataTypeSerDe::convert_field_to_rapidjson(array, json_value, allocator);
+ doc.AddMember("value", json_value, allocator);
+ rapidjson::StringBuffer buffer;
+ rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer);
+ doc.Accept(writer);
+ return std::string(buffer.GetString());
+}
+
+TEST(ColumnVariantTest, advanced_finalize) {
+ auto variant = construct_advanced_varint_column();
+
+ // 4. finalize
+
EXPECT_TRUE(variant->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
+ EXPECT_EQ(variant->size(), 15);
+
+ // check finalized subcolumn
+ // 5 subcolumn + 1 root
+ EXPECT_EQ(variant->subcolumns.size(), 6);
+ for (const auto& column : variant->subcolumns) {
+ if (column->data.is_root) {
+ continue;
+ }
+ EXPECT_EQ(column->data.data.size(), 1);
+ }
+
+ // check sparse column
+ const auto& offsets = variant->serialized_sparse_column_offsets();
+ for (int row = 0; row < 5; ++row) {
+ EXPECT_EQ(offsets[row] - offsets[row - 1], 0);
+ }
+ for (int row = 5; row < 15; ++row) {
+ EXPECT_EQ(offsets[row] - offsets[row - 1], 3);
+ }
+}
+
+TEST(ColumnVariantTest, advanced_deserialize) {
+ auto variant = construct_advanced_varint_column();
+
+ // 4. finalize
+
EXPECT_TRUE(variant->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
+ EXPECT_EQ(variant->size(), 15);
+
+ const auto& [path, value] = variant->get_sparse_data_paths_and_values();
+ const auto& offsets = variant->serialized_sparse_column_offsets();
+ for (size_t row = 5; row < 10; ++row) {
+ size_t start = offsets[row - 1];
+ size_t end = offsets[row];
+
+ auto data = path->get_data_at(start);
+ auto pair = variant->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(data, StringRef("v.b.d", 5));
+ EXPECT_EQ(convert_field_to_string(pair.first),
+ convert_field_to_string(get_jsonb_field("array_int")));
+
+ auto data2 = path->get_data_at(start);
+ auto pair2 = variant->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(data2, StringRef("v.c.d", 5));
+ EXPECT_EQ(convert_field_to_string(pair2.first),
+ convert_field_to_string(get_field("string")));
+
+ auto data3 = path->get_data_at(start);
+ auto pair3 = variant->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(data3, StringRef("v.d.d", 5));
+ EXPECT_EQ(convert_field_to_string(pair3.first),
+ convert_field_to_string(get_jsonb_field("array_int")));
+ EXPECT_EQ(start, end);
+ }
+
+ for (size_t row = 10; row < 15; ++row) {
+ size_t start = offsets[row - 1];
+ size_t end = offsets[row];
+
+ auto data = path->get_data_at(start);
+ auto pair = variant->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(data, StringRef("v.b.d", 5));
+ EXPECT_EQ(convert_field_to_string(pair.first),
+ convert_field_to_string(get_jsonb_field("array_str")));
+
+ auto data2 = path->get_data_at(start);
+ auto pair2 = variant->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(data2, StringRef("v.c.d", 5));
+ EXPECT_EQ(convert_field_to_string(pair2.first),
+ convert_field_to_string(get_jsonb_field("int")));
+
+ auto data3 = path->get_data_at(start);
+ auto pair3 = variant->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(data3, StringRef("v.d.d", 5));
+ EXPECT_EQ(convert_field_to_string(pair3.first),
+ convert_field_to_string(get_jsonb_field("array_str")));
+ EXPECT_EQ(start, end);
+ }
+}
+
+TEST(ColumnVariantTest, advanced_insert_range_from) {
+ auto src = construct_advanced_varint_column();
+ EXPECT_TRUE(src->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
+ EXPECT_EQ(src->size(), 15);
+
+ auto dst = construct_dst_varint_column();
+
+ // subcolumn->subcolumn v.b v.f v.e
+ // subcolumn->sparse_column v.a v.c
+ // sparse_column->subcolumn v.b.d v.c.d
+ // sparse_column->sparse_column v.d.d
+ dst->insert_range_from(*src, 0, src->size());
+ EXPECT_EQ(dst->size(), 15);
+
+ EXPECT_EQ(dst->subcolumns.size(), 6);
+ ColumnObject::Subcolumns dst_subcolumns = dst->subcolumns;
+
+ std::sort(
+ dst_subcolumns.begin(), dst_subcolumns.end(),
+ [](const auto& lhsItem, const auto& rhsItem) { return
lhsItem->path < rhsItem->path; });
+
+ // subcolumns
+ for (const auto& column : dst_subcolumns) {
+ if (column->data.is_root) {
+ continue;
+ }
+ EXPECT_EQ(column->data.data.size(), 1);
+ EXPECT_EQ(column->data.data[0]->size(), 15);
+
+ if (column->path.get_path().size() == 3) {
+ EXPECT_EQ(column->data.get_non_null_value_size(), 15);
+ if (column->path.get_path() == "v.b") {
+ EXPECT_EQ(assert_cast<const
DataTypeNullable*>(column->data.data_types[0].get())
+ ->get_nested_type()
+ ->get_type_id(),
+ TypeIndex::JSONB);
+ }
+ } else if (column->path.get_path().size() == 5) {
+ EXPECT_EQ(column->data.get_non_null_value_size(), 10);
+ EXPECT_EQ(assert_cast<const
DataTypeNullable*>(column->data.data_types[0].get())
+ ->get_nested_type()
+ ->get_type_id(),
+ TypeIndex::JSONB);
+ for (size_t row = 0; row < 5; ++row) {
+ EXPECT_TRUE(column->data.data[0]->is_null_at(row));
+ }
+ }
+ }
+
+ // sparse columns
+ const auto& [path, value] = dst->get_sparse_data_paths_and_values();
+ const auto& offsets = dst->serialized_sparse_column_offsets();
+
+ // v.a v.c
+ for (int row = 0; row < 5; ++row) {
+ size_t start = offsets[row - 1];
+ size_t end = offsets[row];
+
+ auto data = path->get_data_at(start);
+ EXPECT_EQ(data, StringRef("v.a", 3));
+ auto pair = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(pair.first.get<Int64>(), 20);
+
+ auto data2 = path->get_data_at(start);
+ EXPECT_EQ(data2, StringRef("v.c", 3));
+ auto pair2 = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(convert_field_to_string(pair2.first),
+ convert_field_to_string(get_field("array_int")));
+
+ EXPECT_EQ(start, end);
+ }
+
+ for (int row = 5; row < 10; ++row) {
+ size_t start = offsets[row - 1];
+ size_t end = offsets[row];
+
+ auto data = path->get_data_at(start);
+ auto pair = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(data, StringRef("v.a", 3));
+ EXPECT_EQ(pair.first.get<Int64>(), 20);
+
+ auto data2 = path->get_data_at(start);
+ auto pair2 = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(data2, StringRef("v.c", 3));
+ EXPECT_EQ(convert_field_to_string(pair2.first),
+ convert_field_to_string(get_field("array_int")));
+
+ auto data3 = path->get_data_at(start);
+ auto pair3 = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(data3, StringRef("v.d.d", 5));
+ EXPECT_EQ(convert_field_to_string(pair3.first),
+ convert_field_to_string(get_jsonb_field("array_int")));
+
+ EXPECT_EQ(start, end);
+ }
+
+ for (int row = 10; row < 15; ++row) {
+ size_t start = offsets[row - 1];
+ size_t end = offsets[row];
+
+ auto data = path->get_data_at(start);
+ auto pair = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(data, StringRef("v.a", 3));
+ EXPECT_EQ(pair.first.get<Int64>(), 20);
+
+ auto data2 = path->get_data_at(start);
+ auto pair2 = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(data2, StringRef("v.c", 3));
+ EXPECT_EQ(convert_field_to_string(pair2.first),
+ convert_field_to_string(get_field("array_int")));
+
+ auto data3 = path->get_data_at(start);
+ auto pair3 = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(data3, StringRef("v.d.d", 5));
+ EXPECT_EQ(convert_field_to_string(pair3.first),
+ convert_field_to_string(get_jsonb_field("array_str")));
+
+ EXPECT_EQ(start, end);
+ }
+}
+
+auto construct_varint_column_only_subcolumns() {
+ // 1. create an empty variant column
+ auto variant = ColumnObject::create();
+
+ std::vector<std::pair<std::string, doris::vectorized::Field>> data;
+
+ // 2. subcolumn path
+ data.emplace_back("v.a", 20);
+ data.emplace_back("v.b", "20");
+ data.emplace_back("v.c", 20);
+ data.emplace_back("v.f", 20);
+ data.emplace_back("v.e", "50");
+ for (int i = 0; i < 5; ++i) {
+ auto field = construct_variant_map(data);
+ variant->try_insert(field);
+ }
+
+ // 3. root
+ variant->try_insert(doris::vectorized::Field(20));
+ return variant;
+}
+
+auto construct_varint_column_more_subcolumns() {
+ // 1. create an empty variant column
+ auto variant = ColumnObject::create();
+
+ std::vector<std::pair<std::string, doris::vectorized::Field>> data;
+
+ // 2. subcolumn path
+ data.emplace_back("v.a", 20);
+ data.emplace_back("v.b", "20");
+ data.emplace_back("v.c", 20);
+ data.emplace_back("v.f", 20);
+ data.emplace_back("v.e", "50");
+ data.emplace_back("v.x", "str");
+ data.emplace_back("v.y", 20);
+ for (int i = 0; i < 5; ++i) {
+ auto field = construct_variant_map(data);
+ variant->try_insert(field);
+ }
+
+ return variant;
+}
+
+TEST(ColumnVariantTest, empty_inset_range_from) {
+ auto src = construct_varint_column_only_subcolumns();
+ EXPECT_TRUE(src->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
+ EXPECT_EQ(src->size(), 6);
+
+ // dst is an empty column
+ auto dst = ColumnObject::create();
+
+ // subcolumn->subcolumn v.a v.b v.c v.f v.e
+ dst->insert_range_from(*src, 0, 6);
+ EXPECT_EQ(dst->size(), 6);
+
+ // 5 subcolumn
+ EXPECT_EQ(dst->subcolumns.size(), 6);
+
+ for (const auto& column : dst->subcolumns) {
+ if (column->data.is_root) {
+ EXPECT_EQ(column->data.data.size(), 1);
+ EXPECT_EQ(column->data.data[0]->size(), 6);
+ EXPECT_EQ(column->data.get_non_null_value_size(), 1);
+ continue;
+ }
+ EXPECT_EQ(column->data.data.size(), 1);
+ EXPECT_EQ(column->data.data[0]->size(), 6);
+ EXPECT_EQ(column->data.get_non_null_value_size(), 5);
+ }
+
+ // empty sparse column
+ const auto& [path, value] = dst->get_sparse_data_paths_and_values();
+ const auto& offsets = dst->serialized_sparse_column_offsets();
+ EXPECT_EQ(offsets[4], offsets[-1]);
+ EXPECT_EQ(path->size(), value->size());
+
+ auto src_contains_seven_subcolumns =
construct_varint_column_more_subcolumns();
+
+ EXPECT_TRUE(
+
src_contains_seven_subcolumns->finalize(ColumnObject::FinalizeMode::WRITE_MODE).ok());
+ EXPECT_EQ(src_contains_seven_subcolumns->size(), 5);
+
+ // subcolumn->subcolumn v.a v.b v.c v.f v.e
+ // add sprase columns v.x v.y
+ dst->insert_range_from(*src_contains_seven_subcolumns, 0, 5);
+ EXPECT_EQ(dst->size(), 11);
+
+ // 5 subcolumn
+ EXPECT_EQ(dst->subcolumns.size(), 6);
+
+ for (int row = 0; row < 6; ++row) {
+ size_t start = offsets[row - 1];
+ size_t end = offsets[row];
+
+ EXPECT_EQ(start, end);
+ }
+
+ // v.x v.y
+ for (int row = 6; row < 11; ++row) {
+ size_t start = offsets[row - 1];
+ size_t end = offsets[row];
+
+ auto data = path->get_data_at(start);
+ EXPECT_EQ(data, StringRef("v.x", 3));
+ auto pair = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(convert_field_to_string(pair.first),
+ convert_field_to_string(get_field("string")));
+
+ auto data2 = path->get_data_at(start);
+ EXPECT_EQ(data2, StringRef("v.y", 3));
+ auto pair2 = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(pair2.first.get<Int64>(), 20);
+
+ EXPECT_EQ(start, end);
+ }
+
+ auto src_contains_subcoumns_and_sparse_columns =
construct_basic_varint_column();
+ EXPECT_TRUE(src_contains_subcoumns_and_sparse_columns
+ ->finalize(ColumnObject::FinalizeMode::WRITE_MODE)
+ .ok());
+ EXPECT_EQ(src_contains_subcoumns_and_sparse_columns->size(), 10);
+
+ // subcolumn->subcolumn v.a v.b v.c v.f v.e
+ // add sprase columns v.x v.y v.b.d v.c.d v.d.d
+ dst->insert_range_from(*src_contains_subcoumns_and_sparse_columns, 0, 10);
+ EXPECT_EQ(dst->size(), 21);
+
+ // 5 subcolumn
+ EXPECT_EQ(dst->subcolumns.size(), 6);
+
+ for (int row = 0; row < 6; ++row) {
+ size_t start = offsets[row - 1];
+ size_t end = offsets[row];
+
+ EXPECT_EQ(start, end);
+ }
+
+ // v.x v.y
+ for (int row = 6; row < 11; ++row) {
+ size_t start = offsets[row - 1];
+ size_t end = offsets[row];
+
+ auto data = path->get_data_at(start);
+ EXPECT_EQ(data, StringRef("v.x", 3));
+ auto pair = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(convert_field_to_string(pair.first),
+ convert_field_to_string(get_field("string")));
+
+ auto data2 = path->get_data_at(start);
+ EXPECT_EQ(data2, StringRef("v.y", 3));
+ auto pair2 = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(pair2.first.get<Int64>(), 20);
+
+ EXPECT_EQ(start, end);
+ }
+
+ for (int row = 11; row < 16; ++row) {
+ size_t start = offsets[row - 1];
+ size_t end = offsets[row];
+
+ EXPECT_EQ(start, end);
+ }
+
+ //v.b.d v.c.d v.d.d
+ for (int row = 16; row < 21; ++row) {
+ size_t start = offsets[row - 1];
+ size_t end = offsets[row];
+
+ auto data = path->get_data_at(start);
+ EXPECT_EQ(data, StringRef("v.b.d", 5));
+ auto pair = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(pair.first.get<Int64>(), 30);
+
+ auto data2 = path->get_data_at(start);
+ auto pair2 = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(data2, StringRef("v.c.d", 5));
+ EXPECT_EQ(pair2.first.get<Int64>(), 30);
+
+ auto data3 = path->get_data_at(start);
+ auto pair3 = dst->deserialize_from_sparse_column(value, start++);
+ EXPECT_EQ(data3, StringRef("v.d.d", 5));
+ EXPECT_EQ(pair3.first.get<String>(), "50");
+ EXPECT_EQ(start, end);
+ }
+}
diff --git a/regression-test/data/variant_p0/variant_with_rowstore.out
b/regression-test/data/variant_p0/variant_with_rowstore.out
index fbad7c4e44d..ca5e7393e9d 100644
--- a/regression-test/data/variant_p0/variant_with_rowstore.out
+++ b/regression-test/data/variant_p0/variant_with_rowstore.out
@@ -33,5 +33,5 @@
-1 {"a":1123} {"a":1123}
-- !point_select --
-1 1|[""]
+1 1|[""]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]