This is an automated email from the ASF dual-hosted git repository.
lihaopeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new dc9a860811e ·[Refactor](vec) remove the unless api in column nullable
(#46180)
dc9a860811e is described below
commit dc9a860811e2fcaf582764f80161b77b67632f52
Author: HappenLee <[email protected]>
AuthorDate: Wed Jan 1 23:31:37 2025 +0800
·[Refactor](vec) remove the unless api in column nullable (#46180)
```
Remove the unless api in ColumnNullable：`insert_null_elements` replace by
`insert_many_defaults`
```
---
be/src/olap/base_tablet.cpp | 4 ++--
be/src/olap/partial_update_info.cpp | 6 +++---
be/src/olap/rowset/segment_v2/column_reader.cpp | 10 ++++------
be/src/olap/rowset/segment_v2/column_reader.h | 4 ----
be/src/vec/columns/column_nullable.h | 8 +-------
be/src/vec/data_types/data_type_struct.cpp | 2 +-
be/src/vec/exec/format/orc/vorc_reader.cpp | 2 +-
be/src/vec/exec/format/parquet/vparquet_column_reader.cpp | 2 +-
be/src/vec/olap/vgeneric_iterators.cpp | 5 ++---
be/test/vec/columns/column_nullable_test.cpp | 6 +++---
be/test/vec/core/block_test.cpp | 4 ++--
be/test/vec/data_types/serde/data_type_serde_pb_test.cpp | 4 ++--
be/test/vec/data_types/serde/data_type_serde_test.cpp | 4 ++--
be/test/vec/jsonb/serialize_test.cpp | 4 ++--
14 files changed, 26 insertions(+), 39 deletions(-)
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index 40c5443477d..ef87c38d540 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -1043,7 +1043,7 @@ Status BaseTablet::generate_new_block_for_partial_update(
} else if (rs_column.is_nullable()) {
assert_cast<vectorized::ColumnNullable*,
TypeCheckOnRelease::DISABLE>(
mutable_column.get())
- ->insert_null_elements(1);
+ ->insert_default();
} else {
mutable_column->insert_default();
}
@@ -1157,7 +1157,7 @@ Status
BaseTablet::generate_new_block_for_flexible_partial_update(
} else if (tablet_column.is_nullable()) {
assert_cast<vectorized::ColumnNullable*,
TypeCheckOnRelease::DISABLE>(
new_col.get())
- ->insert_null_elements(1);
+ ->insert_default();
} else {
new_col->insert_default();
}
diff --git a/be/src/olap/partial_update_info.cpp
b/be/src/olap/partial_update_info.cpp
index 3e8c5d9750c..5260037578f 100644
--- a/be/src/olap/partial_update_info.cpp
+++ b/be/src/olap/partial_update_info.cpp
@@ -366,7 +366,7 @@ Status FixedReadPlan::fill_missing_columns(
} else if (tablet_column.is_nullable()) {
auto* nullable_column =
assert_cast<vectorized::ColumnNullable*,
TypeCheckOnRelease::DISABLE>(missing_col.get());
- nullable_column->insert_null_elements(1);
+ nullable_column->insert_default();
} else if (tablet_schema.auto_increment_column() ==
tablet_column.name()) {
const auto& column =
*DORIS_TRY(rowset_ctx->tablet_schema->column(tablet_column.name()));
@@ -551,7 +551,7 @@ Status
FlexibleReadPlan::fill_non_primary_key_columns_for_column_store(
} else if (tablet_column.is_nullable()) {
assert_cast<vectorized::ColumnNullable*,
TypeCheckOnRelease::DISABLE>(
new_col.get())
- ->insert_null_elements(1);
+ ->insert_default();
} else {
new_col->insert_default();
}
@@ -625,7 +625,7 @@ Status
FlexibleReadPlan::fill_non_primary_key_columns_for_row_store(
} else if (tablet_column.is_nullable()) {
assert_cast<vectorized::ColumnNullable*,
TypeCheckOnRelease::DISABLE>(
new_col.get())
- ->insert_null_elements(1);
+ ->insert_default();
} else {
new_col->insert_default();
}
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp
b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 1abb60e5850..9948e7fd8cf 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -409,7 +409,7 @@ Status ColumnReader::next_batch_of_zone_map(size_t* n,
vectorized::MutableColumn
auto size = *n - 1;
if (min_value->is_null()) {
-
assert_cast<vectorized::ColumnNullable&>(*dst).insert_null_elements(size);
+
assert_cast<vectorized::ColumnNullable&>(*dst).insert_many_defaults(size);
} else {
if (is_string) {
auto sv = (StringRef*)min_value->cell_ptr();
@@ -489,9 +489,7 @@ Status ColumnReader::_parse_zone_map_skip_null(const
ZoneMapPB& zone_map,
if (zone_map.has_not_null()) {
RETURN_IF_ERROR(min_value_container->from_string(zone_map.min()));
RETURN_IF_ERROR(max_value_container->from_string(zone_map.max()));
- }
-
- if (!zone_map.has_not_null()) {
+ } else {
min_value_container->set_null();
max_value_container->set_null();
}
@@ -1270,7 +1268,7 @@ Status FileColumnIterator::next_batch(size_t* n,
vectorized::MutableColumnPtr& d
const auto* null_col =
vectorized::check_and_get_column<vectorized::ColumnNullable>(dst.get());
if (null_col != nullptr) {
-
const_cast<vectorized::ColumnNullable*>(null_col)->insert_null_elements(
+
const_cast<vectorized::ColumnNullable*>(null_col)->insert_many_defaults(
this_run);
} else {
return Status::InternalError("unexpected column type
in column reader");
@@ -1335,7 +1333,7 @@ Status FileColumnIterator::read_by_rowids(const rowid_t*
rowids, const size_t co
return Status::InternalError("unexpected column
type in column reader");
}
-
const_cast<vectorized::ColumnNullable*>(null_col)->insert_null_elements(
+
const_cast<vectorized::ColumnNullable*>(null_col)->insert_many_defaults(
this_read_count);
} else {
size_t read_count = this_read_count;
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h
b/be/src/olap/rowset/segment_v2/column_reader.h
index 7e32b3a09b3..2afc269a86c 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -768,10 +768,6 @@ public:
Status read_by_rowids(const rowid_t* rowids, const size_t count,
vectorized::MutableColumnPtr& dst) override;
- Status next_batch_of_zone_map(size_t* n, vectorized::MutableColumnPtr&
dst) override {
- return Status::NotSupported("Not supported next_batch_of_zone_map");
- }
-
ordinal_t get_current_ordinal() const override {
if (_sibling_iter) {
return _sibling_iter->get_current_ordinal();
diff --git a/be/src/vec/columns/column_nullable.h
b/be/src/vec/columns/column_nullable.h
index 84b3ce0f82a..801af78ca2b 100644
--- a/be/src/vec/columns/column_nullable.h
+++ b/be/src/vec/columns/column_nullable.h
@@ -246,6 +246,7 @@ public:
get_nested_column().insert_many_continuous_binary_data(data, offsets,
num);
}
+ // Default value in `ColumnNullable` is null
void insert_default() override {
get_nested_column().insert_default();
get_null_map_data().push_back(1);
@@ -265,13 +266,6 @@ public:
_push_false_to_nullmap(num);
}
- void insert_null_elements(int num) {
- get_nested_column().insert_many_defaults(num);
- get_null_map_column().insert_many_vals(1, num);
- _has_null = true;
- _need_update_has_null = false;
- }
-
void pop_back(size_t n) override;
ColumnPtr filter(const Filter& filt, ssize_t result_size_hint) const
override;
diff --git a/be/src/vec/data_types/data_type_struct.cpp
b/be/src/vec/data_types/data_type_struct.cpp
index 780882379ca..acb4eb212ee 100644
--- a/be/src/vec/data_types/data_type_struct.cpp
+++ b/be/src/vec/data_types/data_type_struct.cpp
@@ -204,7 +204,7 @@ Status DataTypeStruct::from_string(ReadBuffer& rb, IColumn*
column) const {
if (field_rb.count() == 4 && strncmp(field_rb.position(), "null", 4)
== 0) {
auto& nested_null_col =
reinterpret_cast<ColumnNullable&>(struct_column->get_column(idx));
- nested_null_col.insert_null_elements(1);
+ nested_null_col.insert_default();
continue;
}
auto st = elems[idx]->from_string(field_rb,
&struct_column->get_column(idx));
diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp
b/be/src/vec/exec/format/orc/vorc_reader.cpp
index 26e41afe3c9..a488bef7455 100644
--- a/be/src/vec/exec/format/orc/vorc_reader.cpp
+++ b/be/src/vec/exec/format/orc/vorc_reader.cpp
@@ -1712,7 +1712,7 @@ Status OrcReader::_fill_doris_data_column(const
std::string& col_name,
col_name);
}
reinterpret_cast<ColumnNullable*>(doris_field->assume_mutable().get())
- ->insert_null_elements(num_values);
+ ->insert_many_defaults(num_values);
}
for (auto read_field : read_fields) {
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
index 207b917666b..ade98ada48b 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
@@ -882,7 +882,7 @@ Status StructColumnReader::read_column_data(ColumnPtr&
doris_column, DataTypePtr
DCHECK(doris_type->is_nullable());
auto mutable_column = doris_field->assume_mutable();
auto* nullable_column =
static_cast<vectorized::ColumnNullable*>(mutable_column.get());
- nullable_column->insert_null_elements(missing_column_sz);
+ nullable_column->insert_many_defaults(missing_column_sz);
}
if (null_map_ptr != nullptr) {
diff --git a/be/src/vec/olap/vgeneric_iterators.cpp
b/be/src/vec/olap/vgeneric_iterators.cpp
index fe37abd08fa..bb107a5c1d6 100644
--- a/be/src/vec/olap/vgeneric_iterators.cpp
+++ b/be/src/vec/olap/vgeneric_iterators.cpp
@@ -73,9 +73,8 @@ Status VStatisticsIterator::next_batch(Block* block) {
? 2
: std::min(_target_rows - _output_rows,
MAX_ROW_SIZE_IN_COUNT);
if (_push_down_agg_type_opt == TPushAggOp::COUNT) {
- size = std::min(_target_rows - _output_rows,
MAX_ROW_SIZE_IN_COUNT);
- for (int i = 0; i < columns.size(); ++i) {
- columns[i]->insert_many_defaults(size);
+ for (auto& column : columns) {
+ column->insert_many_defaults(size);
}
} else {
for (int i = 0; i < columns.size(); ++i) {
diff --git a/be/test/vec/columns/column_nullable_test.cpp
b/be/test/vec/columns/column_nullable_test.cpp
index 9e66d5bb302..3cd269d51da 100644
--- a/be/test/vec/columns/column_nullable_test.cpp
+++ b/be/test/vec/columns/column_nullable_test.cpp
@@ -61,7 +61,7 @@ TEST(ColumnNullableTest, NullTest) {
dst_col->clear();
EXPECT_FALSE(dst_col->has_null());
- dst_col->insert_null_elements(10);
+ dst_col->insert_many_defaults(10);
EXPECT_TRUE(dst_col->has_null());
dst_col->clear();
@@ -93,12 +93,12 @@ TEST(ColumnNullableTest, PredicateTest) {
ColumnNullable::create(PredicateColumnType<TYPE_DATE>::create(),
ColumnUInt8::create());
nullable_pred->insert_many_defaults(3);
EXPECT_TRUE(nullable_pred->has_null());
- nullable_pred->insert_null_elements(10);
+ nullable_pred->insert_many_defaults(10);
EXPECT_TRUE(nullable_pred->has_null());
nullable_pred->clear();
EXPECT_FALSE(nullable_pred->has_null());
- nullable_pred->insert_null_elements(10);
+ nullable_pred->insert_many_defaults(10);
EXPECT_TRUE(nullable_pred->has_null()); // now it have 10 nulls
auto null_dst = ColumnNullable::create(ColumnDate::create(),
ColumnUInt8::create());
diff --git a/be/test/vec/core/block_test.cpp b/be/test/vec/core/block_test.cpp
index 54cb15f8d48..134264f3e80 100644
--- a/be/test/vec/core/block_test.cpp
+++ b/be/test/vec/core/block_test.cpp
@@ -220,7 +220,7 @@ void
serialize_and_deserialize_test(segment_v2::CompressionTypePB compression_ty
vectorized::DataTypePtr nullable_data_type(
std::make_shared<vectorized::DataTypeNullable>(string_data_type));
auto nullable_column = nullable_data_type->create_column();
-
((vectorized::ColumnNullable*)nullable_column.get())->insert_null_elements(1024);
+
((vectorized::ColumnNullable*)nullable_column.get())->insert_many_defaults(1024);
vectorized::ColumnWithTypeAndName
type_and_name(nullable_column->get_ptr(),
nullable_data_type,
"test_nullable");
vectorized::Block block({type_and_name});
@@ -241,7 +241,7 @@ void
serialize_and_deserialize_test(segment_v2::CompressionTypePB compression_ty
vectorized::DataTypePtr nullable_data_type(
std::make_shared<vectorized::DataTypeNullable>(decimal_data_type));
auto nullable_column = nullable_data_type->create_column();
-
((vectorized::ColumnNullable*)nullable_column.get())->insert_null_elements(1024);
+
((vectorized::ColumnNullable*)nullable_column.get())->insert_many_defaults(1024);
vectorized::ColumnWithTypeAndName type_and_name(
nullable_column->get_ptr(), nullable_data_type,
"test_nullable_decimal");
vectorized::Block block({type_and_name});
diff --git a/be/test/vec/data_types/serde/data_type_serde_pb_test.cpp
b/be/test/vec/data_types/serde/data_type_serde_pb_test.cpp
index 852614e84c5..bf6ead9b21b 100644
--- a/be/test/vec/data_types/serde/data_type_serde_pb_test.cpp
+++ b/be/test/vec/data_types/serde/data_type_serde_pb_test.cpp
@@ -404,7 +404,7 @@ inline void serialize_and_deserialize_pb_test() {
vectorized::DataTypePtr nullable_data_type(
std::make_shared<vectorized::DataTypeNullable>(string_data_type));
auto nullable_column = nullable_data_type->create_column();
-
((vectorized::ColumnNullable*)nullable_column.get())->insert_null_elements(1024);
+
((vectorized::ColumnNullable*)nullable_column.get())->insert_many_defaults(1024);
check_pb_col(nullable_data_type, *nullable_column.get());
}
// nullable decimal
@@ -414,7 +414,7 @@ inline void serialize_and_deserialize_pb_test() {
vectorized::DataTypePtr nullable_data_type(
std::make_shared<vectorized::DataTypeNullable>(decimal_data_type));
auto nullable_column = nullable_data_type->create_column();
-
((vectorized::ColumnNullable*)nullable_column.get())->insert_null_elements(1024);
+
((vectorized::ColumnNullable*)nullable_column.get())->insert_many_defaults(1024);
check_pb_col(nullable_data_type, *nullable_column.get());
}
// int with 1024 batch size
diff --git a/be/test/vec/data_types/serde/data_type_serde_test.cpp
b/be/test/vec/data_types/serde/data_type_serde_test.cpp
index 82674b0aa44..b653edf3e4e 100644
--- a/be/test/vec/data_types/serde/data_type_serde_test.cpp
+++ b/be/test/vec/data_types/serde/data_type_serde_test.cpp
@@ -166,7 +166,7 @@ inline void serialize_and_deserialize_pb_test() {
vectorized::DataTypePtr nullable_data_type(
std::make_shared<vectorized::DataTypeNullable>(string_data_type));
auto nullable_column = nullable_data_type->create_column();
-
((vectorized::ColumnNullable*)nullable_column.get())->insert_null_elements(1024);
+
((vectorized::ColumnNullable*)nullable_column.get())->insert_many_defaults(1024);
check_pb_col(nullable_data_type, *nullable_column.get());
}
// nullable decimal
@@ -175,7 +175,7 @@ inline void serialize_and_deserialize_pb_test() {
vectorized::DataTypePtr nullable_data_type(
std::make_shared<vectorized::DataTypeNullable>(decimal_data_type));
auto nullable_column = nullable_data_type->create_column();
-
((vectorized::ColumnNullable*)nullable_column.get())->insert_null_elements(1024);
+
((vectorized::ColumnNullable*)nullable_column.get())->insert_many_defaults(1024);
check_pb_col(nullable_data_type, *nullable_column.get());
}
// int with 1024 batch size
diff --git a/be/test/vec/jsonb/serialize_test.cpp
b/be/test/vec/jsonb/serialize_test.cpp
index 47819c75072..f3bfc4448fa 100644
--- a/be/test/vec/jsonb/serialize_test.cpp
+++ b/be/test/vec/jsonb/serialize_test.cpp
@@ -448,7 +448,7 @@ TEST(BlockSerializeTest, JsonbBlock) {
vectorized::DataTypePtr nullable_data_type(
std::make_shared<vectorized::DataTypeNullable>(string_data_type));
auto nullable_column = nullable_data_type->create_column();
-
((vectorized::ColumnNullable*)nullable_column.get())->insert_null_elements(1024);
+
((vectorized::ColumnNullable*)nullable_column.get())->insert_many_defaults(1024);
vectorized::ColumnWithTypeAndName
type_and_name(nullable_column->get_ptr(),
nullable_data_type,
"test_nullable");
block.insert(type_and_name);
@@ -459,7 +459,7 @@ TEST(BlockSerializeTest, JsonbBlock) {
vectorized::DataTypePtr nullable_data_type(
std::make_shared<vectorized::DataTypeNullable>(decimal_data_type));
auto nullable_column = nullable_data_type->create_column();
-
((vectorized::ColumnNullable*)nullable_column.get())->insert_null_elements(1024);
+
((vectorized::ColumnNullable*)nullable_column.get())->insert_many_defaults(1024);
vectorized::ColumnWithTypeAndName type_and_name(
nullable_column->get_ptr(), nullable_data_type,
"test_nullable_decimal");
block.insert(type_and_name);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
