github-actions[bot] commented on code in PR #40320:
URL: https://github.com/apache/doris/pull/40320#discussion_r1741658901
##########
be/src/vec/columns/column_object.cpp:
##########
@@ -415,11 +456,77 @@ void ColumnObject::Subcolumn::insert(Field field,
FieldInfo info) {
data.back()->insert(field);
}
-void ColumnObject::Subcolumn::insertRangeFrom(const Subcolumn& src, size_t
start, size_t length) {
+static DataTypePtr create_array(TypeIndex type, size_t num_dimensions) {
+ DataTypePtr result_type =
make_nullable(DataTypeFactory::instance().create_data_type(type));
+ for (size_t i = 0; i < num_dimensions; ++i) {
+ result_type =
make_nullable(std::make_shared<DataTypeArray>(result_type));
+ }
+ return result_type;
+}
+
+Array create_empty_array_field(size_t num_dimensions) {
+ if (num_dimensions == 0) {
+ throw doris::Exception(ErrorCode::INVALID_ARGUMENT,
+ "Cannot create array field with 0 dimensions");
+ }
+
+ Array array;
+ Array* current_array = &array;
+ for (size_t i = 1; i < num_dimensions; ++i) {
+ current_array->push_back(Array());
+ current_array = ¤t_array->back().get<Array&>();
+ }
+
+ return array;
+}
+
+// Recreates column with default scalar values and keeps sizes of arrays.
+static ColumnPtr recreate_column_with_default_values(const ColumnPtr& column,
TypeIndex scalar_type,
+ size_t num_dimensions) {
+ const auto* column_array =
check_and_get_column<ColumnArray>(remove_nullable(column).get());
+ if (column_array != nullptr && num_dimensions != 0) {
+ return make_nullable(ColumnArray::create(
+
recreate_column_with_default_values(column_array->get_data_ptr(), scalar_type,
+ num_dimensions - 1),
+ IColumn::mutate(column_array->get_offsets_ptr())));
+ }
+
+ return create_array(scalar_type, num_dimensions)
+ ->create_column()
+ ->clone_resized(column->size());
+}
+
+ColumnObject::Subcolumn ColumnObject::Subcolumn::clone_with_default_values(
+ const FieldInfo& field_info) const {
+ Subcolumn new_subcolumn(*this);
+ new_subcolumn.least_common_type =
+ LeastCommonType {create_array(field_info.scalar_type_id,
field_info.num_dimensions)};
+
+ for (int i = 0; i < new_subcolumn.data.size(); ++i) {
+ new_subcolumn.data[i] = recreate_column_with_default_values(
+ new_subcolumn.data[i], field_info.scalar_type_id,
field_info.num_dimensions);
+ new_subcolumn.data_types[i] =
create_array_of_type(field_info.scalar_type_id,
+
field_info.num_dimensions, is_nullable);
+ }
+
+ return new_subcolumn;
+}
+
+Field ColumnObject::Subcolumn::get_last_field() const {
+ if (data.empty()) {
+ return Field();
Review Comment:
warning: avoid repeating the return type from the declaration; use a braced
initializer list instead [modernize-return-braced-init-list]
```suggestion
return {};
```
##########
be/src/vec/data_types/serde/data_type_nothing_serde.h:
##########
@@ -0,0 +1,112 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <glog/logging.h>
Review Comment:
warning: 'glog/logging.h' file not found [clang-diagnostic-error]
```cpp
#include <glog/logging.h>
^
```
##########
be/src/olap/rowset/segment_v2/column_reader.cpp:
##########
@@ -1663,4 +1663,56 @@ Status VariantRootColumnIterator::read_by_rowids(const
rowid_t* rowids, const si
return Status::OK();
}
+Status DefaultNestedColumnIterator::next_batch(size_t* n,
vectorized::MutableColumnPtr& dst) {
+ bool has_null = false;
+ return next_batch(n, dst, &has_null);
+}
+
+static void fill_nested_with_defaults(vectorized::MutableColumnPtr& dst,
+ vectorized::MutableColumnPtr&
sibling_column, size_t nrows) {
+ const auto* sibling_array =
vectorized::check_and_get_column<vectorized::ColumnArray>(
+ remove_nullable(sibling_column->get_ptr()));
+ const auto* dst_array =
vectorized::check_and_get_column<vectorized::ColumnArray>(
+ remove_nullable(dst->get_ptr()));
+ if (!dst_array || !sibling_array) {
+ throw doris::Exception(ErrorCode::INTERNAL_ERROR,
+ "Expected array column, but met %s and %s",
dst->get_name(),
+ sibling_column->get_name());
+ }
+ auto new_nested =
+
dst_array->get_data_ptr()->clone_resized(sibling_array->get_data_ptr()->size());
+ auto new_array = make_nullable(vectorized::ColumnArray::create(
+ new_nested->assume_mutable(),
sibling_array->get_offsets_ptr()->assume_mutable()));
+ dst->insert_range_from(*new_array, 0, new_array->size());
+#ifndef NDEBUG
+ if (!dst_array->has_equal_offsets(*sibling_array)) {
+ throw doris::Exception(ErrorCode::INTERNAL_ERROR, "Expected same array
offsets");
+ }
+#endif
+}
+
+Status DefaultNestedColumnIterator::next_batch(size_t* n,
vectorized::MutableColumnPtr& dst,
Review Comment:
warning: pointer parameter 'n' can be pointer to const
[readability-non-const-parameter]
```suggestion
Status DefaultNestedColumnIterator::next_batch(const size_t* n,
vectorized::MutableColumnPtr& dst,
```
##########
be/src/vec/columns/column_object.cpp:
##########
@@ -1383,34 +1592,72 @@
return Status::OK();
}
-void ColumnObject::finalize_if_not() {
- if (!is_finalized()) {
- finalize();
- }
-}
-
-void ColumnObject::finalize(bool ignore_sparse) {
+void ColumnObject::unnest(Subcolumns::NodePtr& entry, Subcolumns& subcolumns)
const {
+ entry->data.finalize();
+ auto nested_column =
entry->data.get_finalized_column_ptr()->assume_mutable();
+ auto* nested_column_nullable =
assert_cast<ColumnNullable*>(nested_column.get());
+ auto* nested_column_array =
+
assert_cast<ColumnArray*>(nested_column_nullable->get_nested_column_ptr().get());
+ auto& offset = nested_column_array->get_offsets_ptr();
+
+ auto* nested_object_nullable = assert_cast<ColumnNullable*>(
+ nested_column_array->get_data_ptr()->assume_mutable().get());
+ auto& nested_object_column =
+
assert_cast<ColumnObject&>(nested_object_nullable->get_nested_column());
+ PathInData nested_path = entry->path;
+ for (auto& nested_entry : nested_object_column.subcolumns) {
+ if (nested_entry->data.least_common_type.get_base_type_id() ==
TypeIndex::Nothing) {
+ continue;
+ }
+ nested_entry->data.finalize();
+ PathInDataBuilder path_builder;
+ // format nested path
+ path_builder.append(nested_path.get_parts(), false);
+ path_builder.append(nested_entry->path.get_parts(), true);
+ auto subnested_column = ColumnArray::create(
+
ColumnNullable::create(nested_entry->data.get_finalized_column_ptr(),
+
nested_object_nullable->get_null_map_column_ptr()),
+ offset);
+ auto nullable_subnested_column = ColumnNullable::create(
+ subnested_column,
nested_column_nullable->get_null_map_column_ptr());
+ auto type = make_nullable(
+
std::make_shared<DataTypeArray>(nested_entry->data.least_common_type.get()));
+ Subcolumn subcolumn(nullable_subnested_column->assume_mutable(), type,
is_nullable);
+ subcolumns.add(path_builder.build(), subcolumn);
+ }
+}
+
+void ColumnObject::finalize(FinalizeMode mode) {
Review Comment:
warning: method 'finalize' can be made const
[readability-make-member-function-const]
be/src/vec/columns/column_object.h:350:
```diff
- void finalize(FinalizeMode mode);
+ void finalize(FinalizeMode mode) const;
```
```suggestion
void ColumnObject::finalize(FinalizeMode mode) const {
```
##########
be/src/vec/columns/column_object.h:
##########
@@ -19,6 +19,7 @@
// and modified by Doris
#pragma once
+#include <butil/compiler_specific.h>
Review Comment:
warning: 'butil/compiler_specific.h' file not found [clang-diagnostic-error]
```cpp
#include <butil/compiler_specific.h>
^
```
##########
be/src/vec/data_types/serde/data_type_nothing_serde.h:
##########
@@ -0,0 +1,112 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <glog/logging.h>
+#include <stdint.h>
Review Comment:
warning: inclusion of deprecated C++ header 'stdint.h'; consider using
'cstdint' instead [modernize-deprecated-headers]
```suggestion
#include <cstdint>
```
##########
be/src/vec/columns/column_object.cpp:
##########
@@ -415,11 +456,77 @@
data.back()->insert(field);
}
-void ColumnObject::Subcolumn::insertRangeFrom(const Subcolumn& src, size_t
start, size_t length) {
+static DataTypePtr create_array(TypeIndex type, size_t num_dimensions) {
+ DataTypePtr result_type =
make_nullable(DataTypeFactory::instance().create_data_type(type));
+ for (size_t i = 0; i < num_dimensions; ++i) {
+ result_type =
make_nullable(std::make_shared<DataTypeArray>(result_type));
+ }
+ return result_type;
+}
+
+Array create_empty_array_field(size_t num_dimensions) {
+ if (num_dimensions == 0) {
+ throw doris::Exception(ErrorCode::INVALID_ARGUMENT,
+ "Cannot create array field with 0 dimensions");
+ }
+
+ Array array;
+ Array* current_array = &array;
+ for (size_t i = 1; i < num_dimensions; ++i) {
+ current_array->push_back(Array());
+ current_array = ¤t_array->back().get<Array&>();
+ }
+
+ return array;
+}
+
+// Recreates column with default scalar values and keeps sizes of arrays.
+static ColumnPtr recreate_column_with_default_values(const ColumnPtr& column,
TypeIndex scalar_type,
+ size_t num_dimensions) {
+ const auto* column_array =
check_and_get_column<ColumnArray>(remove_nullable(column).get());
+ if (column_array != nullptr && num_dimensions != 0) {
+ return make_nullable(ColumnArray::create(
+
recreate_column_with_default_values(column_array->get_data_ptr(), scalar_type,
+ num_dimensions - 1),
+ IColumn::mutate(column_array->get_offsets_ptr())));
+ }
+
+ return create_array(scalar_type, num_dimensions)
+ ->create_column()
+ ->clone_resized(column->size());
+}
+
+ColumnObject::Subcolumn ColumnObject::Subcolumn::clone_with_default_values(
+ const FieldInfo& field_info) const {
+ Subcolumn new_subcolumn(*this);
+ new_subcolumn.least_common_type =
+ LeastCommonType {create_array(field_info.scalar_type_id,
field_info.num_dimensions)};
+
+ for (int i = 0; i < new_subcolumn.data.size(); ++i) {
+ new_subcolumn.data[i] = recreate_column_with_default_values(
+ new_subcolumn.data[i], field_info.scalar_type_id,
field_info.num_dimensions);
+ new_subcolumn.data_types[i] =
create_array_of_type(field_info.scalar_type_id,
+
field_info.num_dimensions, is_nullable);
+ }
+
+ return new_subcolumn;
+}
+
+Field ColumnObject::Subcolumn::get_last_field() const {
+ if (data.empty()) {
+ return Field();
+ }
+
+ const auto& last_part = data.back();
+ assert(!last_part->empty());
+ return (*last_part)[last_part->size() - 1];
+}
+
+void ColumnObject::Subcolumn::insert_range_from(const Subcolumn& src, size_t
start, size_t length) {
Review Comment:
warning: function 'insert_range_from' exceeds recommended size/complexity
thresholds [readability-function-size]
```cpp
void ColumnObject::Subcolumn::insert_range_from(const Subcolumn& src, size_t
start, size_t length) {
^
```
<details>
<summary>Additional context</summary>
**be/src/vec/columns/column_object.cpp:524:** 85 lines including whitespace
and comments (threshold 80)
```cpp
void ColumnObject::Subcolumn::insert_range_from(const Subcolumn& src, size_t
start, size_t length) {
^
```
</details>
##########
be/src/vec/json/path_in_data.h:
##########
@@ -29,15 +29,18 @@
#include <vector>
#include "gen_cpp/segment_v2.pb.h"
Review Comment:
warning: 'gen_cpp/segment_v2.pb.h' file not found [clang-diagnostic-error]
```cpp
#include "gen_cpp/segment_v2.pb.h"
^
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
