amorynan commented on code in PR #23489:
URL: https://github.com/apache/doris/pull/23489#discussion_r1305355703
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -904,58 +904,87 @@ size_t ArrayColumnWriter::get_inverted_index_size() {
return 0;
}
-// Now we can only write data one by one.
+// batch append data for array
Status ArrayColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) {
- size_t remaining = num_rows;
- const auto* col_cursor = reinterpret_cast<const CollectionValue*>(*ptr);
- while (remaining > 0) {
- // TODO llj: bulk write
- size_t num_written = 1;
- ordinal_t next_item_ordinal = _item_writer->get_next_rowid();
- RETURN_IF_ERROR(_offset_writer->append_data_in_current_page(
- reinterpret_cast<uint8_t*>(&next_item_ordinal), &num_written));
- if (num_written <
- 1) { // page is full, write first item offset and update current
length page's start ordinal
- RETURN_IF_ERROR(_offset_writer->finish_current_page());
- } else {
- // write child item.
- if (_item_writer->is_nullable()) {
- auto* item_data_ptr =
const_cast<CollectionValue*>(col_cursor)->mutable_data();
- for (size_t i = 0; i < col_cursor->length(); ++i) {
-
RETURN_IF_ERROR(_item_writer->append(col_cursor->is_null_at(i), item_data_ptr));
- item_data_ptr = (uint8_t*)item_data_ptr +
_item_writer->get_field()->size();
- }
- } else {
- const void* data = col_cursor->data();
-
RETURN_IF_ERROR(_item_writer->append_data(reinterpret_cast<const
uint8_t**>(&data),
-
col_cursor->length()));
- }
- if (_opts.inverted_index) {
- auto writer =
dynamic_cast<ScalarColumnWriter*>(_item_writer.get());
- if (writer != nullptr) {
- //NOTE: use array field name as index field, but
item_writer size should be used when moving item_data_ptr
-
_inverted_index_builder->add_array_values(_item_writer->get_field()->size(),
- col_cursor, 1);
- }
- }
- }
- remaining -= num_written;
- col_cursor += num_written;
- *ptr += num_written * sizeof(CollectionValue);
- }
+ // data_ptr contains
+ // [size, offset_ptr, item_data_ptr, item_nullmap_ptr]
Review Comment:
no this will not make disk layout change, just improve flush to disk action,
before we make item data and offset num one by one write to page, now we just
batch append offset data and item data to page
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
