This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit aec036ea222c6dc79a2e710d92910ccd38a11c4c Author: amory <[email protected]> AuthorDate: Wed Sep 6 14:47:37 2023 +0800 [FIX](array)fix array batch append data with right next_array_item_rowid (#23779) --- be/src/olap/rowset/segment_v2/column_writer.cpp | 10 ++-------- be/src/olap/rowset/segment_v2/column_writer.h | 7 +++---- be/src/vec/olap/olap_data_convertor.cpp | 6 ++---- 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/column_writer.cpp b/be/src/olap/rowset/segment_v2/column_writer.cpp index acf5c029d0..7a446f1123 100644 --- a/be/src/olap/rowset/segment_v2/column_writer.cpp +++ b/be/src/olap/rowset/segment_v2/column_writer.cpp @@ -221,7 +221,7 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn* length_column.set_index_length(-1); // no short key index std::unique_ptr<Field> bigint_field(FieldFactory::create(length_column)); auto* length_writer = - new ScalarColumnWriter(length_options, std::move(bigint_field), file_writer); + new OffsetColumnWriter(length_options, std::move(bigint_field), file_writer); // if nullable, create null writer ScalarColumnWriter* null_writer = nullptr; @@ -895,7 +895,7 @@ Status StructColumnWriter::finish_current_page() { //////////////////////////////////////////////////////////////////////////////// ArrayColumnWriter::ArrayColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr<Field> field, - ScalarColumnWriter* offset_writer, + OffsetColumnWriter* offset_writer, ScalarColumnWriter* null_writer, std::unique_ptr<ColumnWriter> item_writer) : ColumnWriter(std::move(field), opts.meta->is_nullable()), @@ -913,7 +913,6 @@ Status ArrayColumnWriter::init() { RETURN_IF_ERROR(_null_writer->init()); } RETURN_IF_ERROR(_item_writer->init()); - _offset_writer->register_flush_page_callback(this); if (_opts.inverted_index) { auto writer = dynamic_cast<ScalarColumnWriter*>(_item_writer.get()); if (writer != nullptr) { @@ -927,11 +926,6 @@ Status ArrayColumnWriter::init() { return Status::OK(); } -Status ArrayColumnWriter::put_extra_info_in_page(DataPageFooterPB* footer) { - footer->set_next_array_item_ordinal(_item_writer->get_next_rowid()); - return Status::OK(); -} - Status ArrayColumnWriter::write_inverted_index() { if (_opts.inverted_index) { return _inverted_index_builder->finish(); diff --git a/be/src/olap/rowset/segment_v2/column_writer.h b/be/src/olap/rowset/segment_v2/column_writer.h index dfda60b53f..1bc0afb972 100644 --- a/be/src/olap/rowset/segment_v2/column_writer.h +++ b/be/src/olap/rowset/segment_v2/column_writer.h @@ -349,10 +349,10 @@ private: ColumnWriterOptions _opts; }; -class ArrayColumnWriter final : public ColumnWriter, public FlushPageCallback { +class ArrayColumnWriter final : public ColumnWriter { public: explicit ArrayColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr<Field> field, - ScalarColumnWriter* offset_writer, ScalarColumnWriter* null_writer, + OffsetColumnWriter* offset_writer, ScalarColumnWriter* null_writer, std::unique_ptr<ColumnWriter> item_writer); ~ArrayColumnWriter() override = default; @@ -394,12 +394,11 @@ public: ordinal_t get_next_rowid() const override { return _offset_writer->get_next_rowid(); } private: - Status put_extra_info_in_page(DataPageFooterPB* header) override; Status write_null_column(size_t num_rows, bool is_null); // 写入num_rows个null标记 bool has_empty_items() const { return _item_writer->get_next_rowid() == 0; } private: - std::unique_ptr<ScalarColumnWriter> _offset_writer; + std::unique_ptr<OffsetColumnWriter> _offset_writer; std::unique_ptr<ScalarColumnWriter> _null_writer; std::unique_ptr<ColumnWriter> _item_writer; std::unique_ptr<InvertedIndexColumnWriter> _inverted_index_builder; diff --git a/be/src/vec/olap/olap_data_convertor.cpp b/be/src/vec/olap/olap_data_convertor.cpp index d339accc7b..e7b59033c4 100644 --- a/be/src/vec/olap/olap_data_convertor.cpp +++ b/be/src/vec/olap/olap_data_convertor.cpp @@ -943,8 +943,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorArray::convert_to_olap( auto elem_size = end_offset - start_offset; _offsets.clear(); - // we need all offsets, so reserve num_rows + 1 to make sure last offset can be got in offset column, - // instead of according to nested item column + // we need all offsets, so reserve num_rows + 1 to make sure last offset can be got in offset column, instead of according to nested item column _offsets.reserve(_num_rows + 1); for (int i = 0; i <= _num_rows; ++i) { _offsets.push_back(column_array->offset_at(i + _row_pos) - start_offset + _base_offset); @@ -1013,8 +1012,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorMap::convert_to_olap( auto elem_size = end_offset - start_offset; _offsets.clear(); - // we need all offsets, so reserve num_rows + 1 to make sure last offset can be got in offset column, - // instead of according to nested item column + // we need all offsets, so reserve num_rows + 1 to make sure last offset can be got in offset column, instead of according to nested item column _offsets.reserve(_num_rows + 1); for (int i = 0; i <= _num_rows; ++i) { _offsets.push_back(column_map->offset_at(i + _row_pos) - start_offset + _base_offset); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
