yiguolei commented on code in PR #64674:
URL: https://github.com/apache/doris/pull/64674#discussion_r3540883294
##########
be/src/storage/rowset/segment_creator.cpp:
##########
@@ -69,20 +69,46 @@ Status SegmentFlusher::flush_single_block(const Block*
block, int32_t segment_id
return Status::OK();
}
Block flush_block(*block);
+ const size_t input_rows = flush_block.rows();
+ segment_v2::TransformExecContext transform_ctx {
+ .tablet_schema = _context.tablet_schema,
+ .write_type = _context.write_type,
+ .tablet = _context.tablet,
+ .mow_context = _context.mow_context,
+ .partial_update_info = _context.partial_update_info,
+ .rowset_ctx = &_context,
+ .rowset_id = _context.rowset_id,
+ .segment_id = segment_id,
+ .derived_column = {},
+ .partial_update_stats = {}};
+ RETURN_IF_ERROR_OR_CATCH_EXCEPTION(
+ segment_v2::build_transform_chain(_context).apply(transform_ctx,
&flush_block));
+ // partial update stats come from the transform chain; add them into the
+ // flusher totals directly, the writers never see them
+ _num_rows_updated += transform_ctx.partial_update_stats.num_rows_updated;
+ _num_rows_deleted += transform_ctx.partial_update_stats.num_rows_deleted;
+ _num_rows_new_added +=
transform_ctx.partial_update_stats.num_rows_new_added;
+ _num_rows_filtered += transform_ctx.partial_update_stats.num_rows_filtered;
bool no_compression = flush_block.bytes() <=
config::segment_compression_threshold_kb * 1024;
- bool use_vertical_segment_writer =
- config::enable_vertical_segment_writer &&
!_context.write_binlog_opt().enable;
+ bool use_vertical_segment_writer = config::enable_vertical_segment_writer;
if (use_vertical_segment_writer) {
std::unique_ptr<segment_v2::VerticalSegmentWriter> writer;
RETURN_IF_ERROR(_create_segment_writer(writer, segment_id,
no_compression));
+ // the vertical writer feeds the derived column in small fixed-size
batches
+ writer->set_derived_column(std::move(transform_ctx.derived_column));
RETURN_IF_ERROR_OR_CATCH_EXCEPTION(_add_rows(writer, &flush_block, 0,
flush_block.rows()));
RETURN_IF_ERROR(_flush_segment_writer(writer, flush_size));
} else {
+ // the horizontal writer has no streaming feed, build it all up front
+
RETURN_IF_ERROR(segment_v2::materialize_derived_columns(transform_ctx.derived_column,
Review Comment:
为什么vertical segment writer 就不需要materialize derived column？
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
