yiguolei commented on code in PR #40144:
URL: https://github.com/apache/doris/pull/40144#discussion_r1745168717
##########
be/src/vec/exec/scan/scanner_scheduler.cpp:
##########
@@ -267,32 +271,42 @@ void
ScannerScheduler::_scanner_scan(std::shared_ptr<ScannerContext> ctx,
if (free_block == nullptr) {
break;
}
+ // We got a new created block or a reused block.
+ ctx->update_peak_memory_usage(free_block->allocated_bytes());
+ ctx->update_peak_memory_usage(-free_block->allocated_bytes());
status = scanner->get_block_after_projects(state,
free_block.get(), &eos);
+ // Projection will truncate useless columns, makes block size
change.
+ auto free_block_bytes = free_block->allocated_bytes();
+ ctx->update_peak_memory_usage(free_block_bytes);
first_read = false;
if (!status.ok()) {
LOG(WARNING) << "Scan thread read VScanner failed: " <<
status.to_string();
break;
}
- auto free_block_bytes = free_block->allocated_bytes();
raw_bytes_read += free_block_bytes;
+
if (!scan_task->cached_blocks.empty() &&
scan_task->cached_blocks.back().first->rows() +
free_block->rows() <=
ctx->batch_size()) {
- size_t block_size =
scan_task->cached_blocks.back().first->allocated_bytes();
vectorized::MutableBlock mutable_block(
scan_task->cached_blocks.back().first.get());
+
ctx->update_peak_memory_usage(-mutable_block.allocated_bytes());
status = mutable_block.merge(*free_block);
+
ctx->update_peak_memory_usage(mutable_block.allocated_bytes());
if (!status.ok()) {
LOG(WARNING) << "Block merge failed: " <<
status.to_string();
break;
}
+ scan_task->cached_blocks.back().second =
mutable_block.allocated_bytes();
scan_task->cached_blocks.back().first.get()->set_columns(
std::move(mutable_block.mutable_columns()));
+ // If block can be reused, its memory usage will be added
back.
ctx->return_free_block(std::move(free_block));
- ctx->inc_free_block_usage(
-
scan_task->cached_blocks.back().first->allocated_bytes() - block_size);
+ // Return block succeed or not, this free_block is not
used by this scan task any more.
+ ctx->update_peak_memory_usage(-free_block_bytes);
+ ctx->inc_free_block_usage(free_block_bytes);
} else {
- ctx->inc_free_block_usage(free_block->allocated_bytes());
+ ctx->inc_free_block_usage(free_block_bytes);
Review Comment:
我感觉，我们之前这个逻辑是错的。。。 这里不应该把free block size 加上去
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
