This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-1.1-lts in repository https://gitbox.apache.org/repos/asf/doris.git
commit 27a5dab3447712bb6f80f0a4447be62e2bbc8efe Author: zhannngchen <[email protected]> AuthorDate: Wed Sep 21 14:33:05 2022 +0800 [Enhancement](load) optimize flush policy to avoid small segments #12706 In current policy, if mem-limit exceeded, load channel will pick tablets that consume most memory, but mem_consumption contains memory in flush, if some delta writer flushing a full memtable(default 200MB), the current memtable might be very small, we should avoid flush such memtable, which can generate a very small segment. --- be/src/olap/delta_writer.cpp | 17 ++++++++++++----- be/src/olap/delta_writer.h | 8 ++++---- be/src/runtime/tablets_channel.cpp | 8 ++++---- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp index b33b5b215b..7dff080a9d 100644 --- a/be/src/olap/delta_writer.cpp +++ b/be/src/olap/delta_writer.cpp @@ -335,13 +335,13 @@ OLAPStatus DeltaWriter::cancel() { return OLAP_SUCCESS; } -int64_t DeltaWriter::save_mem_consumption_snapshot() { - _mem_consumption_snapshot = mem_consumption(); - return _mem_consumption_snapshot; +int64_t DeltaWriter::save_memtable_consumption_snapshot() { + _memtable_consumption_snapshot = memtable_consumption(); + return _memtable_consumption_snapshot; } -int64_t DeltaWriter::get_mem_consumption_snapshot() const { - return _mem_consumption_snapshot; +int64_t DeltaWriter::get_memtable_consumption_snapshot() const { + return _memtable_consumption_snapshot; } int64_t DeltaWriter::mem_consumption() const { @@ -353,6 +353,13 @@ int64_t DeltaWriter::mem_consumption() const { return _mem_tracker->consumption(); } +int64_t DeltaWriter::memtable_consumption() const { + if (_mem_table == nullptr) { + return 0; + } + return _mem_table->memory_usage(); +} + int64_t DeltaWriter::partition_id() const { return _req.partition_id; } diff --git a/be/src/olap/delta_writer.h b/be/src/olap/delta_writer.h index cf65201e14..674601677f 100644 --- a/be/src/olap/delta_writer.h +++ b/be/src/olap/delta_writer.h @@ -92,10 +92,10 @@ public: int32_t schema_hash() { return _tablet->schema_hash(); } int64_t memtable_consumption() const; - - int64_t save_mem_consumption_snapshot(); - int64_t get_mem_consumption_snapshot() const; + int64_t save_memtable_consumption_snapshot(); + + int64_t get_memtable_consumption_snapshot() const; private: DeltaWriter(WriteRequest* req, const std::shared_ptr<MemTracker>& parent, @@ -131,7 +131,7 @@ private: std::mutex _lock; //only used for std::sort more detail see issue(#9237) - int64_t _mem_consumption_snapshot = 0; + int64_t _memtable_consumption_snapshot = 0; }; } // namespace doris diff --git a/be/src/runtime/tablets_channel.cpp b/be/src/runtime/tablets_channel.cpp index 5ce3fe0ab0..1e1ab2cd7b 100644 --- a/be/src/runtime/tablets_channel.cpp +++ b/be/src/runtime/tablets_channel.cpp @@ -235,11 +235,11 @@ Status TabletsChannel::reduce_mem_usage(int64_t mem_limit) { // Sort the DeltaWriters by mem consumption in descend order. std::vector<DeltaWriter*> writers; for (auto& it : _tablet_writers) { - it.second->save_mem_consumption_snapshot(); + it.second->save_memtable_consumption_snapshot(); writers.push_back(it.second); } std::sort(writers.begin(), writers.end(), [](const DeltaWriter* lhs, const DeltaWriter* rhs) { - return lhs->get_mem_consumption_snapshot() > rhs->get_mem_consumption_snapshot(); + return lhs->get_memtable_consumption_snapshot() > rhs->get_memtable_consumption_snapshot(); }); // Decide which writes should be flushed to reduce mem consumption. @@ -257,11 +257,11 @@ Status TabletsChannel::reduce_mem_usage(int64_t mem_limit) { int counter = 0; int64_t sum = 0; for (auto writer : writers) { - if (writer->mem_consumption() <= 0) { + if (writer->memtable_consumption() <= 0) { break; } ++counter; - sum += writer->mem_consumption(); + sum += writer->memtable_consumption(); if (sum > mem_to_flushed) { break; } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
