This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 5f1dca011b9 branch-3.0: [fix](cloud) fix file cache missing when
compaction and improve bvars #43804 (#43951)
5f1dca011b9 is described below
commit 5f1dca011b91625e449534de435ecc902bdae82a
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Nov 14 19:05:52 2024 +0800
branch-3.0: [fix](cloud) fix file cache missing when compaction and improve
bvars #43804 (#43951)
Cherry-picked from #43804
Signed-off-by: zhengyu <[email protected]>
Co-authored-by: zhengyu <[email protected]>
Co-authored-by: Gavin Chou <[email protected]>
---
be/src/cloud/cloud_tablet.cpp | 6 +
be/src/common/config.cpp | 4 +-
be/src/common/config.h | 4 +-
be/src/io/cache/block_file_cache.cpp | 205 +++++++++++++---------------------
be/src/io/cache/block_file_cache.h | 14 +--
be/src/io/cache/file_cache_common.cpp | 9 +-
be/src/io/cache/file_cache_common.h | 2 +
be/src/olap/options.cpp | 24 ++--
be/src/olap/options.h | 4 +-
9 files changed, 113 insertions(+), 159 deletions(-)
diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp
index 8e10bf15298..ebe06ac558f 100644
--- a/be/src/cloud/cloud_tablet.cpp
+++ b/be/src/cloud/cloud_tablet.cpp
@@ -449,6 +449,12 @@ void CloudTablet::recycle_cached_data(const
std::vector<RowsetSharedPtr>& rowset
if (config::enable_file_cache) {
for (const auto& rs : rowsets) {
+ if (rs.use_count() >= 1) {
+ LOG(WARNING) << "Rowset " << rs->rowset_id().to_string() << "
has "
+ << rs.use_count()
+ << " references. File Cache won't be recycled
when query is using it.";
+ continue;
+ }
for (int seg_id = 0; seg_id < rs->num_segments(); ++seg_id) {
// TODO: Segment::file_cache_key
auto file_key =
Segment::file_cache_key(rs->rowset_id().to_string(), seg_id);
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 53c87a74484..02ebefbc0f3 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1016,7 +1016,7 @@ DEFINE_mBool(variant_throw_exeception_on_invalid_json,
"false");
DEFINE_Bool(enable_file_cache, "false");
// format:
[{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240}]
// format:
[{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240},{"path":"/path/to/file_cache2","total_size":21474836480,"query_limit":10737418240}]
-// format: {"path": "/path/to/file_cache", "total_size":53687091200,
"normal_percent":85, "disposable_percent":10, "index_percent":5}
+// format: {"path": "/path/to/file_cache", "total_size":53687091200,
"ttl_percent":50, "normal_percent":40, "disposable_percent":5,
"index_percent":5}
// format: [{"path": "xxx", "total_size":53687091200, "storage": "memory"}]
// Note1: storage is "disk" by default
// Note2: when the storage is "memory", the path is ignored. So you can set
xxx to anything you like
@@ -1315,8 +1315,6 @@
DEFINE_Int64(num_buffered_reader_prefetch_thread_pool_max_thread, "64");
DEFINE_Int64(num_s3_file_upload_thread_pool_min_thread, "16");
// The max thread num for S3FileUploadThreadPool
DEFINE_Int64(num_s3_file_upload_thread_pool_max_thread, "64");
-// The max ratio for ttl cache's size
-DEFINE_mInt64(max_ttl_cache_ratio, "50");
// The maximum jvm heap usage ratio for hdfs write workload
DEFINE_mDouble(max_hdfs_wirter_jni_heap_usage_ratio, "0.5");
// The sleep milliseconds duration when hdfs write exceeds the maximum usage
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 726835ef85f..56d5e8e648a 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1058,7 +1058,7 @@ DECLARE_Int32(pipeline_executor_size);
DECLARE_Bool(enable_file_cache);
// format:
[{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240}]
// format:
[{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240},{"path":"/path/to/file_cache2","total_size":21474836480,"query_limit":10737418240}]
-// format:
[{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240,"normal_percent":85,
"disposable_percent":10, "index_percent":5}]
+// format:
[{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240,
"ttl_percent":50, "normal_percent":40, "disposable_percent":5,
"index_percent":5}]
// format: [{"path": "xxx", "total_size":53687091200, "storage": "memory"}]
// Note1: storage is "disk" by default
// Note2: when the storage is "memory", the path is ignored. So you can set
xxx to anything you like
@@ -1392,8 +1392,6 @@
DECLARE_Int64(num_buffered_reader_prefetch_thread_pool_max_thread);
DECLARE_Int64(num_s3_file_upload_thread_pool_min_thread);
// The max thread num for S3FileUploadThreadPool
DECLARE_Int64(num_s3_file_upload_thread_pool_max_thread);
-// The max ratio for ttl cache's size
-DECLARE_mInt64(max_ttl_cache_ratio);
// The maximum jvm heap usage ratio for hdfs write workload
DECLARE_mDouble(max_hdfs_wirter_jni_heap_usage_ratio);
// The sleep milliseconds duration when hdfs write exceeds the maximum usage
diff --git a/be/src/io/cache/block_file_cache.cpp
b/be/src/io/cache/block_file_cache.cpp
index 4fb3f3e02cb..596afb64232 100644
--- a/be/src/io/cache/block_file_cache.cpp
+++ b/be/src/io/cache/block_file_cache.cpp
@@ -86,42 +86,42 @@ BlockFileCache::BlockFileCache(const std::string&
cache_base_path,
_total_evict_size_metrics = std::make_shared<bvar::Adder<size_t>>(
_cache_base_path.c_str(), "file_cache_total_evict_size");
-
_evict_by_heat_metrics_matrix[FileCacheType::DISPOSABLE][FileCacheType::NORMAL]
=
+
_evict_by_time_metrics_matrix[FileCacheType::DISPOSABLE][FileCacheType::NORMAL]
=
std::make_shared<bvar::Adder<size_t>>(_cache_base_path.c_str(),
-
"file_cache_evict_by_heat_disposable_to_normal");
-
_evict_by_heat_metrics_matrix[FileCacheType::DISPOSABLE][FileCacheType::INDEX] =
+
"file_cache_evict_by_time_disposable_to_normal");
+
_evict_by_time_metrics_matrix[FileCacheType::DISPOSABLE][FileCacheType::INDEX] =
std::make_shared<bvar::Adder<size_t>>(_cache_base_path.c_str(),
-
"file_cache_evict_by_heat_disposable_to_index");
-
_evict_by_heat_metrics_matrix[FileCacheType::DISPOSABLE][FileCacheType::TTL] =
+
"file_cache_evict_by_time_disposable_to_index");
+
_evict_by_time_metrics_matrix[FileCacheType::DISPOSABLE][FileCacheType::TTL] =
std::make_shared<bvar::Adder<size_t>>(_cache_base_path.c_str(),
-
"file_cache_evict_by_heat_disposable_to_ttl");
-
_evict_by_heat_metrics_matrix[FileCacheType::NORMAL][FileCacheType::DISPOSABLE]
=
+
"file_cache_evict_by_time_disposable_to_ttl");
+
_evict_by_time_metrics_matrix[FileCacheType::NORMAL][FileCacheType::DISPOSABLE]
=
std::make_shared<bvar::Adder<size_t>>(_cache_base_path.c_str(),
-
"file_cache_evict_by_heat_normal_to_disposable");
- _evict_by_heat_metrics_matrix[FileCacheType::NORMAL][FileCacheType::INDEX]
=
+
"file_cache_evict_by_time_normal_to_disposable");
+ _evict_by_time_metrics_matrix[FileCacheType::NORMAL][FileCacheType::INDEX]
=
std::make_shared<bvar::Adder<size_t>>(_cache_base_path.c_str(),
-
"file_cache_evict_by_heat_normal_to_index");
- _evict_by_heat_metrics_matrix[FileCacheType::NORMAL][FileCacheType::TTL] =
+
"file_cache_evict_by_time_normal_to_index");
+ _evict_by_time_metrics_matrix[FileCacheType::NORMAL][FileCacheType::TTL] =
std::make_shared<bvar::Adder<size_t>>(_cache_base_path.c_str(),
-
"file_cache_evict_by_heat_normal_to_ttl");
-
_evict_by_heat_metrics_matrix[FileCacheType::INDEX][FileCacheType::DISPOSABLE] =
+
"file_cache_evict_by_time_normal_to_ttl");
+
_evict_by_time_metrics_matrix[FileCacheType::INDEX][FileCacheType::DISPOSABLE] =
std::make_shared<bvar::Adder<size_t>>(_cache_base_path.c_str(),
-
"file_cache_evict_by_heat_index_to_disposable");
- _evict_by_heat_metrics_matrix[FileCacheType::INDEX][FileCacheType::NORMAL]
=
+
"file_cache_evict_by_time_index_to_disposable");
+ _evict_by_time_metrics_matrix[FileCacheType::INDEX][FileCacheType::NORMAL]
=
std::make_shared<bvar::Adder<size_t>>(_cache_base_path.c_str(),
-
"file_cache_evict_by_heat_index_to_normal");
- _evict_by_heat_metrics_matrix[FileCacheType::INDEX][FileCacheType::TTL] =
+
"file_cache_evict_by_time_index_to_normal");
+ _evict_by_time_metrics_matrix[FileCacheType::INDEX][FileCacheType::TTL] =
std::make_shared<bvar::Adder<size_t>>(_cache_base_path.c_str(),
-
"file_cache_evict_by_heat_index_to_ttl");
-
_evict_by_heat_metrics_matrix[FileCacheType::TTL][FileCacheType::DISPOSABLE] =
+
"file_cache_evict_by_time_index_to_ttl");
+
_evict_by_time_metrics_matrix[FileCacheType::TTL][FileCacheType::DISPOSABLE] =
std::make_shared<bvar::Adder<size_t>>(_cache_base_path.c_str(),
-
"file_cache_evict_by_heat_ttl_to_disposable");
- _evict_by_heat_metrics_matrix[FileCacheType::TTL][FileCacheType::NORMAL] =
+
"file_cache_evict_by_time_ttl_to_disposable");
+ _evict_by_time_metrics_matrix[FileCacheType::TTL][FileCacheType::NORMAL] =
std::make_shared<bvar::Adder<size_t>>(_cache_base_path.c_str(),
-
"file_cache_evict_by_heat_ttl_to_normal");
- _evict_by_heat_metrics_matrix[FileCacheType::TTL][FileCacheType::INDEX] =
+
"file_cache_evict_by_time_ttl_to_normal");
+ _evict_by_time_metrics_matrix[FileCacheType::TTL][FileCacheType::INDEX] =
std::make_shared<bvar::Adder<size_t>>(_cache_base_path.c_str(),
-
"file_cache_evict_by_heat_ttl_to_index");
+
"file_cache_evict_by_time_ttl_to_index");
_evict_by_self_lru_metrics_matrix[FileCacheType::DISPOSABLE] =
std::make_shared<bvar::Adder<size_t>>(_cache_base_path.c_str(),
@@ -197,8 +197,8 @@ BlockFileCache::BlockFileCache(const std::string&
cache_base_path,
"file_cache_hit_ratio_5m", 0.0);
_hit_ratio_1h =
std::make_shared<bvar::Status<double>>(_cache_base_path.c_str(),
"file_cache_hit_ratio_1h", 0.0);
- _disk_limit_mode_metrics =
- std::make_shared<bvar::Status<size_t>>(_cache_base_path.c_str(),
"disk_limit_mode", 0);
+ _disk_limit_mode_metrics = std::make_shared<bvar::Status<size_t>>(
+ _cache_base_path.c_str(), "file_cache_disk_limit_mode", 0);
_disposable_queue = LRUQueue(cache_settings.disposable_queue_size,
cache_settings.disposable_queue_elements, 60
* 60);
@@ -970,67 +970,6 @@ void BlockFileCache::find_evict_candidates(LRUQueue&
queue, size_t size, size_t
}
}
-bool BlockFileCache::try_reserve_for_ttl_without_lru(size_t size,
-
std::lock_guard<std::mutex>& cache_lock) {
- size_t removed_size = 0;
- size_t cur_cache_size = _cur_cache_size;
- auto limit = config::max_ttl_cache_ratio * _capacity;
-
- TEST_INJECTION_POINT_CALLBACK("BlockFileCache::change_limit1", &limit);
-
- if ((_cur_ttl_size + size) * 100 > limit) {
- return false;
- }
-
- size_t normal_queue_size = _normal_queue.get_capacity(cache_lock);
- size_t disposable_queue_size = _disposable_queue.get_capacity(cache_lock);
- size_t index_queue_size = _index_queue.get_capacity(cache_lock);
- if (is_overflow(removed_size, size, cur_cache_size) && normal_queue_size
== 0 &&
- disposable_queue_size == 0 && index_queue_size == 0) {
- return false;
- }
- std::vector<FileBlockCell*> to_evict;
- auto collect_eliminate_fragments = [&](LRUQueue& queue) {
- size_t cur_removed_size = 0;
- find_evict_candidates(queue, size, cur_cache_size, removed_size,
to_evict, cache_lock,
- cur_removed_size);
- };
- if (disposable_queue_size != 0) {
- collect_eliminate_fragments(get_queue(FileCacheType::DISPOSABLE));
- }
- if (normal_queue_size != 0) {
- collect_eliminate_fragments(get_queue(FileCacheType::NORMAL));
- }
- if (index_queue_size != 0) {
- collect_eliminate_fragments(get_queue(FileCacheType::INDEX));
- }
- remove_file_blocks(to_evict, cache_lock);
- if (is_overflow(removed_size, size, cur_cache_size)) {
- return false;
- }
- return true;
-}
-
-bool BlockFileCache::try_reserve_for_ttl(size_t size,
std::lock_guard<std::mutex>& cache_lock) {
- if (try_reserve_for_ttl_without_lru(size, cache_lock)) {
- return true;
- } else if (config::enable_ttl_cache_evict_using_lru) {
- auto& queue = get_queue(FileCacheType::TTL);
- size_t removed_size = 0;
- size_t cur_cache_size = _cur_cache_size;
-
- std::vector<FileBlockCell*> to_evict;
- size_t cur_removed_size = 0;
- find_evict_candidates(queue, size, cur_cache_size, removed_size,
to_evict, cache_lock,
- cur_removed_size);
- remove_file_blocks_and_clean_time_maps(to_evict, cache_lock);
-
- return !is_overflow(removed_size, size, cur_cache_size);
- } else {
- return false;
- }
-}
-
// 1. if async load file cache not finish
// a. evict from lru queue
// 2. if ttl cache
@@ -1283,7 +1222,7 @@ void BlockFileCache::reset_range(const UInt128Wrapper&
hash, size_t offset, size
_cur_cache_size += new_size;
}
-bool BlockFileCache::try_reserve_from_other_queue_by_hot_interval(
+bool BlockFileCache::try_reserve_from_other_queue_by_time_interval(
FileCacheType cur_type, std::vector<FileCacheType> other_cache_types,
size_t size,
int64_t cur_time, std::lock_guard<std::mutex>& cache_lock) {
size_t removed_size = 0;
@@ -1316,7 +1255,7 @@ bool
BlockFileCache::try_reserve_from_other_queue_by_hot_interval(
remove_size_per_type += cell_size;
}
}
- *(_evict_by_heat_metrics_matrix[cache_type][cur_type]) <<
remove_size_per_type;
+ *(_evict_by_time_metrics_matrix[cache_type][cur_type]) <<
remove_size_per_type;
}
remove_file_blocks(to_evict, cache_lock);
@@ -1365,7 +1304,7 @@ bool
BlockFileCache::try_reserve_from_other_queue(FileCacheType cur_cache_type,
std::lock_guard<std::mutex>&
cache_lock) {
// currently, TTL cache is not considered as a candidate
auto other_cache_types = get_other_cache_type_without_ttl(cur_cache_type);
- bool reserve_success = try_reserve_from_other_queue_by_hot_interval(
+ bool reserve_success = try_reserve_from_other_queue_by_time_interval(
cur_cache_type, other_cache_types, size, cur_time, cache_lock);
if (reserve_success ||
!config::file_cache_enable_evict_from_other_queue_by_size) {
return reserve_success;
@@ -1777,50 +1716,56 @@ void BlockFileCache::run_background_operation() {
break;
}
}
+ // report
+ {
+ SCOPED_CACHE_LOCK(_mutex);
+ _cur_cache_size_metrics->set_value(_cur_cache_size);
+ _cur_ttl_cache_size_metrics->set_value(_cur_cache_size -
+
_index_queue.get_capacity(cache_lock) -
+
_normal_queue.get_capacity(cache_lock) -
+
_disposable_queue.get_capacity(cache_lock));
+ _cur_ttl_cache_lru_queue_cache_size_metrics->set_value(
+ _ttl_queue.get_capacity(cache_lock));
+ _cur_ttl_cache_lru_queue_element_count_metrics->set_value(
+ _ttl_queue.get_elements_num(cache_lock));
+
_cur_normal_queue_cache_size_metrics->set_value(_normal_queue.get_capacity(cache_lock));
+ _cur_normal_queue_element_count_metrics->set_value(
+ _normal_queue.get_elements_num(cache_lock));
+
_cur_index_queue_cache_size_metrics->set_value(_index_queue.get_capacity(cache_lock));
+ _cur_index_queue_element_count_metrics->set_value(
+ _index_queue.get_elements_num(cache_lock));
+ _cur_disposable_queue_cache_size_metrics->set_value(
+ _disposable_queue.get_capacity(cache_lock));
+ _cur_disposable_queue_element_count_metrics->set_value(
+ _disposable_queue.get_elements_num(cache_lock));
+
+ if (_num_read_blocks->get_value() > 0) {
+ _hit_ratio->set_value((double)_num_hit_blocks->get_value() /
+ _num_read_blocks->get_value());
+ }
+ if (_num_read_blocks_5m->get_value() > 0) {
+
_hit_ratio_5m->set_value((double)_num_hit_blocks_5m->get_value() /
+ _num_read_blocks_5m->get_value());
+ }
+ if (_num_read_blocks_1h->get_value() > 0) {
+
_hit_ratio_1h->set_value((double)_num_hit_blocks_1h->get_value() /
+ _num_read_blocks_1h->get_value());
+ }
+ }
+
recycle_stale_rowset_async_bottom_half();
recycle_deleted_blocks();
// gc
- int64_t cur_time = UnixSeconds();
- SCOPED_CACHE_LOCK(_mutex);
- while (!_time_to_key.empty()) {
- auto begin = _time_to_key.begin();
- if (cur_time < begin->first) {
- break;
+ {
+ int64_t cur_time = UnixSeconds();
+ SCOPED_CACHE_LOCK(_mutex);
+ while (!_time_to_key.empty()) {
+ auto begin = _time_to_key.begin();
+ if (cur_time < begin->first) {
+ break;
+ }
+ remove_if_ttl_file_unlock(begin->second, false, cache_lock);
}
- remove_if_ttl_file_unlock(begin->second, false, cache_lock);
- }
-
- // report
- _cur_cache_size_metrics->set_value(_cur_cache_size);
- _cur_ttl_cache_size_metrics->set_value(_cur_cache_size -
-
_index_queue.get_capacity(cache_lock) -
-
_normal_queue.get_capacity(cache_lock) -
-
_disposable_queue.get_capacity(cache_lock));
-
_cur_ttl_cache_lru_queue_cache_size_metrics->set_value(_ttl_queue.get_capacity(cache_lock));
- _cur_ttl_cache_lru_queue_element_count_metrics->set_value(
- _ttl_queue.get_elements_num(cache_lock));
-
_cur_normal_queue_cache_size_metrics->set_value(_normal_queue.get_capacity(cache_lock));
- _cur_normal_queue_element_count_metrics->set_value(
- _normal_queue.get_elements_num(cache_lock));
-
_cur_index_queue_cache_size_metrics->set_value(_index_queue.get_capacity(cache_lock));
- _cur_index_queue_element_count_metrics->set_value(
- _index_queue.get_elements_num(cache_lock));
- _cur_disposable_queue_cache_size_metrics->set_value(
- _disposable_queue.get_capacity(cache_lock));
- _cur_disposable_queue_element_count_metrics->set_value(
- _disposable_queue.get_elements_num(cache_lock));
-
- if (_num_read_blocks->get_value() > 0) {
- _hit_ratio->set_value((double)_num_hit_blocks->get_value() /
- _num_read_blocks->get_value());
- }
- if (_num_read_blocks_5m->get_value() > 0) {
- _hit_ratio_5m->set_value((double)_num_hit_blocks_5m->get_value() /
- _num_read_blocks_5m->get_value());
- }
- if (_num_read_blocks_1h->get_value() > 0) {
- _hit_ratio_1h->set_value((double)_num_hit_blocks_1h->get_value() /
- _num_read_blocks_1h->get_value());
}
}
}
diff --git a/be/src/io/cache/block_file_cache.h
b/be/src/io/cache/block_file_cache.h
index 0de33dadc82..f23d5a3799e 100644
--- a/be/src/io/cache/block_file_cache.h
+++ b/be/src/io/cache/block_file_cache.h
@@ -397,10 +397,6 @@ private:
size_t get_available_cache_size(FileCacheType cache_type) const;
- bool try_reserve_for_ttl(size_t size, std::lock_guard<std::mutex>&
cache_lock);
-
- bool try_reserve_for_ttl_without_lru(size_t size,
std::lock_guard<std::mutex>& cache_lock);
-
FileBlocks split_range_into_cells(const UInt128Wrapper& hash, const
CacheContext& context,
size_t offset, size_t size,
FileBlock::State state,
std::lock_guard<std::mutex>& cache_lock);
@@ -436,10 +432,10 @@ private:
void recycle_deleted_blocks();
- bool try_reserve_from_other_queue_by_hot_interval(FileCacheType cur_type,
-
std::vector<FileCacheType> other_cache_types,
- size_t size, int64_t
cur_time,
-
std::lock_guard<std::mutex>& cache_lock);
+ bool try_reserve_from_other_queue_by_time_interval(FileCacheType cur_type,
+
std::vector<FileCacheType> other_cache_types,
+ size_t size, int64_t
cur_time,
+
std::lock_guard<std::mutex>& cache_lock);
bool try_reserve_from_other_queue_by_size(FileCacheType cur_type,
std::vector<FileCacheType>
other_cache_types,
@@ -515,7 +511,7 @@ private:
std::shared_ptr<bvar::Status<size_t>>
_cur_disposable_queue_cache_size_metrics;
std::array<std::shared_ptr<bvar::Adder<size_t>>, 4>
_queue_evict_size_metrics;
std::shared_ptr<bvar::Adder<size_t>> _total_evict_size_metrics;
- std::shared_ptr<bvar::Adder<size_t>> _evict_by_heat_metrics_matrix[4][4];
+ std::shared_ptr<bvar::Adder<size_t>> _evict_by_time_metrics_matrix[4][4];
std::shared_ptr<bvar::Adder<size_t>> _evict_by_size_metrics_matrix[4][4];
std::shared_ptr<bvar::Adder<size_t>> _evict_by_self_lru_metrics_matrix[4];
std::shared_ptr<bvar::Adder<size_t>> _evict_by_try_release;
diff --git a/be/src/io/cache/file_cache_common.cpp
b/be/src/io/cache/file_cache_common.cpp
index 67487930045..19041938a08 100644
--- a/be/src/io/cache/file_cache_common.cpp
+++ b/be/src/io/cache/file_cache_common.cpp
@@ -42,7 +42,8 @@ std::string FileCacheSettings::to_string() const {
FileCacheSettings get_file_cache_settings(size_t capacity, size_t
max_query_cache_size,
size_t normal_percent, size_t
disposable_percent,
- size_t index_percent, const
std::string& storage) {
+ size_t index_percent, size_t
ttl_percent,
+ const std::string& storage) {
io::FileCacheSettings settings;
if (capacity == 0) return settings;
settings.capacity = capacity;
@@ -59,12 +60,12 @@ FileCacheSettings get_file_cache_settings(size_t capacity,
size_t max_query_cach
std::max(settings.index_queue_size / settings.max_file_block_size,
REMOTE_FS_OBJECTS_CACHE_DEFAULT_ELEMENTS);
- settings.ttl_queue_size = per_size * config::max_ttl_cache_ratio;
+ settings.ttl_queue_size = per_size * ttl_percent;
settings.ttl_queue_elements = std::max(settings.ttl_queue_size /
settings.max_file_block_size,
REMOTE_FS_OBJECTS_CACHE_DEFAULT_ELEMENTS);
- settings.query_queue_size =
- settings.capacity - settings.disposable_queue_size -
settings.index_queue_size;
+ settings.query_queue_size = settings.capacity -
settings.disposable_queue_size -
+ settings.index_queue_size -
settings.ttl_queue_size;
settings.query_queue_elements =
std::max(settings.query_queue_size / settings.max_file_block_size,
REMOTE_FS_OBJECTS_CACHE_DEFAULT_ELEMENTS);
diff --git a/be/src/io/cache/file_cache_common.h
b/be/src/io/cache/file_cache_common.h
index 30579ba7851..0d700d93031 100644
--- a/be/src/io/cache/file_cache_common.h
+++ b/be/src/io/cache/file_cache_common.h
@@ -29,6 +29,7 @@ inline static constexpr size_t FILE_CACHE_MAX_FILE_BLOCK_SIZE
= 1 * 1024 * 1024;
inline static constexpr size_t DEFAULT_NORMAL_PERCENT = 40;
inline static constexpr size_t DEFAULT_DISPOSABLE_PERCENT = 5;
inline static constexpr size_t DEFAULT_INDEX_PERCENT = 5;
+inline static constexpr size_t DEFAULT_TTL_PERCENT = 50;
using uint128_t = vectorized::UInt128;
@@ -107,6 +108,7 @@ FileCacheSettings get_file_cache_settings(size_t capacity,
size_t max_query_cach
size_t normal_percent =
DEFAULT_NORMAL_PERCENT,
size_t disposable_percent =
DEFAULT_DISPOSABLE_PERCENT,
size_t index_percent =
DEFAULT_INDEX_PERCENT,
+ size_t ttl_percent =
DEFAULT_TTL_PERCENT,
const std::string& storage = "disk");
struct CacheContext {
diff --git a/be/src/olap/options.cpp b/be/src/olap/options.cpp
index 9c500c10993..8668f8319d1 100644
--- a/be/src/olap/options.cpp
+++ b/be/src/olap/options.cpp
@@ -32,6 +32,7 @@
#include "common/status.h"
#include "gutil/strings/split.h"
#include "gutil/strings/strip.h"
+#include "io/cache/file_cache_common.h"
#include "io/fs/local_file_system.h"
#include "olap/olap_define.h"
#include "olap/utils.h"
@@ -56,6 +57,7 @@ static std::string CACHE_QUERY_LIMIT_SIZE = "query_limit";
static std::string CACHE_NORMAL_PERCENT = "normal_percent";
static std::string CACHE_DISPOSABLE_PERCENT = "disposable_percent";
static std::string CACHE_INDEX_PERCENT = "index_percent";
+static std::string CACHE_TTL_PERCENT = "ttl_percent";
static std::string CACHE_STORAGE = "storage";
static std::string CACHE_STORAGE_DISK = "disk";
static std::string CACHE_STORAGE_MEMORY = "memory";
@@ -206,7 +208,7 @@ void parse_conf_broken_store_paths(const string&
config_path, std::set<std::stri
* [
* {"path": "storage1", "total_size":53687091200,"query_limit":
"10737418240"},
* {"path": "storage2", "total_size":53687091200},
- * {"path": "storage3", "total_size":53687091200, "normal_percent":85,
"disposable_percent":10, "index_percent":5}
+ * {"path": "storage3", "total_size":53687091200, "ttl_percent":50,
"normal_percent":40, "disposable_percent":5, "index_percent":5}
* {"path": "xxx", "total_size":53687091200, "storage": "memory"}
* ]
*/
@@ -265,26 +267,30 @@ Status parse_conf_cache_paths(const std::string&
config_path, std::vector<CacheP
return Status::OK();
};
- size_t normal_percent = 85;
- size_t disposable_percent = 10;
- size_t index_percent = 5;
+ size_t normal_percent = io::DEFAULT_NORMAL_PERCENT;
+ size_t disposable_percent = io::DEFAULT_DISPOSABLE_PERCENT;
+ size_t index_percent = io::DEFAULT_INDEX_PERCENT;
+ size_t ttl_percent = io::DEFAULT_TTL_PERCENT;
bool has_normal_percent = map.HasMember(CACHE_NORMAL_PERCENT.c_str());
bool has_disposable_percent =
map.HasMember(CACHE_DISPOSABLE_PERCENT.c_str());
bool has_index_percent = map.HasMember(CACHE_INDEX_PERCENT.c_str());
- if (has_normal_percent && has_disposable_percent && has_index_percent)
{
+ bool has_ttl_percent = map.HasMember(CACHE_TTL_PERCENT.c_str());
+ if (has_normal_percent && has_disposable_percent && has_index_percent
&& has_ttl_percent) {
RETURN_IF_ERROR(get_percent_value(CACHE_NORMAL_PERCENT,
normal_percent));
RETURN_IF_ERROR(get_percent_value(CACHE_DISPOSABLE_PERCENT,
disposable_percent));
RETURN_IF_ERROR(get_percent_value(CACHE_INDEX_PERCENT,
index_percent));
- } else if (has_normal_percent || has_disposable_percent ||
has_index_percent) {
+ RETURN_IF_ERROR(get_percent_value(CACHE_TTL_PERCENT, ttl_percent));
+ } else if (has_normal_percent || has_disposable_percent ||
has_index_percent ||
+ has_ttl_percent) {
return Status::InvalidArgument(
"cache percent config must either be all set or all
unset.");
}
- if ((normal_percent + disposable_percent + index_percent) != 100) {
+ if ((normal_percent + disposable_percent + index_percent +
ttl_percent) != 100) {
return Status::InvalidArgument("The sum of cache percent config
must equal 100.");
}
paths.emplace_back(std::move(path), total_size, query_limit_bytes,
normal_percent,
- disposable_percent, index_percent, storage);
+ disposable_percent, index_percent, ttl_percent,
storage);
}
if (paths.empty()) {
return Status::InvalidArgument("fail to parse storage_root_path
config. value={}",
@@ -295,7 +301,7 @@ Status parse_conf_cache_paths(const std::string&
config_path, std::vector<CacheP
io::FileCacheSettings CachePath::init_settings() const {
return io::get_file_cache_settings(total_bytes, query_limit_bytes,
normal_percent,
- disposable_percent, index_percent,
storage);
+ disposable_percent, index_percent,
ttl_percent, storage);
}
} // end namespace doris
diff --git a/be/src/olap/options.h b/be/src/olap/options.h
index 7fec49468f8..cb352e56f6e 100644
--- a/be/src/olap/options.h
+++ b/be/src/olap/options.h
@@ -54,13 +54,14 @@ struct CachePath {
CachePath(std::string path, int64_t total_bytes, int64_t query_limit_bytes,
size_t normal_percent, size_t disposable_percent, size_t
index_percent,
- std::string storage)
+ size_t ttl_percent, std::string storage)
: path(std::move(path)),
total_bytes(total_bytes),
query_limit_bytes(query_limit_bytes),
normal_percent(normal_percent),
disposable_percent(disposable_percent),
index_percent(index_percent),
+ ttl_percent(ttl_percent),
storage(storage) {}
std::string path;
@@ -69,6 +70,7 @@ struct CachePath {
size_t normal_percent = io::DEFAULT_NORMAL_PERCENT;
size_t disposable_percent = io::DEFAULT_DISPOSABLE_PERCENT;
size_t index_percent = io::DEFAULT_INDEX_PERCENT;
+ size_t ttl_percent = io::DEFAULT_TTL_PERCENT;
std::string storage = "disk";
};
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
