Gabriel39 commented on code in PR #63376:
URL: https://github.com/apache/doris/pull/63376#discussion_r3295811164
##########
be/src/io/fs/file_meta_cache.cpp:
##########
@@ -40,4 +45,54 @@ std::string FileMetaCache::get_key(io::FileReaderSPtr
file_reader,
_file_description.file_size == -1 ? file_reader->size() :
_file_description.file_size);
}
+bool FileMetaCache::should_enable_memory_cache(int64_t num_scan_ranges) const {
+ return enabled() && num_scan_ranges <
config::max_external_file_meta_cache_num / 3;
+}
+
+bool FileMetaCache::should_enable_for_reader(int64_t num_scan_ranges) const {
+ return should_enable_memory_cache(num_scan_ranges) ||
Review Comment:
这个条件只用config::enable_external_file_meta_disk_cache就行
##########
be/src/exec/scan/file_scanner.cpp:
##########
@@ -1160,25 +1160,29 @@ Status FileScanner::_get_next_reader() {
break;
}
case TFileFormatType::FORMAT_PARQUET: {
- auto file_meta_cache_ptr = _should_enable_file_meta_cache()
- ?
ExecEnv::GetInstance()->file_meta_cache()
- : nullptr;
+ auto* file_meta_cache_ptr = _should_enable_file_meta_cache()
+ ?
ExecEnv::GetInstance()->file_meta_cache()
+ : nullptr;
+ const bool enable_file_meta_memory_cache =
_should_enable_file_meta_memory_cache();
Review Comment:
不需要这个
##########
be/src/exec/scan/file_scanner.cpp:
##########
@@ -993,9 +993,9 @@ void FileScanner::_truncate_char_or_varchar_column(Block*
block, int idx, int le
std::shared_ptr<segment_v2::RowIdColumnIteratorV2>
FileScanner::_create_row_id_column_iterator() {
auto& id_file_map = _state->get_id_file_map();
- auto file_id = id_file_map->get_file_mapping_id(
-
std::make_shared<FileMapping>(((FileScanLocalState*)_local_state)->parent_id(),
- _current_range,
_should_enable_file_meta_cache()));
+ auto file_id =
id_file_map->get_file_mapping_id(std::make_shared<FileMapping>(
+ ((FileScanLocalState*)_local_state)->parent_id(), _current_range,
+ _should_enable_file_meta_cache(),
_should_enable_file_meta_memory_cache()));
Review Comment:
这里不需要两个开关，我们只需要一个是否开启cache的开关，cache里内存的占比应该是有cache内部机制通过一个比例来控制的
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
