This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 099722d36aa branch-3.0: [opt](filecache) Do not remove parent
directory when clear file cache #49199 (#49360)
099722d36aa is described below
commit 099722d36aa5ce640e6711e32626a7851e14fe7a
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Mar 25 09:57:03 2025 +0800
branch-3.0: [opt](filecache) Do not remove parent directory when clear file
cache #49199 (#49360)
Cherry-picked from #49199
Co-authored-by: Gavin Chou <[email protected]>
---
be/src/io/cache/fs_file_cache_storage.cpp | 42 ++++++++++++++++++++-----------
1 file changed, 28 insertions(+), 14 deletions(-)
diff --git a/be/src/io/cache/fs_file_cache_storage.cpp
b/be/src/io/cache/fs_file_cache_storage.cpp
index c6df21aceed..68efa5a9df0 100644
--- a/be/src/io/cache/fs_file_cache_storage.cpp
+++ b/be/src/io/cache/fs_file_cache_storage.cpp
@@ -700,22 +700,36 @@ void
FSFileCacheStorage::load_blocks_directly_unlocked(BlockFileCache* mgr, cons
}
Status FSFileCacheStorage::clear(std::string& msg) {
+ LOG(INFO) << "clear file storage, path=" << _cache_base_path;
+ std::error_code ec;
+ std::filesystem::directory_iterator key_it {_cache_base_path, ec};
+ if (ec) {
+ LOG(WARNING) << "Failed to list directory: " << _cache_base_path
+ << ", error: " << ec.message();
+ return Status::InternalError("Failed to list dir {}: {}",
_cache_base_path, ec.message());
+ }
+ int failed = 0;
+ int total = 0;
+ auto t0 = std::chrono::steady_clock::now();
+ for (; key_it != std::filesystem::directory_iterator(); ++key_it) {
+ if (!key_it->is_directory()) continue; // all file cache data is in
sub-directories
+ ++total;
+ std::string cache_key = key_it->path().filename().native();
+ auto st = global_local_filesystem()->delete_directory(cache_key);
+ if (st.ok()) continue;
+ failed++;
+ LOG(WARNING) << "failed to clear base_path=" << _cache_base_path
+ << " path_to_delete=" << cache_key << " error=" << st;
+ }
+ auto t1 = std::chrono::steady_clock::now();
std::stringstream ss;
- auto st = global_local_filesystem()->delete_directory(_cache_base_path);
- if (!st.ok()) {
- ss << "failed to clear_file_cache_directly, path=" << _cache_base_path
- << " delete dir failed: " << st;
- LOG(WARNING) << ss.str();
- msg = ss.str();
- return Status::InternalError(ss.str());
- }
- st = global_local_filesystem()->create_directory(_cache_base_path);
- if (!st.ok()) {
- ss << "failed to clear_file_cache_directly, path=" << _cache_base_path
- << " create dir failed: " << st;
- LOG(WARNING) << ss.str();
+ ss << "finished clear file storage, path=" << _cache_base_path
+ << " deleted=" << (total - failed) << " failed=" << failed
+ << " elapsed_ms=" <<
std::chrono::duration_cast<std::chrono::milliseconds>(t1 - t0).count();
+ LOG(INFO) << ss.str();
+ if (failed > 0) {
msg = ss.str();
- return Status::InternalError(ss.str());
+ return Status::InternalError(msg);
}
return Status::OK();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
