github-actions[bot] commented on code in PR #40226:
URL: https://github.com/apache/doris/pull/40226#discussion_r1740878527
##########
be/src/io/cache/fs_file_cache_storage.cpp:
##########
@@ -338,6 +386,72 @@
return Path(_cache_base_path) / "version";
}
+Status FSFileCacheStorage::parse_filename_suffix_to_cache_type(
+ const std::shared_ptr<LocalFileSystem>& fs, const Path& file_path,
long expiration_time,
+ size_t size, size_t* offset, bool* is_tmp, FileCacheType* cache_type)
const {
+ std::error_code ec;
+ std::string offset_with_suffix = file_path.native();
+ auto delim_pos1 = offset_with_suffix.find('_');
+ bool parsed = true;
+
+ try {
+ if (delim_pos1 == std::string::npos) {
+ // same as type "normal"
+ *offset = stoull(offset_with_suffix);
+ } else {
+ *offset = stoull(offset_with_suffix.substr(0, delim_pos1));
+ std::string suffix = offset_with_suffix.substr(delim_pos1 + 1);
+ // not need persistent anymore
+ // if suffix is equals to "tmp", it should be removed too.
+ if (suffix == "tmp") [[unlikely]] {
+ *is_tmp = true;
+ } else {
+ *cache_type = BlockFileCache::string_to_cache_type(suffix);
+ }
+ }
+ } catch (...) {
+ parsed = false;
+ }
+
+ // File in dir with expiration time > 0 should all be TTL type
+ // while expiration time == 0 should all be NORMAL type but
+ // in old days, bug happens, thus break such consistency, e.g.
+ // BEs shut down during cache type transition.
+ // Nowadays, we only use expiration time to decide the type,
+ // i.e. whenever expiration time > 0, it IS TTL, otherwise
+ // it is NORMAL or INDEX depending on its suffix.
+ // From now on, the ttl type encoding in file name is only for
+ // compatibility. It won't be build into the filename, and existing
+ // ones will be ignored.
+ if (expiration_time > 0) {
+ *cache_type = FileCacheType::TTL;
+ } else if (*cache_type == FileCacheType::TTL && expiration_time == 0) {
+ *cache_type = FileCacheType::NORMAL;
+ }
+
+ if (!parsed) {
+ LOG(WARNING) << "parse offset err, path=" << file_path.native();
+ return Status::InternalError("parse offset err, path={}",
file_path.native());
+ }
+ TEST_SYNC_POINT_CALLBACK("BlockFileCache::REMOVE_FILE",
&offset_with_suffix);
+
+ if (ec) {
+ LOG(WARNING) << "failed to file_size: file_name=" << offset_with_suffix
+ << "error=" << ec.message();
+ return Status::InternalError("failed to file_size: file_name={},
error={}",
+ offset_with_suffix, ec.message());
+ }
+
+ if (size == 0 && !(*is_tmp)) {
+ auto st = fs->delete_file(file_path);
+ if (!st.ok()) {
+ LOG_WARNING("delete file {} error", file_path.native()).error(st);
+ }
+ return Status::InternalError("file size is 0, file_name={}",
offset_with_suffix);
+ }
+ return Status::OK();
+}
+
void FSFileCacheStorage::load_cache_info_into_memory(BlockFileCache* _mgr)
const {
Review Comment:
warning: function 'load_cache_info_into_memory' has cognitive complexity of
53 (threshold 50) [readability-function-cognitive-complexity]
```cpp
void FSFileCacheStorage::load_cache_info_into_memory(BlockFileCache* _mgr)
const {
^
```
<details>
<summary>Additional context</summary>
**be/src/io/cache/fs_file_cache_storage.cpp:458:** nesting level increased
to 1
```cpp
auto add_cell_batch_func = [&]() {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:460:** nesting level increased
to 2
```cpp
auto f = [&](const BatchLoadArgs& args) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:462:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (_mgr->_files.contains(args.hash) &&
_mgr->_files[args.hash].contains(args.offset)) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:462:** +1
```cpp
if (_mgr->_files.contains(args.hash) &&
_mgr->_files[args.hash].contains(args.offset)) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:464:** +4, including nesting
penalty of 3, nesting level increased to 4
```cpp
if (file_block->expiration_time() !=
args.ctx.expiration_time ||
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:464:** +1
```cpp
if (file_block->expiration_time() !=
args.ctx.expiration_time ||
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:478:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (!args.is_tmp) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:485:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (ec) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:493:** nesting level increased
to 1
```cpp
auto scan_file_cache = [&](std::filesystem::directory_iterator& key_it) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:495:** +2, including nesting
penalty of 1, nesting level increased to 2
```cpp
for (; key_it != std::filesystem::directory_iterator(); ++key_it) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:504:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (ec) [[unlikely]] {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:513:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
for (; offset_it != std::filesystem::directory_iterator();
++offset_it) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:518:** +4, including nesting
penalty of 3, nesting level increased to 4
```cpp
if (!parse_filename_suffix_to_cache_type(fs,
offset_it->path().filename().native(),
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:535:** +4, including nesting
penalty of 3, nesting level increased to 4
```cpp
if (batch_load_buffer.size() >= scan_length) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:543:** +1, including nesting
penalty of 0, nesting level increased to 1
```cpp
if constexpr (USE_CACHE_VERSION2) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:545:** +2, including nesting
penalty of 1, nesting level increased to 2
```cpp
if (ec) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:549:** +2, including nesting
penalty of 1, nesting level increased to 2
```cpp
for (; key_prefix_it != std::filesystem::directory_iterator();
++key_prefix_it) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:550:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (!key_prefix_it->is_directory()) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:554:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (key_prefix_it->path().filename().native().size() !=
KEY_PREFIX_LENGTH) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:559:** +4, including nesting
penalty of 3, nesting level increased to 4
```cpp
if (ec) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:566:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (ec) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:572:** +1, nesting level
increased to 1
```cpp
} else {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:574:** +2, including nesting
penalty of 1, nesting level increased to 2
```cpp
if (ec) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:580:** +1, including nesting
penalty of 0, nesting level increased to 1
```cpp
if (!batch_load_buffer.empty()) {
^
```
</details>
##########
be/src/io/cache/fs_file_cache_storage.cpp:
##########
@@ -338,6 +352,71 @@
return Path(_cache_base_path) / "version";
}
+Status parse_filename_suffix_to_cache_type(const
std::shared_ptr<LocalFileSystem>& fs,
+ const Path& file_path, long
expiration_time, size_t size,
+ size_t* offset, bool* is_tmp,
FileCacheType* cache_type) {
+ std::error_code ec;
+ std::string offset_with_suffix = file_path.native();
+ auto delim_pos1 = offset_with_suffix.find('_');
+ bool parsed = true;
+
+ try {
+ if (delim_pos1 == std::string::npos) {
+ // same as type "normal"
+ *offset = stoull(offset_with_suffix);
+ } else {
+ *offset = stoull(offset_with_suffix.substr(0, delim_pos1));
+ std::string suffix = offset_with_suffix.substr(delim_pos1 + 1);
+ // not need persistent anymore
+ // if suffix is equals to "tmp", it should be removed too.
+ if (suffix == "tmp") [[unlikely]] {
+ *is_tmp = true;
+ } else {
+ *cache_type = BlockFileCache::string_to_cache_type(suffix);
+ }
+ }
+ } catch (...) {
+ parsed = false;
+ }
+
+ // File in dir with expiration time > 0 should all be TTL type
+ // while expiration time == 0 should all be NORMAL type but
+ // in old days, bug happens, thus break such consistency, e.g.
+ // BEs shut down during cache type transition.
+ // Nowadays, we only use expiration time to decide the type,
+ // i.e. whenever expiration time > 0, it IS TTL, otherwise
+ // it is NORMAL or INDEX depending on its suffix.
+ // From now on, the ttl type encoding in file name is only for
+ // compatibility. It won't be build into the filename, and existing
+ // ones will be ignored.
+ if (expiration_time > 0) {
+ *cache_type = FileCacheType::TTL;
+ } else if (*cache_type == FileCacheType::TTL && expiration_time == 0) {
+ *cache_type = FileCacheType::NORMAL;
+ }
+
+ if (!parsed) {
+ LOG(WARNING) << "parse offset err, path=" << file_path.native();
+ return Status::InternalError("parse offset err, path={}",
file_path.native());
+ }
+ TEST_SYNC_POINT_CALLBACK("BlockFileCache::REMOVE_FILE",
&offset_with_suffix);
+
+ if (ec) {
+ LOG(WARNING) << "failed to file_size: file_name=" << offset_with_suffix
+ << "error=" << ec.message();
+ return Status::InternalError("failed to file_size: file_name={},
error={}",
+ offset_with_suffix, ec.message());
+ }
+
+ if (size == 0 && !(*is_tmp)) {
+ auto st = fs->delete_file(file_path);
+ if (!st.ok()) {
+ LOG_WARNING("delete file {} error", file_path.native()).error(st);
+ }
+ return Status::InternalError("file size is 0, file_name={}",
offset_with_suffix);
+ }
+}
+
void FSFileCacheStorage::load_cache_info_into_memory(BlockFileCache* _mgr)
const {
Review Comment:
warning: function 'load_cache_info_into_memory' has cognitive complexity of
53 (threshold 50) [readability-function-cognitive-complexity]
```cpp
void FSFileCacheStorage::load_cache_info_into_memory(BlockFileCache* _mgr)
const {
^
```
<details>
<summary>Additional context</summary>
**be/src/io/cache/fs_file_cache_storage.cpp:423:** nesting level increased
to 1
```cpp
auto add_cell_batch_func = [&]() {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:425:** nesting level increased
to 2
```cpp
auto f = [&](const BatchLoadArgs& args) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:427:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (_mgr->_files.contains(args.hash) &&
_mgr->_files[args.hash].contains(args.offset)) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:427:** +1
```cpp
if (_mgr->_files.contains(args.hash) &&
_mgr->_files[args.hash].contains(args.offset)) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:431:** +4, including nesting
penalty of 3, nesting level increased to 4
```cpp
if (file_block->expiration_time() !=
args.ctx.expiration_time ||
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:431:** +1
```cpp
if (file_block->expiration_time() !=
args.ctx.expiration_time ||
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:445:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (!args.is_tmp) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:452:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (ec) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:460:** nesting level increased
to 1
```cpp
auto scan_file_cache = [&](std::filesystem::directory_iterator& key_it) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:462:** +2, including nesting
penalty of 1, nesting level increased to 2
```cpp
for (; key_it != std::filesystem::directory_iterator(); ++key_it) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:471:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (ec) [[unlikely]] {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:480:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
for (; offset_it != std::filesystem::directory_iterator();
++offset_it) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:485:** +4, including nesting
penalty of 3, nesting level increased to 4
```cpp
if (!parse_filename_suffix_to_cache_type(fs,
offset_it->path().filename().native(),
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:502:** +4, including nesting
penalty of 3, nesting level increased to 4
```cpp
if (batch_load_buffer.size() >= scan_length) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:510:** +1, including nesting
penalty of 0, nesting level increased to 1
```cpp
if constexpr (USE_CACHE_VERSION2) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:512:** +2, including nesting
penalty of 1, nesting level increased to 2
```cpp
if (ec) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:516:** +2, including nesting
penalty of 1, nesting level increased to 2
```cpp
for (; key_prefix_it != std::filesystem::directory_iterator();
++key_prefix_it) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:517:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (!key_prefix_it->is_directory()) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:521:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (key_prefix_it->path().filename().native().size() !=
KEY_PREFIX_LENGTH) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:526:** +4, including nesting
penalty of 3, nesting level increased to 4
```cpp
if (ec) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:533:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (ec) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:539:** +1, nesting level
increased to 1
```cpp
} else {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:541:** +2, including nesting
penalty of 1, nesting level increased to 2
```cpp
if (ec) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:547:** +1, including nesting
penalty of 0, nesting level increased to 1
```cpp
if (!batch_load_buffer.empty()) {
^
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
