This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 35a1404bbe [fix](load) add error handle when load data dir (#23457)
35a1404bbe is described below
commit 35a1404bbe0aed3f5163e766dce3a3725cc828d6
Author: HHoflittlefish777 <[email protected]>
AuthorDate: Mon Aug 28 19:33:50 2023 +0800
[fix](load) add error handle when load data dir (#23457)
---
be/src/olap/storage_engine.cpp | 30 +++++++++++++++++++-----------
be/src/olap/storage_engine.h | 8 ++++----
2 files changed, 23 insertions(+), 15 deletions(-)
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index 7956d41cd2..0d1dbd6fb7 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -163,21 +163,29 @@ StorageEngine::~StorageEngine() {
_s_instance = nullptr;
}
-void StorageEngine::load_data_dirs(const std::vector<DataDir*>& data_dirs) {
+Status StorageEngine::load_data_dirs(const std::vector<DataDir*>& data_dirs) {
std::vector<std::thread> threads;
- for (auto data_dir : data_dirs) {
- threads.emplace_back([data_dir] {
- auto res = data_dir->load();
- if (!res.ok()) {
- LOG(WARNING) << "io error when init load tables. res=" << res
- << ", data dir=" << data_dir->path();
- // TODO(lingbin): why not exit progress, to force OP to change
the conf
- }
- });
+ std::vector<Status> results(data_dirs.size());
+ for (size_t i = 0; i < data_dirs.size(); ++i) {
+ threads.emplace_back(
+ [&results, data_dir = data_dirs[i]](size_t index) {
+ results[index] = data_dir->load();
+ if (!results[index].ok()) {
+ LOG(WARNING) << "io error when init load tables. res="
<< results[index]
+ << ", data dir=" << data_dir->path();
+ }
+ },
+ i);
}
for (auto& thread : threads) {
thread.join();
}
+ for (const auto& result : results) {
+ if (!result.ok()) {
+ return result;
+ }
+ }
+ return Status::OK();
}
Status StorageEngine::_open() {
@@ -192,7 +200,7 @@ Status StorageEngine::_open() {
RETURN_NOT_OK_STATUS_WITH_WARN(_check_file_descriptor_number(), "check fd
number failed");
auto dirs = get_stores<false>();
- load_data_dirs(dirs);
+ RETURN_IF_ERROR(load_data_dirs(dirs));
_memtable_flush_executor.reset(new MemTableFlushExecutor());
_memtable_flush_executor->init(dirs);
diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h
index 9ab23463f2..a6ede08162 100644
--- a/be/src/olap/storage_engine.h
+++ b/be/src/olap/storage_engine.h
@@ -91,14 +91,14 @@ public:
void clear_transaction_task(const TTransactionId transaction_id,
const std::vector<TPartitionId>&
partition_ids);
- // Note: 这里只能reload原先已经存在的root path，即re-load启动时就登记的root path
- // 是允许的，但re-load全新的path是不允许的，因为此处没有彻底更新ce调度器信息
- void load_data_dirs(const std::vector<DataDir*>& stores);
+ // Note: Only the previously existing root path can be reloaded here, that
is, the root path registered when re load starts is allowed,
+ // but the brand new path of re load is not allowed because the ce
scheduler information has not been thoroughly updated here
+ Status load_data_dirs(const std::vector<DataDir*>& stores);
template <bool include_unused = false>
std::vector<DataDir*> get_stores();
- // @brief 获取所有root_path信息
+ // get all info of root_path
Status get_all_data_dir_info(std::vector<DataDirInfo>* data_dir_infos,
bool need_update);
int64_t get_file_or_directory_size(const std::string& file_path);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
