This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 53705d110b289e068873c983a88ddb1210a593b1 Author: HHoflittlefish777 <[email protected]> AuthorDate: Mon Aug 28 19:33:50 2023 +0800 [fix](load) add error handle when load data dir (#23457) --- be/src/olap/storage_engine.cpp | 30 +++++++++++++++++++----------- be/src/olap/storage_engine.h | 8 ++++---- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index f57ac9295b..188b290e82 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -164,21 +164,29 @@ StorageEngine::~StorageEngine() { _s_instance = nullptr; } -void StorageEngine::load_data_dirs(const std::vector<DataDir*>& data_dirs) { +Status StorageEngine::load_data_dirs(const std::vector<DataDir*>& data_dirs) { std::vector<std::thread> threads; - for (auto data_dir : data_dirs) { - threads.emplace_back([data_dir] { - auto res = data_dir->load(); - if (!res.ok()) { - LOG(WARNING) << "io error when init load tables. res=" << res - << ", data dir=" << data_dir->path(); - // TODO(lingbin): why not exit progress, to force OP to change the conf - } - }); + std::vector<Status> results(data_dirs.size()); + for (size_t i = 0; i < data_dirs.size(); ++i) { + threads.emplace_back( + [&results, data_dir = data_dirs[i]](size_t index) { + results[index] = data_dir->load(); + if (!results[index].ok()) { + LOG(WARNING) << "io error when init load tables. res=" << results[index] + << ", data dir=" << data_dir->path(); + } + }, + i); } for (auto& thread : threads) { thread.join(); } + for (const auto& result : results) { + if (!result.ok()) { + return result; + } + } + return Status::OK(); } Status StorageEngine::_open() { @@ -193,7 +201,7 @@ Status StorageEngine::_open() { RETURN_NOT_OK_STATUS_WITH_WARN(_check_file_descriptor_number(), "check fd number failed"); auto dirs = get_stores<false>(); - load_data_dirs(dirs); + RETURN_IF_ERROR(load_data_dirs(dirs)); _memtable_flush_executor.reset(new MemTableFlushExecutor()); _memtable_flush_executor->init(dirs); diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index 19a5d6b7db..ec80fd9144 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -91,14 +91,14 @@ public: void clear_transaction_task(const TTransactionId transaction_id, const std::vector<TPartitionId>& partition_ids); - // Note: 这里只能reload原先已经存在的root path，即re-load启动时就登记的root path - // 是允许的，但re-load全新的path是不允许的，因为此处没有彻底更新ce调度器信息 - void load_data_dirs(const std::vector<DataDir*>& stores); + // Note: Only the previously existing root path can be reloaded here, that is, the root path registered when re load starts is allowed, + // but the brand new path of re load is not allowed because the ce scheduler information has not been thoroughly updated here + Status load_data_dirs(const std::vector<DataDir*>& stores); template <bool include_unused = false> std::vector<DataDir*> get_stores(); - // @brief 获取所有root_path信息 + // get all info of root_path Status get_all_data_dir_info(std::vector<DataDirInfo>* data_dir_infos, bool need_update); int64_t get_file_or_directory_size(const std::string& file_path); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
