This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 44b432cad24 branch-3.0: [fix](binlog) Fix linking binlog failure due
to existing files #45445 (#45512)
44b432cad24 is described below
commit 44b432cad249b8bc0c731722931495be8e4c8085
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Dec 17 19:53:11 2024 +0800
branch-3.0: [fix](binlog) Fix linking binlog failure due to existing files
#45445 (#45512)
Cherry-picked from #45445
Co-authored-by: walter <[email protected]>
---
be/src/common/status.h | 2 +-
be/src/olap/rowset/beta_rowset.cpp | 32 ++++++++++++++++++++++----------
be/src/olap/txn_manager.cpp | 5 +++--
be/test/olap/rowset/beta_rowset_test.cpp | 10 ++++++++++
4 files changed, 36 insertions(+), 13 deletions(-)
diff --git a/be/src/common/status.h b/be/src/common/status.h
index 344f82a81b8..d059f289402 100644
--- a/be/src/common/status.h
+++ b/be/src/common/status.h
@@ -570,7 +570,7 @@ private:
// and another thread is call to_string method, it may core, because the
_err_msg is an unique ptr and
// it is deconstructed during copy method.
// And also we could not use lock, because we need get status frequently to
check if it is cancelled.
-// The defaule value is ok.
+// The default value is ok.
class AtomicStatus {
public:
AtomicStatus() : error_st_(Status::OK()) {}
diff --git a/be/src/olap/rowset/beta_rowset.cpp
b/be/src/olap/rowset/beta_rowset.cpp
index bbb2ca72b4a..70afe21065d 100644
--- a/be/src/olap/rowset/beta_rowset.cpp
+++ b/be/src/olap/rowset/beta_rowset.cpp
@@ -557,10 +557,6 @@ Status BetaRowset::add_to_binlog() {
}
const auto& fs = io::global_local_filesystem();
-
- // all segments are in the same directory, so cache binlog_dir without
multi times check
- std::string binlog_dir;
-
auto segments_num = num_segments();
VLOG_DEBUG << fmt::format("add rowset to binlog. rowset_id={},
segments_num={}",
rowset_id().to_string(), segments_num);
@@ -569,28 +565,34 @@ Status BetaRowset::add_to_binlog() {
std::vector<string> linked_success_files;
Defer remove_linked_files {[&]() { // clear linked files if errors happen
if (!status.ok()) {
- LOG(WARNING) << "will delete linked success files due to error "
<< status;
+ LOG(WARNING) << "will delete linked success files due to error "
+ << status.to_string_no_stack();
std::vector<io::Path> paths;
for (auto& file : linked_success_files) {
paths.emplace_back(file);
LOG(WARNING) << "will delete linked success file " << file <<
" due to error";
}
static_cast<void>(fs->batch_delete(paths));
- LOG(WARNING) << "done delete linked success files due to error "
<< status;
+ LOG(WARNING) << "done delete linked success files due to error "
+ << status.to_string_no_stack();
}
}};
+ // all segments are in the same directory, so cache binlog_dir without
multi times check
+ std::string binlog_dir;
for (int i = 0; i < segments_num; ++i) {
auto seg_file = local_segment_path(_tablet_path,
rowset_id().to_string(), i);
if (binlog_dir.empty()) {
binlog_dir =
std::filesystem::path(seg_file).parent_path().append("_binlog").string();
+ // Delete all existing files in binlog dir, to keep binlog dir
clean.
bool exists = true;
RETURN_IF_ERROR(fs->exists(binlog_dir, &exists));
- if (!exists) {
- RETURN_IF_ERROR(fs->create_directory(binlog_dir));
+ if (exists) {
+ RETURN_IF_ERROR(fs->delete_directory(binlog_dir));
}
+ RETURN_IF_ERROR(fs->create_directory(binlog_dir));
}
auto binlog_file =
@@ -614,7 +616,12 @@ Status BetaRowset::add_to_binlog() {
std::filesystem::path(index_file).filename())
.string();
VLOG_DEBUG << "link " << index_file << " to " <<
binlog_index_file;
- RETURN_IF_ERROR(fs->link_file(index_file, binlog_index_file));
+ if (!fs->link_file(index_file, binlog_index_file).ok()) {
+ status = Status::Error<OS_ERROR>(
+ "fail to create hard link. from={}, to={},
errno={}", index_file,
+ binlog_index_file, Errno::no());
+ return status;
+ }
linked_success_files.push_back(binlog_index_file);
}
} else {
@@ -625,7 +632,12 @@ Status BetaRowset::add_to_binlog() {
std::filesystem::path(index_file).filename())
.string();
VLOG_DEBUG << "link " << index_file << " to " <<
binlog_index_file;
- RETURN_IF_ERROR(fs->link_file(index_file, binlog_index_file));
+ if (!fs->link_file(index_file, binlog_index_file).ok()) {
+ status = Status::Error<OS_ERROR>(
+ "fail to create hard link. from={}, to={},
errno={}", index_file,
+ binlog_index_file, Errno::no());
+ return status;
+ }
linked_success_files.push_back(binlog_index_file);
}
}
diff --git a/be/src/olap/txn_manager.cpp b/be/src/olap/txn_manager.cpp
index 54436668c85..be4ad0c166e 100644
--- a/be/src/olap/txn_manager.cpp
+++ b/be/src/olap/txn_manager.cpp
@@ -548,8 +548,9 @@ Status TxnManager::publish_txn(OlapMeta* meta, TPartitionId
partition_id,
if (!status.ok()) {
return Status::Error<ROWSET_ADD_TO_BINLOG_FAILED>(
"add rowset to binlog failed. when publish txn rowset_id:
{}, tablet id: {}, "
- "txn id: {}",
- rowset->rowset_id().to_string(), tablet_id,
transaction_id);
+ "txn id: {}, status: {}",
+ rowset->rowset_id().to_string(), tablet_id, transaction_id,
+ status.to_string_no_stack());
}
}
diff --git a/be/test/olap/rowset/beta_rowset_test.cpp
b/be/test/olap/rowset/beta_rowset_test.cpp
index 1ed3a9ed04b..0c3001758f0 100644
--- a/be/test/olap/rowset/beta_rowset_test.cpp
+++ b/be/test/olap/rowset/beta_rowset_test.cpp
@@ -292,4 +292,14 @@ TEST_F(BetaRowsetTest, ReadTest) {
}
}
+TEST_F(BetaRowsetTest, AddToBinlogTest) {
+ RowsetMetaSharedPtr rowset_meta = std::make_shared<RowsetMeta>();
+ BetaRowset rowset(nullptr, rowset_meta, "");
+ std::string resource_id = "10000";
+ Status s = rowset.add_to_binlog();
+ ASSERT_TRUE(s.ok()) << "first add_to_binlog(): " << s;
+ s = rowset.add_to_binlog();
+ ASSERT_TRUE(s.ok()) << "second add_to_binlog(): " << s;
+}
+
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
