This is an automated email from the ASF dual-hosted git repository.
liaoxin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2d8a86286b7 [improve](move-memtable) improve error log and message for
"not enough streams" (#47470)
2d8a86286b7 is described below
commit 2d8a86286b72c59bb1ddea331074d360bfcfb8e7
Author: Kaijie Chen <[email protected]>
AuthorDate: Mon Feb 10 21:09:04 2025 +0800
[improve](move-memtable) improve error log and message for "not enough
streams" (#47470)
improve error log and message for "not enough streams", make it easier
to understand.
---
be/src/vec/sink/writer/vtablet_writer_v2.cpp | 21 +++++++++++++++++++--
.../test_multi_replica_fault_injection.groovy | 2 +-
2 files changed, 20 insertions(+), 3 deletions(-)
diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.cpp
b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
index c332e3bf2f2..fa57b9b18fb 100644
--- a/be/src/vec/sink/writer/vtablet_writer_v2.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
@@ -379,6 +379,7 @@ void
VTabletWriterV2::_generate_rows_for_tablet(std::vector<RowPartTabletIds>& r
Status VTabletWriterV2::_select_streams(int64_t tablet_id, int64_t
partition_id, int64_t index_id,
std::vector<std::shared_ptr<LoadStreamStub>>& streams) {
+ std::vector<int64_t> failed_node_ids;
const auto* location = _location->find_tablet(tablet_id);
DBUG_EXECUTE_IF("VTabletWriterV2._select_streams.location_null", {
location = nullptr; });
if (location == nullptr) {
@@ -398,6 +399,9 @@ Status VTabletWriterV2::_select_streams(int64_t tablet_id,
int64_t partition_id,
<< ", stream_ok=" << (stream == nullptr ? "no" : "yes");
});
if (stream == nullptr) {
+ LOG(WARNING) << "skip writing tablet " << tablet_id << " to
backend " << node_id
+ << ": stream is not open";
+ failed_node_ids.push_back(node_id);
continue;
}
streams.emplace_back(std::move(stream));
@@ -408,8 +412,21 @@ Status VTabletWriterV2::_select_streams(int64_t tablet_id,
int64_t partition_id,
<< ", num_nodes=" << location->node_ids.size();
});
if (streams.size() <= location->node_ids.size() / 2) {
- return Status::InternalError("not enough streams {}/{}",
streams.size(),
- location->node_ids.size());
+ std::ostringstream success_msg;
+ std::ostringstream failed_msg;
+ for (auto& s : streams) {
+ success_msg << ", " << s->dst_id();
+ }
+ for (auto id : failed_node_ids) {
+ failed_msg << ", " << id;
+ }
+ LOG(INFO) << "failed to write enough replicas " << streams.size() <<
"/"
+ << location->node_ids.size() << " for tablet " << tablet_id
+ << " due to connection errors; success nodes" <<
success_msg.str()
+ << "; failed nodes" << failed_msg.str() << ".";
+ return Status::InternalError(
+ "failed to write enough replicas {}/{} for tablet {} due to
connection errors",
+ streams.size(), location->node_ids.size(), tablet_id);
}
Status st;
for (auto& stream : streams) {
diff --git
a/regression-test/suites/fault_injection_p0/test_multi_replica_fault_injection.groovy
b/regression-test/suites/fault_injection_p0/test_multi_replica_fault_injection.groovy
index 4c01513c612..072d5bf1c93 100644
---
a/regression-test/suites/fault_injection_p0/test_multi_replica_fault_injection.groovy
+++
b/regression-test/suites/fault_injection_p0/test_multi_replica_fault_injection.groovy
@@ -102,7 +102,7 @@ suite("test_multi_replica_fault_injection",
"nonConcurrent") {
// test one backend open failure
load_with_injection("VTabletWriterV2._open_streams.skip_one_backend",
"success", true)
// test two backend open failure
- load_with_injection("VTabletWriterV2._open_streams.skip_two_backends",
"not enough streams 1/3", false, "succ replica num 1 < load required replica
num 2")
+ load_with_injection("VTabletWriterV2._open_streams.skip_two_backends",
"failed to write enough replicas 1/3 for tablet", false, "succ replica num 1 <
load required replica num 2")
sql """ set enable_memtable_on_sink_node=false """
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
