This ticket revisit the waiting for peer info and
fix the problem of disordered peer_up and peer info
in the commit d1593b03b3c9bec292b14dde65264c261760bf46
---
src/rde/rded/rde_main.cc | 1 +
src/rde/rded/role.cc | 63 +++++++++++++++++++++++++++++++++++++++-
src/rde/rded/role.h | 7 +++++
3 files changed, 70 insertions(+), 1 deletion(-)
diff --git a/src/rde/rded/rde_main.cc b/src/rde/rded/rde_main.cc
index 8ed6b046e..33dd645e2 100644
--- a/src/rde/rded/rde_main.cc
+++ b/src/rde/rded/rde_main.cc
@@ -125,6 +125,7 @@ static void handle_mbx_event() {
}
case RDE_MSG_PEER_DOWN:
LOG_NO("Peer down on node 0x%x", msg->fr_node_id);
+ role->RemovePeer(msg->fr_node_id);
break;
case RDE_MSG_NEW_ACTIVE_CALLBACK: {
const std::string my_node = base::Conf::NodeName();
diff --git a/src/rde/rded/role.cc b/src/rde/rded/role.cc
index 3732be449..344702e63 100644
--- a/src/rde/rded/role.cc
+++ b/src/rde/rded/role.cc
@@ -196,9 +196,13 @@ Role::Role(NODE_ID own_node_id)
discover_peer_timeout_{base::GetEnv("RDE_DISCOVER_PEER_TIMEOUT",
kDefaultDiscoverPeerTimeout)},
pre_active_script_timeout_{base::GetEnv(
- "RDE_PRE_ACTIVE_SCRIPT_TIMEOUT", kDefaultPreActiveScriptTimeout)} {}
+ "RDE_PRE_ACTIVE_SCRIPT_TIMEOUT", kDefaultPreActiveScriptTimeout)},
+ received_peer_info_{true},
+ peer_info_wait_time_{},
+ peer_info_wait_timeout_ {kDefaultWaitPeerInfoTimeout} {}
timespec* Role::Poll(timespec* ts) {
+ TRACE_ENTER();
timespec* timeout = nullptr;
if (role_ == PCS_RDA_UNDEFINED) {
timespec now = base::ReadMonotonicClock();
@@ -238,6 +242,25 @@ timespec* Role::Poll(timespec* ts) {
cb->state_refresh_thread_started = true;
std::thread(&Role::RefreshConsensusState, this, cb).detach();
}
+ if (consensus_service.IsEnabled() == false) {
+ // We are already ACTIVE, and has just discovered a new node
+ // which makes the election_end_time_ reset
+ if (received_peer_info_ == false) {
+ timespec now = base::ReadMonotonicClock();
+ if (peer_info_wait_time_ >= now) {
+ *ts = peer_info_wait_time_ - now;
+ timeout = ts;
+ } else {
+ // Timeout but haven't received peer info
+ // The peer RDE could be in ACTIVE
+ // thus self-fence to avoid split-brain risk
+ LOG_ER("Discovery peer up without peer info. Risk in split-brain,"
+ "rebooting this node");
+ opensaf_quick_reboot("Probable split-brain due to "
+ "unknown RDE peer info");
+ }
+ }
+ }
}
}
return timeout;
@@ -251,12 +274,25 @@ void Role::ExecutePreActiveScript() {
}
void Role::AddPeer(NODE_ID node_id) {
+ TRACE_ENTER();
auto result = known_nodes_.insert(node_id);
if (result.second) {
ResetElectionTimer();
+ if (role_ == PCS_RDA_ACTIVE) {
+ ResetPeerInfoWaitTimer();
+ received_peer_info_ = false;
+ }
}
}
+void Role::RemovePeer(NODE_ID node_id) {
+ TRACE_ENTER();
+ if (received_peer_info_ == false && role_ != PCS_RDA_ACTIVE) {
+ StopPeerInfoWaitTimer();
+ }
+ known_nodes_.erase(node_id);
+}
+
// call from main thread only
bool Role::IsCandidate() {
TRACE_ENTER();
@@ -330,10 +366,24 @@ uint32_t Role::SetRole(PCS_RDA_ROLE new_role) {
}
void Role::ResetElectionTimer() {
+ TRACE_ENTER();
election_end_time_ = base::ReadMonotonicClock() +
base::MillisToTimespec(discover_peer_timeout_);
}
+void Role::ResetPeerInfoWaitTimer() {
+ TRACE_ENTER();
+ LOG_NO("Start/restart waiting peer info timer");
+ peer_info_wait_time_ = base::ReadMonotonicClock() +
+ base::MillisToTimespec(peer_info_wait_timeout_);
+}
+
+void Role::StopPeerInfoWaitTimer() {
+ TRACE_ENTER();
+ // Turn off peer_info_timer
+ received_peer_info_ = true;
+}
+
uint32_t Role::UpdateMdsRegistration(PCS_RDA_ROLE new_role,
PCS_RDA_ROLE old_role) {
uint32_t rc = NCSCC_RC_SUCCESS;
@@ -357,6 +407,7 @@ uint32_t Role::UpdateMdsRegistration(PCS_RDA_ROLE new_role,
void Role::SetPeerState(PCS_RDA_ROLE node_role, NODE_ID node_id,
uint64_t peer_promote_pending) {
+ TRACE_ENTER();
if (role() == PCS_RDA_UNDEFINED) {
bool give_up = false;
RDE_CONTROL_BLOCK *cb = rde_get_control_block();
@@ -372,6 +423,14 @@ void Role::SetPeerState(PCS_RDA_ROLE node_role, NODE_ID
node_id,
}
if (node_role == PCS_RDA_ACTIVE || node_role == PCS_RDA_STANDBY ||
give_up) {
+ // broadcast QUIESCED role to all peers to stop their waiting peer
+ // info timer
+ rde_msg peer_info_req;
+ peer_info_req.type = RDE_MSG_PEER_INFO_RESP;
+ peer_info_req.info.peer_info.ha_role = PCS_RDA_QUIESCED;
+ peer_info_req.info.peer_info.promote_pending = 0;
+ rde_mds_broadcast(&peer_info_req);
+
SetRole(PCS_RDA_QUIESCED);
LOG_NO("Giving up election against 0x%" PRIx32
" with role %s. "
@@ -379,6 +438,8 @@ void Role::SetPeerState(PCS_RDA_ROLE node_role, NODE_ID
node_id,
node_id, to_string(node_role), to_string(role()));
}
}
+ known_nodes_.insert(node_id);
+ StopPeerInfoWaitTimer();
}
void Role::PromoteNodeLate() {
diff --git a/src/rde/rded/role.h b/src/rde/rded/role.h
index 2d24361c5..218897892 100644
--- a/src/rde/rded/role.h
+++ b/src/rde/rded/role.h
@@ -50,12 +50,16 @@ class Role {
void NodePromoted();
void PromoteNodeLate();
void RefreshConsensusState(RDE_CONTROL_BLOCK* cb);
+ void RemovePeer(NODE_ID node_id);
private:
static const uint64_t kDefaultDiscoverPeerTimeout = 2000;
+ static const uint64_t kDefaultWaitPeerInfoTimeout = 2000;
static const uint64_t kDefaultPreActiveScriptTimeout = 5000;
void ExecutePreActiveScript();
void ResetElectionTimer();
+ void ResetPeerInfoWaitTimer();
+ void StopPeerInfoWaitTimer();
uint32_t UpdateMdsRegistration(PCS_RDA_ROLE new_role, PCS_RDA_ROLE old_role);
void PromoteNode(const uint64_t cluster_size, const bool relaxed_mode);
@@ -68,6 +72,9 @@ class Role {
uint64_t pre_active_script_timeout_;
static const char* const role_names_[];
static const char* const pre_active_script_;
+ bool received_peer_info_;
+ timespec peer_info_wait_time_;
+ uint64_t peer_info_wait_timeout_;
DELETE_COPY_AND_MOVE_OPERATORS(Role);
};
--
2.20.1
_______________________________________________
Opensaf-devel mailing list
Opensaf-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/opensaf-devel
