[devel] [PATCH 1/1] amfnd: fix unexpected reboot after headless due to split-brain [#3162]

[thuan.tran]

- Split-brain recovery in headless enable, IMMND may expected restart.
If AMFND not wait IMMND restart but reinit CLM, CLM callback trigger,
clm_to_amf_node() is called then AMFND stuck in init IMM OM causes delay
restart IMMND, delay resend node_up then AMFD will order reboot node.
- Slow down this reinit CLM help avoid node unexpected reboot order.
---
 src/amf/amfnd/avnd_cb.h | 1 +
 src/amf/amfnd/clc.cc    | 7 +++++++
 src/amf/amfnd/clm.cc    | 9 +++++++++
 src/amf/amfnd/main.cc   | 1 +
 4 files changed, 18 insertions(+)

diff --git a/src/amf/amfnd/avnd_cb.h b/src/amf/amfnd/avnd_cb.h
index 8b0cc2304..0fa0590ff 100644
--- a/src/amf/amfnd/avnd_cb.h
+++ b/src/amf/amfnd/avnd_cb.h
@@ -125,6 +125,7 @@ typedef struct avnd_cb_tag {
   SaTimeT scs_absence_max_duration;
   /* the timer for supervision of the absence of SC */
   AVND_TMR sc_absence_tmr;
+  bool immnd_down;
 } AVND_CB;
 
 #define AVND_CB_NULL ((AVND_CB *)0)
diff --git a/src/amf/amfnd/clc.cc b/src/amf/amfnd/clc.cc
index f78e1a707..e8e309051 100644
--- a/src/amf/amfnd/clc.cc
+++ b/src/amf/amfnd/clc.cc
@@ -3333,6 +3333,13 @@ uint32_t avnd_comp_clc_cmd_execute(AVND_CB *cb, 
AVND_COMP *comp,
     // outcome of command is reported in comp_clc_resp_callback()
   }
 
+  if (comp->name.find("safComp=IMMND,") != std::string::npos) {
+    if (cmd_type == AVND_COMP_CLC_CMD_TYPE_CLEANUP)
+      cb->immnd_down = true;
+    else if (cmd_type == AVND_COMP_CLC_CMD_TYPE_INSTANTIATE)
+      cb->immnd_down = false;
+  }
+
   TRACE_2("success");
   goto done;
 
diff --git a/src/amf/amfnd/clm.cc b/src/amf/amfnd/clm.cc
index 06eb229c7..01151b2a7 100644
--- a/src/amf/amfnd/clm.cc
+++ b/src/amf/amfnd/clm.cc
@@ -312,6 +312,15 @@ done:
 static void *avnd_clm_init_thread(void *arg) {
   TRACE_ENTER();
   AVND_CB *cb = static_cast<AVND_CB *>(arg);
+  int count = 0;
+
+  while (cb->immnd_down && count++ < 100)
+    osaf_nanosleep(&kHundredMilliseconds);
+    /* Split-brain recovery in headless enable, IMMND may expected restart.
+     * If AMFND not wait IMMND restart but reinit CLM, CLM callback trigger,
+     * clm_to_amf_node() is called then AMFND stuck in init IMM OM causes delay
+     * restart IMMND, delay resend node_up then AMFD will order reboot node.
+     * Slow down this reinit CLM help avoid node unexpected reboot order */
 
   avnd_clm_init(cb);
 
diff --git a/src/amf/amfnd/main.cc b/src/amf/amfnd/main.cc
index d7857fabe..cef0543e6 100644
--- a/src/amf/amfnd/main.cc
+++ b/src/amf/amfnd/main.cc
@@ -334,6 +334,7 @@ AVND_CB *avnd_cb_create() {
 
   cb->is_avd_down = true;
   cb->amfd_sync_required = false;
+  cb->immnd_down = false;
 
   // retrieve hydra configuration from IMM
   hydra_config_get(cb);
-- 
2.17.1



_______________________________________________
Opensaf-devel mailing list
Opensaf-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/opensaf-devel