Ack. -----Original Message----- From: Hoang Minh Vu <hoang.m...@dektech.com.au> Sent: Thursday, April 28, 2022 5:30 AM To: Thien Minh Huynh <thien.m.hu...@dektech.com.au>; Thang Duc Nguyen <thang.d.ngu...@dektech.com.au> Cc: opensaf-devel@lists.sourceforge.net; Hoang Minh Vu <hoang.m...@dektech.com.au> Subject: [PATCH 1/1] imm: fix PBE logging inconsistent [#3311]
There are some logging syslog for PBE but they are inconsistent make it hard to troubleshooting E.g, - LOG_ER("IMM RELOAD with NO persistent back end => ensure cluster restart by IMMD exit at both SCs, exiting"); - TRACE_5("ERR_TRY_AGAIN: Persistent back end is down"); - TRACE("Persistent Back-End OI %s is attaching", Solution is to make all of them to Persistent Back End --- README | 2 +- configure.ac | 6 +++--- src/imm/README | 2 +- src/imm/agent/imma_om_api.cc | 2 +- src/imm/immd/immd_proc.c | 4 ++-- src/imm/immnd/ImmModel.cc | 30 +++++++++++++++--------------- src/imm/immnd/immnd.conf | 4 ++-- src/imm/immnd/immnd_cb.h | 4 ++-- src/imm/immnd/immnd_evt.c | 28 ++++++++++++++-------------- src/imm/immnd/immnd_main.c | 2 +- src/imm/immnd/immnd_proc.c | 14 +++++++------- src/smf/smfd/SmfUpgradeCampaign.h | 4 ++-- 12 files changed, 51 insertions(+), 51 deletions(-) diff --git a/README b/README index 158aee47e..622bf44dd 100644 --- a/README +++ b/README @@ -542,7 +542,7 @@ available w.r.t enabling/disabling the build for a particular OpenSAF service: --enable-tipc enable building the TIPC [default=no] --enable-tests enable building the OpenSAF testing programs [default=no] - --enable-imm-pbe enable the IMM Persistent Backend Interface + --enable-imm-pbe enable the IMM Persistent Back End Interface [default=yes] --enable-ntf-imcn enable the NTF Information Model Notification producer [default=no] diff --git a/configure.ac b/configure.ac index 7e35da2c4..857669371 100644 --- a/configure.ac +++ b/configure.ac @@ -330,12 +330,12 @@ AM_CONDITIONAL([ENABLE_TESTS], [test "$enable_tests" = yes]) AC_SUBST([TESTS_ENABLED], ["$enable_tests"]) # -# Enable/disable IMM Persistent Backend +# Enable/disable IMM Persistent Back End # -AC_MSG_CHECKING([whether to use the IMM persistent backend]) +AC_MSG_CHECKING([whether to use the IMM Persistent Back End]) AC_ARG_ENABLE([imm-pbe], [AS_HELP_STRING([--enable-imm-pbe], - [enable the IMM Persistent Backend Interface [default=no]])], + [enable the IMM Persistent Back End Interface [default=no]])], [], [enable_imm_pbe=no]) AC_MSG_RESULT([$enable_imm_pbe]) diff --git a/src/imm/README b/src/imm/README index 11b45bb9e..4fa62a25f 100644 --- a/src/imm/README +++ b/src/imm/README @@ -427,7 +427,7 @@ such data. The configuration data and persistent runtime data will survive cluster restart, but only the latest dumped version will be recovered. One of the main new immsv features available in OpenSAF4.0 is the -Persistent Back-End (PBE). By default this feature is not enabled. +Persistent Back End (PBE). By default this feature is not enabled. See the instructions below for how to enable it. Use of the PBE has a signifigant (negative) impact on performance for CCBs and persistent runtime data, both response time and throughput. diff --git a/src/imm/agent/imma_om_api.cc b/src/imm/agent/imma_om_api.cc index d4914872d..30d562b2e 100644 --- a/src/imm/agent/imma_om_api.cc +++ b/src/imm/agent/imma_om_api.cc @@ -3676,7 +3676,7 @@ additional ccb-operations can be added to the ccb until the current set of successfully validated ccb is highly likely to succeed, but still not guaranteed to succeed. If an apply of a validated ccb fails, it will obviously not be due to any failure in validation, but for "physical" reasons, - such as some problem with the PBE (persistent back end), or a + such as some problem with the PBE (Persistent Back + End), or a crash of some vital IMM process. Thus after a successfull saImmOmCcbValidate, the ccb is in a diff --git a/src/imm/immd/immd_proc.c b/src/imm/immd/immd_proc.c index c59c666ef..a3a2c2c65 100644 --- a/src/imm/immd/immd_proc.c +++ b/src/imm/immd/immd_proc.c @@ -119,10 +119,10 @@ void immd_proc_immd_reset(IMMD_CB *cb, bool active) // sleep(3); //} LOG_ER( - "IMM RELOAD with NO persistent back end => ensure cluster restart by IMMD exit at both SCs, exiting"); + "IMM RELOAD with NO Persistent Back End => ensure cluster restart +by IMMD exit at both SCs, exiting"); exit(1); } else { - /*LOG_WA("IMM RELOAD with persistent back end => No need to + /*LOG_WA("IMM RELOAD with Persistent Back End => No need to * restart cluster");*/ LOG_ER( "IMM RELOAD => ensure cluster restart by IMMD exit at both SCs, exiting"); diff --git a/src/imm/immnd/ImmModel.cc b/src/imm/immnd/ImmModel.cc index e10335ae1..4ce43d848 100644 --- a/src/imm/immnd/ImmModel.cc +++ b/src/imm/immnd/ImmModel.cc @@ -334,7 +334,7 @@ typedef enum { 7, // Explicit validate has completed (saImmOmCcbValidate only) IMM_CCB_PREPARE = 8, // Waiting for nodes prepare & completed calls/replies IMM_CCB_CRITICAL = 9, // Unilateral abort no longer allowed (except by PBE). - IMM_CCB_PBE_ABORT = 10, // The Persistent back end replied with abort + IMM_CCB_PBE_ABORT = 10, // The Persistent Back End replied with + abort IMM_CCB_COMMITTED = 11, // Committed at nodes pending implementer apply calls IMM_CCB_ABORTED = 12, // READY->ABORTED PREPARE->ABORTED IMM_CCB_ILLEGAL = 13 // CCB has been removed. @@ -2383,7 +2383,7 @@ bool ImmModel::immNotWritable() { /* immNotPbeWritable returning true means: (1) immNotWriteable is true OR... (2) immNotWritable is false (imm service is writable), but according to - configuration there should be a persistent back-end (Pbe) and the Pbe is + configuration there should be a Persistent Back End (Pbe) and the + Pbe is currently not operational. OR.. (3) PBE is operational, but backlog on PRTOs or Ccbs is large enough to warant back-presure (TRY_AGAIN) towards the application. @@ -8771,7 +8771,7 @@ SaAisErrorT ImmModel::ccbObjectCreate( } } - // Prepare for call on PersistentBackEnd + // Prepare for call on Persistent Back End if ((err == SA_AIS_OK) && pbeNodeIdPtr) { void* pbe = getPbeOi(pbeConnPtr, pbeNodeIdPtr); if (!pbe) { @@ -8780,13 +8780,13 @@ SaAisErrorT ImmModel::ccbObjectCreate( err = SA_AIS_ERR_FAILED_OPERATION; ccb->mVeto = err; LOG_WA( - "ERR_FAILED_OPERATION: Persistent back end is down " + "ERR_FAILED_OPERATION: Persistent Back End is down " "ccb %u is aborted", ccbId); setCcbErrorString(ccb, IMM_RESOURCE_ABORT "PBE is down"); } else { /* Pristine ccb can not start because PBE down */ - TRACE_5("ERR_TRY_AGAIN: Persistent back end is down"); + TRACE_5("ERR_TRY_AGAIN: Persistent Back End is down"); err = SA_AIS_ERR_TRY_AGAIN; } } @@ -9905,7 +9905,7 @@ SaAisErrorT ImmModel::ccbObjectModify( } } // for (p = ....) - // Prepare for call on PersistentBackEnd + // Prepare for call on Persistent Back End if ((err == SA_AIS_OK) && pbeNodeIdPtr) { void* pbe = getPbeOi(pbeConnPtr, pbeNodeIdPtr); if (!pbe) { @@ -9914,13 +9914,13 @@ SaAisErrorT ImmModel::ccbObjectModify( err = SA_AIS_ERR_FAILED_OPERATION; ccb->mVeto = err; LOG_WA( - "ERR_FAILED_OPERATION: Persistent back end is down " + "ERR_FAILED_OPERATION: Persistent Back End is down " "ccb %u is aborted", ccbId); setCcbErrorString(ccb, IMM_RESOURCE_ABORT "PBE is down"); } else { /* Pristine ccb can not start because PBE down */ - TRACE_5("ERR_TRY_AGAIN: Persistent back end is down"); + TRACE_5("ERR_TRY_AGAIN: Persistent Back End is down"); err = SA_AIS_ERR_TRY_AGAIN; } } @@ -10544,7 +10544,7 @@ SaAisErrorT ImmModel::ccbObjectDelete( return SA_AIS_ERR_BAD_OPERATION; } - // Prepare for call on PersistentBackEnd + // Prepare for call on Persistent Back End if ((err == SA_AIS_OK) && pbeNodeIdPtr) { void* pbe = getPbeOi(pbeConnPtr, pbeNodeIdPtr); @@ -10554,13 +10554,13 @@ SaAisErrorT ImmModel::ccbObjectDelete( err = SA_AIS_ERR_FAILED_OPERATION; ccb->mVeto = err; LOG_WA( - "ERR_FAILED_OPERATION: Persistent back end is down " + "ERR_FAILED_OPERATION: Persistent Back End is down " "ccb %u is aborted", ccbId); setCcbErrorString(ccb, IMM_RESOURCE_ABORT "PBE is down"); } else { /* Pristine ccb can not start because PBE down */ - TRACE_5("ERR_TRY_AGAIN: Persistent back end is down"); + TRACE_5("ERR_TRY_AGAIN: Persistent Back End is down"); err = SA_AIS_ERR_TRY_AGAIN; } } else { @@ -11362,7 +11362,7 @@ bool ImmModel::ccbWaitForCompletedAck(SaUint32T ccbId, SaAisErrorT* err, ccb->mVeto = SA_AIS_ERR_FAILED_OPERATION; *err = ccb->mVeto; LOG_WA( - "ERR_FAILED_OPERATION: Persistent back end is down " + "ERR_FAILED_OPERATION: Persistent Back End is down " "ccb %u is aborted", ccbId); setCcbErrorString(ccb, IMM_RESOURCE_ABORT "PBE is down"); @@ -17059,7 +17059,7 @@ SaAisErrorT ImmModel::rtObjectCreate( } if (isPersistent) { - if (pbe) { /* Persistent back end is up (somewhere) */ + if (pbe) { /* Persistent Back End is up (somewhere) */ object->mObjFlags |= IMM_CREATE_LOCK; /* Dont overwrite IMM_DN_INTERNAL_REP*/ @@ -17930,7 +17930,7 @@ SaAisErrorT ImmModel::rtObjectUpdate( */ pbe = getPbeOi(pbeConnPtr, pbeNodeIdPtr); if (!pbe) { - LOG_WA("ERR_TRY_AGAIN: Persistent back end is down - unexpected here"); + LOG_WA("ERR_TRY_AGAIN: Persistent Back End is down - unexpected + here"); err = SA_AIS_ERR_TRY_AGAIN; goto rtObjectUpdateExit; /* We have already checked for PbeWritable with success inside @@ -18552,7 +18552,7 @@ SaAisErrorT ImmModel::rtObjectDelete( */ pbe = getPbeOi(pbeConnPtr, pbeNodeIdPtr); if (!pbe) { - LOG_NO("ERR_TRY_AGAIN: Persistent back end is down - unexpected here"); + LOG_NO("ERR_TRY_AGAIN: Persistent Back End is down - unexpected + here"); err = SA_AIS_ERR_TRY_AGAIN; goto rtObjectDeleteExit; /* We have already checked for PbeWritable with success inside diff --git a/src/imm/immnd/immnd.conf b/src/imm/immnd/immnd.conf index f9a809e16..b0f137683 100644 --- a/src/imm/immnd/immnd.conf +++ b/src/imm/immnd/immnd.conf @@ -33,10 +33,10 @@ export IMMSV_ROOT_DIRECTORY=$pkgsysconfdir export IMMSV_LOAD_FILE=imm.xml # The file name to be used by the "internal repository", also called -# persistent back-end. If the configuration attribute SaImmRepositoryInitMode +# Persistent Back End. If the configuration attribute +SaImmRepositoryInitMode # has the value SA_IMM_KEEP_REPOSITORY (1), then IMMSV_LOAD_FILE will ONLY be # used for first time cluster start. Subsequent restarts will try to load from -# the persistent back-end. The persistent back-end needs a file for storing +# the Persistent Back End. The Persistent Back End needs a file for +storing # committed CCBs and updates to persistent runtime attribute. # The IMMSV_PBE_FILE file is used for this. This file will also reside under # the IMMSV_ROOT_DIRECTORY. diff --git a/src/imm/immnd/immnd_cb.h b/src/imm/immnd/immnd_cb.h index 057d6fa5b..85e3663ca 100644 --- a/src/imm/immnd/immnd_cb.h +++ b/src/imm/immnd/immnd_cb.h @@ -61,7 +61,7 @@ typedef struct immnd_immom_client_node { NCSMDS_SVC_ID sv_id; /* OM or OI */ IMMND_OM_SEARCH_NODE *searchOpList; uint8_t mIsSync; /* Client is special sync client */ - uint8_t mIsPbe; /* Client is persistent back end */ + uint8_t mIsPbe; /* Client is Persistent Back End */ uint8_t mSyncBlocked; /* Sync client expects reply */ uint8_t mIsStale; /* Client disconnected when IMMD is unavailable => postpone @@ -168,7 +168,7 @@ typedef struct immnd_cb_tag { int32_t preLoadPid; // Forked preloader probes PBE file, for 2Pbe. int32_t loaderPid; int32_t syncPid; - int32_t pbePid; // Persistent back end (PBE) is running if pbePid > 0 + int32_t pbePid; // Persistent Back End (PBE) is running if pbePid > 0 int32_t pbePid2; // Redundant PBE (2PBE) is running if pbePid2 > 0 IMMND_SERVER_STATE mState; uint32_t mStep; // Measures progress in immnd_proc_server diff --git a/src/imm/immnd/immnd_evt.c b/src/imm/immnd/immnd_evt.c index af8ffae81..2578d7d7a 100644 --- a/src/imm/immnd/immnd_evt.c +++ b/src/imm/immnd/immnd_evt.c @@ -6395,11 +6395,11 @@ static void immnd_evt_proc_class_create(IMMND_CB *cb, IMMND_EVT *evt, &continuationId, &pbeConn, pbeNodeIdPtr); if (pbeNodeId && error == SA_AIS_OK) { - /*The persistent back-end is present => wait for reply. */ + /*The Persistent Back End is present => wait for reply. */ delayedReply = true; if (pbeConn) { const char *osafImmDn = OPENSAF_IMM_OBJECT_DN; - /*The persistent back-end is executing at THIS node. */ + /*The Persistent Back End is executing at THIS node. */ osafassert(cb->mIsCoord); osafassert(pbeNodeId); osafassert(pbeNodeId == cb->node_id); @@ -6557,11 +6557,11 @@ static void immnd_evt_proc_class_delete(IMMND_CB *cb, IMMND_EVT *evt, &continuationId, &pbeConn, pbeNodeIdPtr); if (pbeNodeId && error == SA_AIS_OK) { - /*The persistent back-end is present => wait for reply. */ + /*The Persistent Back End is present => wait for reply. */ delayedReply = true; if (pbeConn) { const char *osafImmDn = OPENSAF_IMM_OBJECT_DN; - /*The persistent back-end is executing at THIS node. */ + /*The Persistent Back End is executing at THIS node. */ osafassert(cb->mIsCoord); osafassert(pbeNodeId); osafassert(pbeNodeId == cb->node_id); @@ -6931,10 +6931,10 @@ static void immnd_evt_proc_rt_object_create(IMMND_CB *cb, IMMND_EVT *evt, } if (pbeNodeId && err == SA_AIS_OK) { - /*The persistent back-end is present => wait for reply. */ + /*The Persistent Back End is present => wait for reply. */ delayedReply = true; if (pbeConn) { - /*The persistent back-end is executing at THIS node. */ + /*The Persistent Back End is executing at THIS node. */ osafassert(!pbe2BConn); osafassert(cb->mIsCoord); osafassert(pbeNodeId == cb->node_id); @@ -7190,7 +7190,7 @@ static void immnd_evt_proc_object_create(IMMND_CB *cb, IMMND_EVT *evt, evt->type == IMMND_EVT_A2ND_OBJ_CREATE_2); if (pbeNodeIdPtr && pbeConn && err == SA_AIS_OK) { - /*The persistent back-end is present and executing at THIS node. + /*The Persistent Back End is present and executing at THIS node. */ osafassert(cb->mIsCoord); osafassert(pbeNodeId); @@ -7469,7 +7469,7 @@ static void immnd_evt_proc_object_modify(IMMND_CB *cb, IMMND_EVT *evt, writableAttrHasLongDns = hasLongDns; if (pbeNodeIdPtr && pbeConn && err == SA_AIS_OK) { - /*The persistent back-end is present and executing at THIS node. + /*The Persistent Back End is present and executing at THIS node. */ osafassert(cb->mIsCoord); osafassert(pbeNodeId); @@ -7924,10 +7924,10 @@ static void immnd_evt_proc_rt_object_modify(IMMND_CB *cb, IMMND_EVT *evt, */ if (pbeNodeId && err == SA_AIS_OK) { - /*The persistent back-end is present => wait for reply. */ + /*The Persistent Back End is present => wait for reply. */ delayedReply = true; if (pbeConn) { - /*The persistent back-end is executing at THIS node. */ + /*The Persistent Back End is executing at THIS node. */ osafassert(!pbe2BConn); osafassert(cb->mIsCoord); osafassert(pbeNodeId); @@ -8416,7 +8416,7 @@ static void immnd_evt_proc_object_delete(IMMND_CB *cb, IMMND_EVT *evt, immnd_proc_global_abort_ccb(cb, evt->info.objDelete.ccbId); } } - } /* End of PersistentBackEnd handling. */ + } /* End of Persistent Back End handling. */ if (err == SA_AIS_OK) { if (arrSize) { @@ -8768,12 +8768,12 @@ static void immnd_evt_proc_rt_object_delete(IMMND_CB *cb, IMMND_EVT *evt, } if (pbeNodeId && (err == SA_AIS_OK)) { - /*The persistent back-end is present & PRTOs deleted in subtree + /*The Persistent Back End is present & PRTOs deleted in subtree * => wait for reply. */ delayedReply = true; if (pbeConn && arrSize) { TRACE("PBE at this node arrSize:%u", arrSize); - /*The persistent back-end is executing at THIS node. */ + /*The Persistent Back End is executing at THIS node. */ osafassert(!pbe2BConn); osafassert(cb->mIsCoord); osafassert(pbeNodeId == cb->node_id); @@ -8866,7 +8866,7 @@ static void immnd_evt_proc_rt_object_delete(IMMND_CB *cb, IMMND_EVT *evt, pbe_cl_node = NULL; } else if (pbe2BConn && arrSize) { TRACE("PBE SLAVE at this node arrSize:%u", arrSize); - /*The persistent back-end is executing at THIS node. */ + /*The Persistent Back End is executing at THIS node. */ osafassert(!(cb->mIsCoord)); osafassert(cb->mCanBeCoord); osafassert(pbeNodeId != cb->node_id); diff --git a/src/imm/immnd/immnd_main.c b/src/imm/immnd/immnd_main.c index a26005528..50bbb57ec 100644 --- a/src/imm/immnd/immnd_main.c +++ b/src/imm/immnd/immnd_main.c @@ -334,7 +334,7 @@ static uint32_t immnd_initialize(char *progname) if ((immnd_cb->mPbeFile = getenv("IMMSV_PBE_FILE")) != NULL) { LOG_NO( - "Persistent Back-End capability configured, Pbe file:%s (suffix may get added)", + "Persistent Back End capability configured, Pbe file:%s (suffix +may get added)", immnd_cb->mPbeFile); } immnd_cb->mFevsMaxPending = 16; diff --git a/src/imm/immnd/immnd_proc.c b/src/imm/immnd/immnd_proc.c index 84d4906ca..bf14d18a8 100644 --- a/src/imm/immnd/immnd_proc.c +++ b/src/imm/immnd/immnd_proc.c @@ -918,7 +918,7 @@ void immnd_adjustEpoch(IMMND_CB *cb, bool increment) if (pbeNodeId && pbeConn) { IMMND_IMM_CLIENT_NODE *pbe_cl_node = NULL; - /*The persistent back-end is executing at THIS node. */ + /*The Persistent Back End is executing at THIS node. */ osafassert(cb->mIsCoord); osafassert(pbeNodeId == cb->node_id); SaImmOiHandleT implHandle = @@ -1689,11 +1689,11 @@ static bool immnd_ccbsTerminated(IMMND_CB *cb, SaUint32T duration, */ if ((cb->mPbeKills++) == 0) { LOG_WA( - "Persistent back end process appears hung, restarting it."); + "Persistent Back End process appears hung, restarting it."); kill(cb->pbePid, SIGTERM); } else if (cb->mPbeKills > 10) { LOG_WA( - "Persistent back end process appears hung, sending SIGKILL"); + "Persistent Back End process appears hung, sending SIGKILL"); kill(cb->pbePid, SIGKILL); } } else { @@ -2441,10 +2441,10 @@ uint32_t immnd_proc_server(uint32_t *timeout) if (cb->mRim == SA_IMM_KEEP_REPOSITORY) { LOG_WA( - "Persistent back-end process has apparently died."); + "Persistent Back End process has apparently died."); } else { LOG_NO( - "Persistent back-end process has terminated."); + "Persistent Back End process has terminated."); } cb->pbePid = 0; cb->mPbeKills = 0; @@ -2546,10 +2546,10 @@ uint32_t immnd_proc_server(uint32_t *timeout) if (waitpid(cb->pbePid, &status, WNOHANG) > 0) { if (cb->mRim == SA_IMM_KEEP_REPOSITORY) { LOG_WA( - "Persistent back-end process has apparently died."); + "Persistent Back End process has apparently died."); } else { LOG_NO( - "Persistent back-end process has terminated."); + "Persistent Back End process has terminated."); } cb->pbePid = 0; cb->mPbeKills = 0; diff --git a/src/smf/smfd/SmfUpgradeCampaign.h b/src/smf/smfd/SmfUpgradeCampaign.h index 351bd7079..11b9fd8e5 100644 --- a/src/smf/smfd/SmfUpgradeCampaign.h +++ b/src/smf/smfd/SmfUpgradeCampaign.h @@ -272,13 +272,13 @@ class SmfUpgradeCampaign { SaAisErrorT tooManyRestarts(bool* o_result); /// - /// Purpose: Disable IMM persistent backend (PBE) + /// Purpose: Disable IMM Persistent Back End (PBE) /// @return SaAisErrorT /// SaAisErrorT disablePbe(); /// - /// Purpose: Activate IMM persistent backend (PBE) if activated when campaign + /// Purpose: Activate IMM Persistent Back End (PBE) if activated when + campaign /// was started /// @return SaAisErrorT /// -- 2.17.1 _______________________________________________ Opensaf-devel mailing list Opensaf-devel@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/opensaf-devel