Hi Neelakanta, Can you fix indentation ? In some places the code looks very messy.
I have problem with understanding why protocol52 has been introduced since CLM support is only local per node. If IMMND support CLM integration, then all IMM clients on that node can have this feature. The only way I see this feature to be cluster wise is that if this feature is supported in the cluster, or not. But per node, I don't see we need a new protocol. Find other comments inline started with [Zoran] -----Original Message----- From: reddy.neelaka...@oracle.com [mailto:reddy.neelaka...@oracle.com] Sent: den 21 februari 2017 07:03 To: Hung Duc Nguyen <hung.d.ngu...@dektech.com.au>; Zoran Milinkovic <zoran.milinko...@ericsson.com> Cc: opensaf-devel@lists.sourceforge.net Subject: [PATCH 3 of 3] imm: immnd changes for integrating IMM with CLMS [#1640] v2 src/imm/Makefile.am | 5 +- src/imm/immnd/ImmModel.cc | 44 +++++++- src/imm/immnd/ImmModel.h | 2 + src/imm/immnd/immnd_cb.h | 18 +++ src/imm/immnd/immnd_clm.c | 200 ++++++++++++++++++++++++++++++++++++++++++ src/imm/immnd/immnd_db.c | 125 ++++++++++++++++++++++++++ src/imm/immnd/immnd_evt.c | 31 +++++- src/imm/immnd/immnd_init.h | 7 + src/imm/immnd/immnd_main.c | 42 ++++++++- src/imm/immnd/immnd_mds.c | 30 ++++++- src/nid/nodeinit.conf.payload | 2 +- 11 files changed, 489 insertions(+), 17 deletions(-) diff --git a/src/imm/Makefile.am b/src/imm/Makefile.am --- a/src/imm/Makefile.am +++ b/src/imm/Makefile.am @@ -2,6 +2,7 @@ # # (C) Copyright 2016 The OpenSAF Foundation # Copyright Ericsson AB 2017 - All Rights Reserved. +# Copyright (C) 2017, Oracle and/or its affiliates. All rights reserved. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY @@ -352,6 +353,7 @@ bin_osafimmnd_SOURCES = \ src/imm/immnd/immnd_main.c \ src/imm/immnd/immnd_mds.c \ src/imm/immnd/immnd_proc.c \ + src/imm/immnd/immnd_clm.c \ src/imm/immnd/ImmAttrValue.cc \ src/imm/immnd/ImmSearchOp.cc \ src/imm/immnd/ImmModel.cc @@ -359,7 +361,8 @@ bin_osafimmnd_SOURCES = \ bin_osafimmnd_LDADD = \ lib/libimm_common.la \ lib/libSaAmf.la \ - lib/libopensaf_core.la + lib/libopensaf_core.la \ + lib/libSaClm.la bin_osafimmpbed_CXXFLAGS = $(AM_CXXFLAGS) diff --git a/src/imm/immnd/ImmModel.cc b/src/imm/immnd/ImmModel.cc --- a/src/imm/immnd/ImmModel.cc +++ b/src/imm/immnd/ImmModel.cc @@ -2,6 +2,7 @@ * * (C) Copyright 2008 The OpenSAF Foundation * Copyright Ericsson AB 2009, 2017 - All Rights Reserved. + * Copyright (C) 2017, Oracle and/or its affiliates. All rights reserved. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY @@ -671,12 +672,12 @@ immModel_pbePrtoPurgeMutations(IMMND_CB ConnVector::iterator cvi; unsigned int ix = 0; ImmModel::instance(&cb->immModel)->pbePrtoPurgeMutations(nodeId, cv); - *reqArrSize = (SaUint32T) cv.size(); - if(*reqArrSize) { - *reqConnArr = (SaUint32T *) malloc((*reqArrSize)* sizeof(SaUint32T)); - for(cvi = cv.begin(); cvi!= cv.end();++cvi,++ix) { - (*reqConnArr)[ix] = (*cvi); - } + if(reqArrSize && reqConnArr && cv.size()){ + *reqArrSize = (SaUint32T) cv.size(); + *reqConnArr = (SaUint32T *) malloc((*reqArrSize)* sizeof(SaUint32T)); + for(cvi = cv.begin(); cvi!= cv.end();++cvi,++ix) { + (*reqConnArr)[ix] = (*cvi); + } [Zoran] Why has this code been changed ? Neither reqArrSize nor reqConnArr can be NULL. Earlier code looked quite ok for me. } } @@ -1162,6 +1163,12 @@ immModel_protocol51Allowed(IMMND_CB *cb) return ImmModel::instance(&cb->immModel)->protocol51Allowed(); } +bool +immModel_protocol52Allowed(IMMND_CB *cb) +{ + return ImmModel::instance(&cb->immModel)->protocol52Allowed(); +} + OsafImmAccessControlModeT immModel_accessControlMode(IMMND_CB *cb) { @@ -4056,6 +4063,30 @@ ImmModel::protocol51Allowed() return noStdFlags & OPENSAF_IMM_FLAG_PRT51_ALLOW; } +bool +ImmModel::protocol52Allowed() +{ + //TRACE_ENTER(); + /* Assume that all nodes are running the same version when loading + if (sImmNodeState == IMM_NODE_LOADING) { + return true; + }*/ + ObjectMap::iterator oi = sObjectMap.find(immObjectDn); + if(oi == sObjectMap.end()) { + return false; + } + + ObjectInfo* immObject = oi->second; + ImmAttrValueMap::iterator avi = + immObject->mAttrValueMap.find(immAttrNostFlags); + osafassert(avi != immObject->mAttrValueMap.end()); + osafassert(!(avi->second->isMultiValued())); + ImmAttrValue* valuep = avi->second; + unsigned int noStdFlags = valuep->getValue_int(); + + //TRACE_LEAVE(); + return noStdFlags & OPENSAF_IMM_FLAG_PRT52_ALLOW; +} bool ImmModel::protocol41Allowed() @@ -5077,6 +5108,7 @@ ImmModel::adminOwnerDelete(SaUint32T own noStdFlags |= OPENSAF_IMM_FLAG_PRT51_ALLOW; } + noStdFlags |= OPENSAF_IMM_FLAG_PRT52_ALLOW; valuep->setValue_int(noStdFlags); LOG_NO("%s changed to: 0x%x", immAttrNostFlags.c_str(), noStdFlags); /* END Temporary code. */ diff --git a/src/imm/immnd/ImmModel.h b/src/imm/immnd/ImmModel.h --- a/src/imm/immnd/ImmModel.h +++ b/src/imm/immnd/ImmModel.h @@ -1,6 +1,7 @@ /* -*- OpenSAF -*- * * (C) Copyright 2008 The OpenSAF Foundation + * Copyright (C) 2017, Oracle and/or its affiliates. All rights reserved. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY @@ -117,6 +118,7 @@ public: bool protocol47Allowed(); bool protocol50Allowed(); bool protocol51Allowed(); + bool protocol52Allowed(); bool oneSafe2PBEAllowed(); bool purgeSyncRequest(SaUint32T clientId); bool verifySchemaChange(const std::string& className, diff --git a/src/imm/immnd/immnd_cb.h b/src/imm/immnd/immnd_cb.h --- a/src/imm/immnd/immnd_cb.h +++ b/src/imm/immnd/immnd_cb.h @@ -1,6 +1,7 @@ /* -*- OpenSAF -*- * * (C) Copyright 2008 The OpenSAF Foundation + * Copyright (C) 2017, Oracle and/or its affiliates. All rights reserved. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY @@ -43,6 +44,11 @@ typedef struct immnd_om_search_node { struct immnd_om_search_node *next; } IMMND_OM_SEARCH_NODE; +typedef struct immnd_clm_node_list { + NCS_PATRICIA_NODE patnode; + NCS_NODE_ID node_id; +} IMMND_CLM_NODE_LIST; + typedef struct immnd_immom_client_node { NCS_PATRICIA_NODE patnode; SaImmHandleT imm_app_hdl; /* index for the client tree */ @@ -181,6 +187,11 @@ typedef struct immnd_cb_tag { SaSelectionObjectT amf_sel_obj; /* Selection Object for AMF events */ int nid_started; /* true if started by NID */ bool isNodeTypeController; // true node type is controller + SaSelectionObjectT clmSelectionObject; /* Selection object to wait for clms events*/ + NCS_SEL_OBJ clm_init_sel_obj; /* Selection object wait for clms intialization*/ + bool isClmNodeJoined; /* True => If clm joined the cluster*/ + //bool protocol52Allowed; // True, if protocol52 is allowed. [Zoran] Remove upper commented line + NCS_PATRICIA_TREE immnd_clm_list; /* IMMND_IMM_CLIENT_NODE - node */ } IMMND_CB; /* CB prototypes */ @@ -200,6 +211,13 @@ uint32_t immnd_client_node_tree_init(IMM void immnd_client_node_tree_cleanup(IMMND_CB *cb); void immnd_client_node_tree_destroy(IMMND_CB *cb); +uint32_t immnd_clm_node_list_init(IMMND_CB *cb); +void immnd_clm_node_get(IMMND_CB *cb, NODE_ID node, IMMND_CLM_NODE_LIST **imm_clm_node); +uint32_t immnd_clm_node_add(IMMND_CB *cb, NODE_ID key); +uint32_t immnd_clm_node_delete(IMMND_CB *cb, IMMND_CLM_NODE_LIST *immnd_clm_node); +void immnd_clm_node_cleanup(IMMND_CB *cb); +void immnd_clm_node_destroy(IMMND_CB *cb); + /* #define m_IMMSV_CONVERT_EXPTIME_TEN_MILLI_SEC(t) \ SaTimeT now; \ diff --git a/src/imm/immnd/immnd_clm.c b/src/imm/immnd/immnd_clm.c new file mode 100644 --- /dev/null +++ b/src/imm/immnd/immnd_clm.c @@ -0,0 +1,200 @@ +/* -*- OpenSAF -*- + * + * Copyright (C) 2017, Oracle and/or its affiliates. All rights reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Oracle + * + */ + +#include "immnd.h" +#include "base/osaf_time.h" + +/**************************************************************************** + * Name : immnd_clm_node_change + * + * Description : + * When the clm clusterchange happens the function is called. + * + * Arguments: + * left: True if clm left the node else false + * + * Return Values : + * NCSCC_RC_SUCCESS if change is completed sucessfully. + * +****************************************************************************/ +uint32_t immnd_clm_node_change(bool left){ + IMMSV_EVT send_evt; + TRACE_ENTER(); + uint32_t rc = NCSCC_RC_SUCCESS; + if(left){ + immnd_cb->isClmNodeJoined = false; + TRACE("isClmNodeJoined is set to false, protocol52=%x", immModel_protocol52Allowed(immnd_cb)); + } else { + immnd_cb->isClmNodeJoined = true; + TRACE("isClmNodeJoined is set to true, protocol52=%x", immModel_protocol52Allowed(immnd_cb)); + } + + if (immModel_protocol52Allowed(immnd_cb)){ + SaImmHandleT clientHdl = 0; + IMMND_IMM_CLIENT_NODE * client_node = NULL; + send_evt.type = IMMSV_EVT_TYPE_IMMA; + immnd_client_node_getnext(immnd_cb, 0, &client_node); + while(client_node){ + memset(&send_evt, '\0', sizeof(IMMSV_EVT)); + if (client_node->version.minorVersion >= 0x12 && + client_node->version.majorVersion == 0x2 && + client_node->version.releaseCode == 'A') { + if(immnd_cb->isClmNodeJoined){ + send_evt.info.imma.type = IMMA_EVT_ND2A_IMM_CLM_NODE_JOINED; + TRACE("Sending Node Joined message to client handle %llx", client_node->imm_app_hdl); + } else { + send_evt.info.imma.type = IMMA_EVT_ND2A_IMM_CLM_NODE_LEFT; + TRACE("Sending Node Left message to client handle %llx", client_node->imm_app_hdl); + } + + if(immnd_mds_msg_send(immnd_cb, client_node->sv_id, + client_node->agent_mds_dest, &send_evt) != NCSCC_RC_SUCCESS) { + TRACE("Sending clm change to client id %llx failed", client_node->imm_app_hdl); + }else { + TRACE("immnd_mds_msg_send success"); + } + } + clientHdl = client_node->imm_app_hdl; + immnd_client_node_getnext(immnd_cb, clientHdl, &client_node); + } + } + TRACE_LEAVE(); + return rc; +} + +/**************************************************************************** + * Name : immnd_clm_track_cbk + * + * Description : + * CLM callback for tracking the node membership status. + * Depending upon the membership status (joining/leaving cluster) + * of a node with node_id of present node clm_join or clm_left will + * be broadcasted to all imma agents. + * + * Return Values : None. + * +****************************************************************************/ +static void immnd_clm_track_cbk(const SaClmClusterNotificationBufferT_4 *notificationBuffer, + SaUint32T numberOfMembers, SaInvocationT invocation, + const SaNameT *rootCauseEntity, const SaNtfCorrelationIdsT *correlationIds, + SaClmChangeStepT step, SaTimeT timeSupervision, SaAisErrorT error) +{ + NCS_NODE_ID node_id; + uint32_t i = 0; + IMMND_CLM_NODE_LIST * imm_clm_node = NULL ; + + TRACE_ENTER2("Returned error value from callback is %d",error); + + if (error != SA_AIS_OK) + return; + + for (i = 0; i < notificationBuffer->numberOfItems; i++) { + switch(step) { + case SA_CLM_CHANGE_COMPLETED: + node_id = notificationBuffer->notification[i].clusterNode.nodeId; + immnd_clm_node_get(immnd_cb, node_id, &imm_clm_node); [Zoran] The whole FOR loop will work correctly only for the first iteration when imm_clm_node = NULL. If in the second immnd_clm_node_get() a CLM node is not found, imm_clm_node will have value from previous search. I suggest to set imm_clm_node to NULL before immnd_clm_node_get() is called. + if(notificationBuffer->notification[i].clusterChange == SA_CLM_NODE_LEFT){ + if(imm_clm_node){ + if( node_id == immnd_cb->node_id){ + if (immnd_clm_node_change(true) != NCSCC_RC_SUCCESS) { + TRACE_4(" immnd_proc_ckpt_clm_node_left failed"); + } + } + immnd_clm_node_delete(immnd_cb, imm_clm_node); + TRACE("Node %x left the CLM membership", node_id); + }else if(node_id == immnd_cb->node_id){ + TRACE("IMMND restarted when node is locked"); + if (immnd_clm_node_change(true) != NCSCC_RC_SUCCESS) { + TRACE_4(" immnd_proc_ckpt_clm_node_left failed"); + } + } + } else if(!imm_clm_node){ + if ((notificationBuffer->notification[i].clusterChange == SA_CLM_NODE_NO_CHANGE) || + (notificationBuffer->notification[i].clusterChange == SA_CLM_NODE_JOINED) || + (notificationBuffer->notification[i].clusterChange == SA_CLM_NODE_RECONFIGURED)) { + if( node_id == immnd_cb->node_id){ + if (immnd_clm_node_change(false) != NCSCC_RC_SUCCESS) { + TRACE_4("immnd_proc_ckpt_clm_node_joined failed"); + } + } + if (immnd_clm_node_add(immnd_cb, node_id) != NCSCC_RC_SUCCESS) { + TRACE_4("immnd_clm_node_add failed"); + } + + TRACE("Node %x Joined the CLM membership", node_id); + } + } + + break; + default: + break; + } + } + TRACE_LEAVE(); +} + +static SaVersionT clmVersion = { 'B', 0x04, 0x01 }; +static const SaClmCallbacksT_4 clm_callbacks = { + 0, + immnd_clm_track_cbk /*saClmClusterTrackCallback*/ +}; + +/**************************************************************************** + * Name : immnd_clm_init_thread + * + * Description : + * Registers with the CLM service (B.04.01). + * Return Values : None. + * + ****************************************************************************/ +void immnd_init_with_clm(void) +{ + SaAisErrorT rc = SA_AIS_OK; + + TRACE_ENTER(); + + rc = saClmInitialize_4(&immnd_cb->clm_hdl, &clm_callbacks, &clmVersion); + while ((rc == SA_AIS_ERR_TRY_AGAIN) || (rc == SA_AIS_ERR_TIMEOUT) || + (rc == SA_AIS_ERR_UNAVAILABLE)) { + osaf_nanosleep(&kHundredMilliseconds); + rc = saClmInitialize_4(&immnd_cb->clm_hdl, &clm_callbacks, &clmVersion); + } + if (rc != SA_AIS_OK) { + LOG_ER("saClmInitialize failed with error: %d", rc); + TRACE_LEAVE(); + exit(EXIT_FAILURE); + } + + rc = saClmSelectionObjectGet(immnd_cb->clm_hdl, &immnd_cb->clmSelectionObject); + if (rc != SA_AIS_OK) { + LOG_ER("saClmSelectionObjectGet failed with error: %d", rc); + TRACE_LEAVE(); + exit(EXIT_FAILURE); + } + //rc = saClmClusterTrack_4(immnd_cb->clm_hdl, SA_TRACK_CHANGES_ONLY, NULL); [Zoran] Remove upper commented line + rc = saClmClusterTrack_4(immnd_cb->clm_hdl, SA_TRACK_CURRENT|SA_TRACK_CHANGES, NULL); + if (rc != SA_AIS_OK) { + LOG_ER("saClmClusterTrack failed with error: %d", rc); + TRACE_LEAVE(); + exit(EXIT_FAILURE); + } + TRACE("CLM Initialization SUCCESS......"); + TRACE_LEAVE(); + //return NULL; + return ; +} + diff --git a/src/imm/immnd/immnd_db.c b/src/imm/immnd/immnd_db.c --- a/src/imm/immnd/immnd_db.c +++ b/src/imm/immnd/immnd_db.c @@ -1,6 +1,7 @@ /* -*- OpenSAF -*- * * (C) Copyright 2008 The OpenSAF Foundation + * Copyright (C) 2017, Oracle and/or its affiliates. All rights reserved. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY @@ -172,6 +173,130 @@ void immnd_client_node_tree_destroy(IMMN return; } +/**************************************************************************** + Name : immnd_clm_node_list_init + Description : This routine is used to initialize the IMMND clm node list init + Arguments : cb - pointer to the IMMND Control Block + Return Values : NCSCC_RC_SUCCESS/NCSCC_RC_FAILURE + Notes : None +*****************************************************************************/ +uint32_t immnd_clm_node_list_init(IMMND_CB *cb) +{ + NCS_PATRICIA_PARAMS param; + memset(¶m, 0, sizeof(NCS_PATRICIA_PARAMS)); + + param.key_size = sizeof(NODE_ID); + if (ncs_patricia_tree_init(&cb->immnd_clm_list, ¶m) != NCSCC_RC_SUCCESS) { + return NCSCC_RC_FAILURE; + } + return NCSCC_RC_SUCCESS; +} + +/**************************************************************************** + * Name : immnd_clm_node_get + * Description : Function to get the clm node from the clm list. + * Arguments : IMMND_CB *cb, - IMMND Control Block + * : NODE_ID - CLM nodeid. + * Return Values : IMMND_CLM_NODE_LIST ** immnd_clm_node_list + * Notes : None. + *****************************************************************************/ +void immnd_clm_node_get(IMMND_CB *cb, NODE_ID node, IMMND_CLM_NODE_LIST **imm_clm_node) +{ + *imm_clm_node = (IMMND_CLM_NODE_LIST*) + ncs_patricia_tree_get(&cb->immnd_clm_list, (uint8_t *)&node); + return; +} + +/**************************************************************************** + Name : immnd_clm_node_add + Description : This routine adds the new node to immnd_clm_node_list + Arguments : immnd_tree - IMMND Tree. + NODE_ID - CLM Node. + Return Values : NCSCC_RC_SUCCESS/NCSCC_RC_FAILURE + Notes : The caller takes the cb lock before calling this function +*****************************************************************************/ +uint32_t immnd_clm_node_add(IMMND_CB *cb, NODE_ID key) +{ + IMMND_CLM_NODE_LIST *immnd_clm_node = calloc(1, sizeof(IMMND_CLM_NODE_LIST)); + immnd_clm_node->node_id = key; + immnd_clm_node->patnode.key_info = (uint8_t *)&immnd_clm_node->node_id; + + if (ncs_patricia_tree_add(&cb->immnd_clm_list, &immnd_clm_node->patnode) != NCSCC_RC_SUCCESS) { + LOG_ER("IMMND - ncs_patricia_tree_add failed in immnd_clm_node_add"); + free(immnd_clm_node); + return NCSCC_RC_FAILURE; + } + + return NCSCC_RC_SUCCESS; +} + +/**************************************************************************** + Name : immnd_clm_node_delete + Description : This routine deletes the node from immnd_clm_node_list + Arguments : IMMD_CB *cb - IMMD Control Block. + : NODE_ID - CLM Node. + Return Values : None +*****************************************************************************/ +uint32_t immnd_clm_node_delete(IMMND_CB *cb, IMMND_CLM_NODE_LIST *immnd_clm_node) +{ + uint32_t rc = NCSCC_RC_SUCCESS; + + /* Remove the Node from the client tree */ + if (ncs_patricia_tree_del(&cb->immnd_clm_list, (NCS_PATRICIA_NODE *)&immnd_clm_node->patnode) != NCSCC_RC_SUCCESS) { + LOG_WA("IMMND CLM NODE DELETE FROM PAT TREE FAILED"); + rc = NCSCC_RC_FAILURE; + } + + /* Free the Client Node */ + if (immnd_clm_node) { + free(immnd_clm_node); + } + return rc; +} + +/**************************************************************************** + Name : immnd_clm_node_cleanup + Description : This routine Free all the nodes in clm_node_list. + Arguments : IMMD_CB *cb - IMMD Control Block. + Return Values : None +****************************************************************************/ +void immnd_clm_node_cleanup(IMMND_CB *cb) +{ + IMMND_CLM_NODE_LIST *immnd_clm_node; + NODE_ID key; + memset(&key, 0, sizeof(NODE_ID)); + + /* Get the First Node */ + immnd_clm_node = (IMMND_CLM_NODE_LIST*) + ncs_patricia_tree_getnext(&cb->immnd_clm_list, (uint8_t *)&key); + while (immnd_clm_node) { + key = immnd_clm_node->node_id; + immnd_clm_node_delete(cb, immnd_clm_node); + + immnd_clm_node = (IMMND_CLM_NODE_LIST*) + ncs_patricia_tree_getnext(&cb->immnd_clm_list, (uint8_t *)&key); + } + + return; +} + +/**************************************************************************** + Name : immnd_clm_node_destroy + Description : This routine destroys the IMMND clm node list. + Arguments : IMMD_CB *cb - IMMD Control Block. + Return Values : None +*****************************************************************************/ +void immnd_clm_node_destroy(IMMND_CB *cb) +{ + /* cleanup the clm list */ + immnd_clm_node_cleanup(cb); + + /* destroy the tree */ + ncs_patricia_tree_destroy(&cb->immnd_clm_list); + + return; +} + /* FEVS MESSAGE QUEUEING */ /************************************************************************* diff --git a/src/imm/immnd/immnd_evt.c b/src/imm/immnd/immnd_evt.c --- a/src/imm/immnd/immnd_evt.c +++ b/src/imm/immnd/immnd_evt.c @@ -1,6 +1,7 @@ /* -*- OpenSAF -*- * * (C) Copyright 2008 The OpenSAF Foundation + * Copyright (C) 2017, Oracle and/or its affiliates. All rights reserved. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY @@ -786,6 +787,16 @@ static uint32_t immnd_evt_proc_imm_init( } } + if (evt->info.initReq.version.minorVersion >= 0x12 && + evt->info.initReq.version.majorVersion == 0x2 && + evt->info.initReq.version.releaseCode == 'A') { + if ( immModel_protocol52Allowed(cb) && !cb->isClmNodeJoined && cb->mIntroduced != 2) { + error = SA_AIS_ERR_UNAVAILABLE; + LOG_ER("CLM node went down prot=%x nodeJoined=%x",immModel_protocol52Allowed(cb), cb->isClmNodeJoined); + goto clm_left; + } + } + cl_node = calloc(1, sizeof(IMMND_IMM_CLIENT_NODE)); if (cl_node == NULL) { LOG_ER("IMMND - Client Alloc Failed"); @@ -835,6 +846,7 @@ static uint32_t immnd_evt_proc_imm_init( send_evt.info.imma.info.initRsp.immHandle = cl_node->imm_app_hdl; error = SA_AIS_OK; + clm_left: agent_rsp: send_evt.type = IMMSV_EVT_TYPE_IMMA; send_evt.info.imma.type = IMMA_EVT_ND2A_IMM_INIT_RSP; @@ -9419,6 +9431,14 @@ static void immnd_evt_proc_finalize_sync "SA_IMM_KEEP_REPOSITORY":"SA_IMM_INIT_FROM_FILE"); } immnd_adjustEpoch(cb, true); + /* If the node is payload give the indication to clm_init_sel_obj + * because payload nodes are not subscribing for AVD up + */ + if(!immnd_cb->isNodeTypeController && !cb->clm_hdl){ + TRACE_8("clm_init_sel_obj indication is given at payload"); + ncs_sel_obj_ind(&immnd_cb->clm_init_sel_obj); + + } /* Sync completed for client => trigger active resurrect. */ memset(&send_evt, '\0', sizeof(IMMSV_EVT)); @@ -9429,25 +9449,25 @@ static void immnd_evt_proc_finalize_sync prev_hdl = cl_node->imm_app_hdl; if(!(cl_node->mIsResurrect)) { LOG_WA("Found active client id: %llx version:%c %u %u, after sync, should not happen", - cl_node->imm_app_hdl, cl_node->version.releaseCode, - cl_node->version.majorVersion, - cl_node->version.minorVersion); + cl_node->imm_app_hdl, cl_node->version.releaseCode, + cl_node->version.majorVersion, + cl_node->version.minorVersion); immnd_client_node_getnext(cb, prev_hdl, &cl_node); continue; } /* Send resurrect message. */ if (immnd_mds_msg_send(cb, cl_node->sv_id, - cl_node->agent_mds_dest, &send_evt)!=NCSCC_RC_SUCCESS) + cl_node->agent_mds_dest, &send_evt)!=NCSCC_RC_SUCCESS) { LOG_WA("Failed to send active resurrect message"); } + ++count; /* Remove the temporary client node. */ immnd_client_node_del(cb, cl_node); memset(cl_node, '\0', sizeof(IMMND_IMM_CLIENT_NODE)); free(cl_node); cl_node = NULL; - ++count; immnd_client_node_getnext(cb, 0, &cl_node); } TRACE_2("Triggered %u active resurrects", count); @@ -10417,6 +10437,7 @@ static uint32_t immnd_evt_proc_mds_evt(I } } LOG_NO("IMMD SERVICE IS DOWN, HYDRA IS CONFIGURED => UNREGISTERING IMMND form MDS"); + immnd_mds_unregister(cb); /* Discard local clients ... */ immnd_proc_discard_other_nodes(cb); /* Isolate from the rest of cluster */ diff --git a/src/imm/immnd/immnd_init.h b/src/imm/immnd/immnd_init.h --- a/src/imm/immnd/immnd_init.h +++ b/src/imm/immnd/immnd_init.h @@ -1,6 +1,7 @@ /* -*- OpenSAF -*- * * (C) Copyright 2008 The OpenSAF Foundation + * Copyright (C) 2017, Oracle and/or its affiliates. All rights reserved. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY @@ -322,6 +323,7 @@ extern "C" { bool immModel_protocol46Allowed(IMMND_CB *cb); bool immModel_protocol47Allowed(IMMND_CB *cb); bool immModel_protocol50Allowed(IMMND_CB *cb); + bool immModel_protocol52Allowed(IMMND_CB *cb); bool immModel_oneSafe2PBEAllowed(IMMND_CB *cb); OsafImmAccessControlModeT immModel_accessControlMode(IMMND_CB *cb); const char *immModel_authorizedGroup(IMMND_CB *cb); @@ -502,4 +504,9 @@ void freeSearchNext(IMMSV_OM_RSP_SEARCH_ uint32_t immnd_proc_server(uint32_t *timeout); /* End : ---- immnd_proc.c */ +/* File : ---- immnd_clm.c */ +uint32_t immnd_clm_node_change(bool left); +void immnd_init_with_clm(); +/* Ebd : ---- immnd_clm.c */ + #endif // IMM_IMMND_IMMND_INIT_H_ diff --git a/src/imm/immnd/immnd_main.c b/src/imm/immnd/immnd_main.c --- a/src/imm/immnd/immnd_main.c +++ b/src/imm/immnd/immnd_main.c @@ -1,6 +1,7 @@ /* -*- OpenSAF -*- * * (C) Copyright 2008-2010 The OpenSAF Foundation + * Copyright (C) 2017, Oracle and/or its affiliates. All rights reserved. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY @@ -41,6 +42,8 @@ #define FD_TERM 0 #define FD_AMF 1 #define FD_MBX 2 +#define FD_CLM_INIT 3 +#define FD_CLM 4 static IMMND_CB _immnd_cb; IMMND_CB *immnd_cb = &_immnd_cb; @@ -86,6 +89,7 @@ static uint32_t immnd_cb_db_init(IMMND_C if (rc == NCSCC_RC_FAILURE) LOG_ER("client node tree init failed"); + rc = immnd_clm_node_list_init(cb); return (rc); } @@ -144,6 +148,8 @@ static uint32_t immnd_initialize(char *p immnd_cb->mProgName = progname; immnd_cb->mDir = getenv("IMMSV_ROOT_DIRECTORY"); immnd_cb->mFile = getenv("IMMSV_LOAD_FILE"); + immnd_cb->clm_hdl = 0; + immnd_cb->clmSelectionObject = -1; if ((envVar = getenv("IMMSV_NUM_NODES"))) { int numNodes = atoi(envVar); if(numNodes > 255) { @@ -224,6 +230,12 @@ static uint32_t immnd_initialize(char *p LOG_ER("m_NCS_IPC_ATTACH FAILED"); goto done; } + + /* Create a selection object for clm intialization*/ + if ((rc = ncs_sel_obj_create(&immnd_cb->clm_init_sel_obj)) != NCSCC_RC_SUCCESS) { + LOG_ER("ncs_sel_obj_create failed for clm intialization"); + goto done; + } if ((rc = immnd_mds_register(immnd_cb)) != NCSCC_RC_SUCCESS) { TRACE("immnd_mds_register FAILED %u", rc); @@ -278,8 +290,8 @@ int main(int argc, char *argv[]) server task when we are very bussy. */ int maxEvt = 100; struct timespec start_time; - struct pollfd fds[3]; - int term_fd; + struct pollfd fds[5]; + int term_fd, nfds=4;; daemonize(argc, argv); @@ -310,6 +322,8 @@ int main(int argc, char *argv[]) fds[FD_AMF].events = POLLIN; fds[FD_MBX].fd = mbx_fd.rmv_obj; fds[FD_MBX].events = POLLIN; + fds[FD_CLM_INIT].fd = immnd_cb->clm_init_sel_obj.rmv_obj; + fds[FD_CLM_INIT].events = POLLIN; while (1) { /* Watch out for performance bug. Possibly change from event-count @@ -327,7 +341,7 @@ int main(int argc, char *argv[]) maxEvt = (timeout == 100) ? 50 : 100; /* Wait for events */ - int ret = poll(fds, 3, (passed_time_ms < timeout) ? (timeout - passed_time_ms) : 0); + int ret = poll(fds, nfds, (passed_time_ms < timeout) ? (timeout - passed_time_ms) : 0); if (ret == -1) { if (errno == EINTR) @@ -377,6 +391,28 @@ int main(int argc, char *argv[]) } } + if (fds[FD_CLM_INIT].revents & POLLIN && !immnd_cb->clm_hdl) { + TRACE("Initalize CLM "); + ncs_sel_obj_rmv_ind(&immnd_cb->clm_init_sel_obj, true, true); + immnd_init_with_clm(); + nfds=5; + fds[FD_CLM].fd = immnd_cb->clmSelectionObject; + fds[FD_CLM].events = POLLIN; + } + + if (fds[FD_CLM].revents & POLLIN) { + if ((error = saClmDispatch(immnd_cb->clm_hdl, SA_DISPATCH_ALL)) != SA_AIS_OK) { + LOG_ER("saClmDispatch failed: %u", error); + if(error == SA_AIS_ERR_BAD_HANDLE){ + LOG_NO("Re-initializing with CLMS"); + immnd_clm_node_cleanup(immnd_cb); [Zoran] Shouldn't previous CLM handle be closed ? Otherwise I see resource leak here. Thanks, Zoran + immnd_init_with_clm(); + } else { + break; + } + } + } + if (eventCount >= maxEvt) { /* Make some progress on background task, even when we are very busy. */ diff --git a/src/imm/immnd/immnd_mds.c b/src/imm/immnd/immnd_mds.c --- a/src/imm/immnd/immnd_mds.c +++ b/src/imm/immnd/immnd_mds.c @@ -1,6 +1,7 @@ /* -*- OpenSAF -*- * * (C) Copyright 2008 The OpenSAF Foundation + * Copyright (C) 2017, Oracle and/or its affiliates. All rights reserved. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY @@ -188,6 +189,23 @@ uint32_t immnd_mds_register(IMMND_CB *cb goto error1; } + if(cb->isNodeTypeController){ + + /* STEP 6: Subscribe to AVD events in MDS. This will be + used for CLM registration at controllers.*/ + + svc_id[0] = NCSMDS_SVC_ID_AVD; + svc_info.i_op = MDS_SUBSCRIBE; + svc_info.info.svc_subscribe.i_scope = NCSMDS_SCOPE_INTRANODE; + svc_info.info.svc_subscribe.i_num_svcs = 1; + svc_info.info.svc_subscribe.i_svc_ids = svc_id; + + if (ncsmds_api(&svc_info) == NCSCC_RC_FAILURE) { + LOG_WA("MDS AVD Subscription Failed"); + goto error1; + } + } + cb->node_id = m_NCS_GET_NODE_ID; TRACE_2("cb->node_id:%x", cb->node_id); @@ -601,7 +619,17 @@ static uint32_t immnd_mds_svc_evt(IMMND_ priority = NCS_IPC_PRIORITY_VERY_HIGH; m_NCS_UNLOCK(&cb->immnd_immd_up_lock, NCS_LOCK_WRITE); - } + } else if (svc_evt->i_svc_id == NCSMDS_SVC_ID_AVD) { + if (svc_evt->i_change == NCSMDS_UP) { + TRACE_8("MDS UP dest: %" PRIx64 ", node ID: %x, svc_id: %d", + svc_evt->i_dest, svc_evt->i_node_id, svc_evt->i_svc_id); + //Subscribed for only INTRA NODE, only one ADEST will come. + if (m_MDS_DEST_IS_AN_ADEST(svc_evt->i_dest) ) { + TRACE_8("AVD ADEST UP"); + ncs_sel_obj_ind(&immnd_cb->clm_init_sel_obj); + } + } + } /* IMMA events from other nodes can not happen */ if ((svc_evt->i_svc_id == NCSMDS_SVC_ID_IMMA_OM) || (svc_evt->i_svc_id == NCSMDS_SVC_ID_IMMA_OI)) diff --git a/src/nid/nodeinit.conf.payload b/src/nid/nodeinit.conf.payload --- a/src/nid/nodeinit.conf.payload +++ b/src/nid/nodeinit.conf.payload @@ -54,6 +54,6 @@ ############################################################################# xxCLCCLIDIRxx/osaf-transport:TRANSPORT:S:xxCLCCLIDIRxx/osaf-transport:6000:-6:2:1:start:stop +xxCLCCLIDIRxx/osaf-clmna:CLMNA:S:xxCLCCLIDIRxx/osaf-clmna:4000::9:1:start:stop xxCLCCLIDIRxx/osaf-immnd:IMMND:S:xxCLCCLIDIRxx/osaf-immnd:48000:4:2:1:start:stop -xxCLCCLIDIRxx/osaf-clmna:CLMNA:S:xxCLCCLIDIRxx/osaf-clmna:4000::9:1:start:stop xxCLCCLIDIRxx/osaf-amfnd:AMFND:S:xxCLCCLIDIRxx/osaf-amfnd:99000::0:1:start:stop ------------------------------------------------------------------------------ Check out the vibrant tech community on one of the world's most engaging tech sites, SlashDot.org! http://sdm.link/slashdot _______________________________________________ Opensaf-devel mailing list Opensaf-devel@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/opensaf-devel
