For both RC and UD connect requests, the private data is not being received on socket and passed back via the active side REJECT event.
UD requires new extended reject event type of DAT_IB_UD_CONNECTION_REJECT_EVENT to distiquish between RC and UD type rejects. cr_thread exit/cleanup processing fixed to insure all items are off the list before exiting. Signed-off-by: Arlin Davis <[email protected]> --- dapl/openib_scm/dapl_ib_cm.c | 96 +++++++++++++++++++++++---------- dat/include/dat2/dat_ib_extensions.h | 7 ++- 2 files changed, 72 insertions(+), 31 deletions(-) diff --git a/dapl/openib_scm/dapl_ib_cm.c b/dapl/openib_scm/dapl_ib_cm.c index db0a25b..d116c62 100644 --- a/dapl/openib_scm/dapl_ib_cm.c +++ b/dapl/openib_scm/dapl_ib_cm.c @@ -539,17 +539,6 @@ static void dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr) sin_addr)); goto bail; } - /* check for consumer reject */ - if (cm_ptr->dst.rej) { - dapl_log(DAPL_DBG_TYPE_CM, - " CONN_RTU read: PEER REJ reason=0x%x -> %s\n", - ntohs(cm_ptr->dst.rej), - inet_ntoa(((struct sockaddr_in *) - ep_ptr->param.remote_ia_address_ptr)-> - sin_addr)); - event = IB_CME_DESTINATION_REJECT_PRIVATE_DATA; - goto bail; - } /* convert peer response values to host order */ cm_ptr->dst.port = ntohs(cm_ptr->dst.port); @@ -601,6 +590,23 @@ static void dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr) } } + /* check for consumer reject */ + if (cm_ptr->dst.rej) { + dapl_log(DAPL_DBG_TYPE_CM, + " CONN_RTU read: PEER REJ reason=0x%x -> %s\n", + ntohs(cm_ptr->dst.rej), + inet_ntoa(((struct sockaddr_in *) + ep_ptr->param.remote_ia_address_ptr)-> + sin_addr)); + event = IB_CME_DESTINATION_REJECT_PRIVATE_DATA; +#ifdef DAT_EXTENSIONS + if (cm_ptr->dst.qp_type == IBV_QPT_UD) + goto ud_bail; + else +#endif + goto bail; + } + /* modify QP to RTR and then to RTS with remote info */ dapl_os_lock(&ep_ptr->header.lock); if (dapls_modify_qp_state(ep_ptr->qp_handle, @@ -637,9 +643,11 @@ static void dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr) /* init cm_handle and post the event with private data */ ep_ptr->cm_handle = cm_ptr; cm_ptr->state = SCM_CONNECTED; + event = IB_CME_CONNECTED; dapl_dbg_log(DAPL_DBG_TYPE_EP, " ACTIVE: connected!\n"); #ifdef DAT_EXTENSIONS +ud_bail: if (cm_ptr->dst.qp_type == IBV_QPT_UD) { DAT_IB_EXTENSION_EVENT_DATA xevent; @@ -652,26 +660,35 @@ static void dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr) &cm_ptr->dst.ia_address, sizeof(cm_ptr->dst.ia_address)); + if (event == IB_CME_CONNECTED) + event = DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED; + else + event = DAT_IB_UD_CONNECTION_REJECT_EVENT; + dapls_evd_post_connection_event_ext((DAPL_EVD *) ep_ptr->param. connect_evd_handle, - DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED, + event, (DAT_EP_HANDLE) ep_ptr, - (DAT_COUNT) cm_ptr->dst. - p_size, - (DAT_PVOID *) cm_ptr-> - p_data, - (DAT_PVOID *) & xevent); + (DAT_COUNT) cm_ptr->dst.p_size, + (DAT_PVOID *) cm_ptr->p_data, + (DAT_PVOID *) &xevent); + + /* done with socket, don't destroy cm_ptr, need pdata */ + closesocket(cm_ptr->socket); + cm_ptr->socket = DAPL_INVALID_SOCKET; } else #endif dapl_evd_connection_callback(cm_ptr, IB_CME_CONNECTED, cm_ptr->p_data, ep_ptr); return; - bail: - /* close socket, free cm structure and post error event */ - dapli_cm_destroy(cm_ptr); + +bail: + /* close socket, and post error event */ dapls_ib_reinit_ep(ep_ptr); /* reset QP state */ - dapl_evd_connection_callback(NULL, event, NULL, ep_ptr); + closesocket(cm_ptr->socket); + cm_ptr->socket = DAPL_INVALID_SOCKET; + dapl_evd_connection_callback(NULL, event, cm_ptr->p_data, ep_ptr); } /* @@ -1034,11 +1051,13 @@ void dapli_socket_accept_rtu(dp_ib_cm_handle_t cm_ptr) param.connect_evd_handle, DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED, (DAT_EP_HANDLE) cm_ptr->ep, - (DAT_COUNT) cm_ptr->dst. - p_size, - (DAT_PVOID *) cm_ptr-> - p_data, - (DAT_PVOID *) & xevent); + (DAT_COUNT) cm_ptr->dst.p_size, + (DAT_PVOID *) cm_ptr->p_data, + (DAT_PVOID *) &xevent); + + /* done with socket, don't destroy cm_ptr, need pdata */ + closesocket(cm_ptr->socket); + cm_ptr->socket = DAPL_INVALID_SOCKET; } else #endif dapls_cr_callback(cm_ptr, IB_CME_CONNECTED, NULL, cm_ptr->sp); @@ -1295,10 +1314,16 @@ dapls_ib_reject_connection(IN dp_ib_cm_handle_t cm_ptr, " reject(cm %p reason %x, pdata %p, psize %d)\n", cm_ptr, reason, pdata, psize); + if (psize > IB_MAX_REJ_PDATA_SIZE) + return DAT_LENGTH_ERROR; + /* write reject data to indicate reject */ if (cm_ptr->socket != DAPL_INVALID_SOCKET) { cm_ptr->dst.rej = (uint16_t) reason; cm_ptr->dst.rej = htons(cm_ptr->dst.rej); + cm_ptr->dst.p_size = htonl(psize); + /* get qp_type from request */ + cm_ptr->dst.qp_type = ntohs(cm_ptr->dst.qp_type); iov[0].iov_base = (void *)&cm_ptr->dst; iov[0].iov_len = sizeof(ib_qp_cm_t); @@ -1316,7 +1341,7 @@ dapls_ib_reject_connection(IN dp_ib_cm_handle_t cm_ptr, } /* cr_thread will destroy CR */ - cm_ptr->state = SCM_REJECTED; + cm_ptr->state = SCM_DESTROY; if (send(cm_ptr->hca->ib_trans.scm[1], "w", sizeof "w", 0) == -1) dapl_log(DAPL_DBG_TYPE_CM, " cm_destroy: thread wakeup error = %s\n", @@ -1555,7 +1580,7 @@ void cr_thread(void *arg) dapl_os_lock(&hca_ptr->ib_trans.lock); hca_ptr->ib_trans.cr_state = IB_THREAD_RUN; - while (hca_ptr->ib_trans.cr_state == IB_THREAD_RUN) { + while (1) { dapl_fd_zero(set); dapl_fd_set(hca_ptr->ib_trans.scm[0], set, DAPL_FD_READ); @@ -1574,10 +1599,13 @@ void cr_thread(void *arg) dapl_llist_remove_entry(&hca_ptr->ib_trans.list, (DAPL_LLIST_ENTRY *) & cr->entry); + dapl_dbg_log(DAPL_DBG_TYPE_CM, + " CR FREE: %p ep=%p st=%d sock=%d\n", + cr, cr->ep, cr->state, cr->socket); dapl_os_free(cr, sizeof(*cr)); continue; } - if (cr->socket == DAPL_INVALID_SOCKET) + if (cr->socket == DAPL_INVALID_SOCKET) continue; event = (cr->state == SCM_CONN_PENDING) ? @@ -1664,6 +1692,11 @@ void cr_thread(void *arg) dapl_os_lock(&hca_ptr->ib_trans.lock); } + /* set to exit and all resources destroyed */ + if ((hca_ptr->ib_trans.cr_state != IB_THREAD_RUN) && + (dapl_llist_is_empty(&hca_ptr->ib_trans.list))) + break; + dapl_os_unlock(&hca_ptr->ib_trans.lock); dapl_select(set); @@ -1676,6 +1709,11 @@ void cr_thread(void *arg) strerror(errno)); } dapl_os_lock(&hca_ptr->ib_trans.lock); + + /* set to exit and all resources destroyed */ + if ((hca_ptr->ib_trans.cr_state != IB_THREAD_RUN) && + (dapl_llist_is_empty(&hca_ptr->ib_trans.list))) + break; } dapl_os_unlock(&hca_ptr->ib_trans.lock); diff --git a/dat/include/dat2/dat_ib_extensions.h b/dat/include/dat2/dat_ib_extensions.h index 2c082b1..59df1de 100755 --- a/dat/include/dat2/dat_ib_extensions.h +++ b/dat/include/dat2/dat_ib_extensions.h @@ -70,8 +70,10 @@ * 2.0.3 - Add query/print counter support for IA, EP, and EVD's * dat_query_counters(), dat_print_counters() * + * 2.0.4 - Add DAT_IB_UD_CONNECTION_REJECT_EVENT extended UD event + * */ -#define DAT_IB_EXTENSION_VERSION 203 /* 2.0.3 */ +#define DAT_IB_EXTENSION_VERSION 204 /* 2.0.4 */ #define DAT_ATTR_COUNTERS "DAT_COUNTERS" #define DAT_IB_ATTR_FETCH_AND_ADD "DAT_IB_FETCH_AND_ADD" #define DAT_IB_ATTR_CMP_AND_SWAP "DAT_IB_CMP_AND_SWAP" @@ -89,7 +91,8 @@ typedef enum dat_ib_event_number { DAT_IB_DTO_EVENT = DAT_IB_EXTENSION_RANGE_BASE, DAT_IB_UD_CONNECTION_REQUEST_EVENT, - DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED + DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED, + DAT_IB_UD_CONNECTION_REJECT_EVENT } DAT_IB_EVENT_NUMBER; -- 1.5.2.5 _______________________________________________ general mailing list [email protected] http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
