James, Update the uDAPL openib_cma provider to work with the new uCMA event channel interface. I ran a full set of Intel-MPI test suites with these latest changes and it looks fine. Sync up with Sean on commits.
Signed-off by: Arlin Davis <[EMAIL PROTECTED]> Index: dapl/openib_cma/dapl_ib_util.c =================================================================== --- dapl/openib_cma/dapl_ib_util.c (revision 6942) +++ dapl/openib_cma/dapl_ib_util.c (working copy) @@ -67,6 +67,7 @@ static const char rcsid[] = "$Id: $"; int g_dapl_loopback_connection = 0; int g_ib_pipe[2]; +struct rdma_event_channel *g_cm_events = NULL; ib_thread_state_t g_ib_thread_state = 0; DAPL_OS_THREAD g_ib_thread; DAPL_OS_LOCK g_hca_lock; @@ -184,6 +185,7 @@ int32_t dapls_ib_release(void) { dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " dapl_ib_release: \n"); dapli_ib_thread_destroy(); + rdma_destroy_event_channel(g_cm_events); return 0; } @@ -214,9 +216,17 @@ DAT_RETURN dapls_ib_open_hca(IN IB_HCA_N dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " open_hca: %s - %p\n", hca_name, hca_ptr); + /* Setup the global cm event channel */ + dapl_os_lock(&g_hca_lock); + if (g_cm_events == NULL) { + g_cm_events = rdma_create_event_channel(); + if (g_cm_events == NULL) + return DAT_INTERNAL_ERROR; + } + dapl_os_unlock(&g_hca_lock); + if (dapli_ib_thread_init()) return DAT_INTERNAL_ERROR; - /* HCA name will be hostname or IP address */ if (getipaddr((char*)hca_name, @@ -224,14 +234,13 @@ DAT_RETURN dapls_ib_open_hca(IN IB_HCA_N sizeof(DAT_SOCK_ADDR6))) return DAT_INVALID_ADDRESS; - /* cm_id will bind local device/GID based on IP address */ - if (rdma_create_id(&cm_id, (void*)hca_ptr)) + if (rdma_create_id(g_cm_events, &cm_id, (void*)hca_ptr)) return DAT_INTERNAL_ERROR; ret = rdma_bind_addr(cm_id, (struct sockaddr *)&hca_ptr->hca_address); - if (ret) { + if ((ret) || (cm_id->verbs == NULL)) { rdma_destroy_id(cm_id); dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " open_hca: ERR bind (%d) %s \n", @@ -551,8 +560,8 @@ int dapli_ib_thread_init(void) } /* uCMA events non-blocking */ - opts = fcntl(rdma_get_fd(), F_GETFL); /* uCMA */ - if (opts < 0 || fcntl(rdma_get_fd(), + opts = fcntl(g_cm_events->fd, F_GETFL); /* uCMA */ + if (opts < 0 || fcntl(g_cm_events->fd, F_SETFL, opts | O_NONBLOCK) < 0) { dapl_dbg_log (DAPL_DBG_TYPE_ERR, " dapl_ib_init: ERR with uCMA FD\n" ); @@ -741,7 +750,7 @@ void dapli_thread(void *arg) dapl_dbg_log (DAPL_DBG_TYPE_UTIL, " ib_thread(%d,0x%x): ENTER: pipe %d ucma %d\n", - getpid(), g_ib_thread, g_ib_pipe[0], rdma_get_fd()); + getpid(), g_ib_thread, g_ib_pipe[0], g_cm_events->fd); /* Poll across pipe, CM, AT never changes */ dapl_os_lock( &g_hca_lock ); @@ -749,7 +758,7 @@ void dapli_thread(void *arg) ufds[0].fd = g_ib_pipe[0]; /* pipe */ ufds[0].events = POLLIN; - ufds[1].fd = rdma_get_fd(); /* uCMA */ + ufds[1].fd = g_cm_events->fd; /* uCMA */ ufds[1].events = POLLIN; while (g_ib_thread_state == IB_THREAD_RUN) { Index: dapl/openib_cma/dapl_ib_cm.c =================================================================== --- dapl/openib_cma/dapl_ib_cm.c (revision 6942) +++ dapl/openib_cma/dapl_ib_cm.c (working copy) @@ -59,6 +59,8 @@ #include <sys/poll.h> #include <signal.h> +extern struct rdma_event_channel *g_cm_events; + /* local prototypes */ static struct dapl_cm_id * dapli_req_recv(struct dapl_cm_id *conn, struct rdma_cm_event *event); @@ -614,7 +616,7 @@ dapls_ib_setup_conn_listener(IN DAPL_IA dapl_os_lock_init(&conn->lock); /* create CM_ID, bind to local device, create QP */ - if (rdma_create_id(&conn->cm_id, (void*)conn)) { + if (rdma_create_id(g_cm_events, &conn->cm_id, (void*)conn)) { dapl_os_free(conn, sizeof(*conn)); return(dapl_convert_errno(errno,"setup_listener")); } @@ -1067,7 +1069,7 @@ void dapli_cma_event_cb(void) dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " cm_event()\n"); /* process one CM event, fairness */ - if(!rdma_get_cm_event(&event)) { + if(!rdma_get_cm_event(g_cm_events, &event)) { struct dapl_cm_id *conn; /* set proper conn from cm_id context*/ Index: dapl/openib_cma/dapl_ib_qp.c =================================================================== --- dapl/openib_cma/dapl_ib_qp.c (revision 6942) +++ dapl/openib_cma/dapl_ib_qp.c (working copy) @@ -35,6 +35,8 @@ #include "dapl.h" #include "dapl_adapter_util.h" +extern struct rdma_event_channel *g_cm_events; + /* * dapl_ib_qp_alloc * @@ -128,7 +130,7 @@ DAT_RETURN dapls_ib_qp_alloc(IN DAPL_IA dapl_os_lock_init(&conn->lock); /* create CM_ID, bind to local device, create QP */ - if (rdma_create_id(&cm_id, (void*)conn)) { + if (rdma_create_id(g_cm_events, &cm_id, (void*)conn)) { dapl_os_free(conn, sizeof(*conn)); return(dapl_convert_errno(errno, "create_qp")); } _______________________________________________ openib-general mailing list openib-general@openib.org http://openib.org/mailman/listinfo/openib-general To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general