James,
Update the uDAPL openib_cma provider to work with the new uCMA event channel
interface. I ran a full
set of Intel-MPI test suites with these latest changes and it looks fine. Sync
up with Sean on
commits.
Signed-off by: Arlin Davis <[EMAIL PROTECTED]>
Index: dapl/openib_cma/dapl_ib_util.c
===================================================================
--- dapl/openib_cma/dapl_ib_util.c (revision 6942)
+++ dapl/openib_cma/dapl_ib_util.c (working copy)
@@ -67,6 +67,7 @@ static const char rcsid[] = "$Id: $";
int g_dapl_loopback_connection = 0;
int g_ib_pipe[2];
+struct rdma_event_channel *g_cm_events = NULL;
ib_thread_state_t g_ib_thread_state = 0;
DAPL_OS_THREAD g_ib_thread;
DAPL_OS_LOCK g_hca_lock;
@@ -184,6 +185,7 @@ int32_t dapls_ib_release(void)
{
dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " dapl_ib_release: \n");
dapli_ib_thread_destroy();
+ rdma_destroy_event_channel(g_cm_events);
return 0;
}
@@ -214,9 +216,17 @@ DAT_RETURN dapls_ib_open_hca(IN IB_HCA_N
dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
" open_hca: %s - %p\n", hca_name, hca_ptr);
+ /* Setup the global cm event channel */
+ dapl_os_lock(&g_hca_lock);
+ if (g_cm_events == NULL) {
+ g_cm_events = rdma_create_event_channel();
+ if (g_cm_events == NULL)
+ return DAT_INTERNAL_ERROR;
+ }
+ dapl_os_unlock(&g_hca_lock);
+
if (dapli_ib_thread_init())
return DAT_INTERNAL_ERROR;
-
/* HCA name will be hostname or IP address */
if (getipaddr((char*)hca_name,
@@ -224,14 +234,13 @@ DAT_RETURN dapls_ib_open_hca(IN IB_HCA_N
sizeof(DAT_SOCK_ADDR6)))
return DAT_INVALID_ADDRESS;
-
/* cm_id will bind local device/GID based on IP address */
- if (rdma_create_id(&cm_id, (void*)hca_ptr))
+ if (rdma_create_id(g_cm_events, &cm_id, (void*)hca_ptr))
return DAT_INTERNAL_ERROR;
ret = rdma_bind_addr(cm_id,
(struct sockaddr *)&hca_ptr->hca_address);
- if (ret) {
+ if ((ret) || (cm_id->verbs == NULL)) {
rdma_destroy_id(cm_id);
dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
" open_hca: ERR bind (%d) %s \n",
@@ -551,8 +560,8 @@ int dapli_ib_thread_init(void)
}
/* uCMA events non-blocking */
- opts = fcntl(rdma_get_fd(), F_GETFL); /* uCMA */
- if (opts < 0 || fcntl(rdma_get_fd(),
+ opts = fcntl(g_cm_events->fd, F_GETFL); /* uCMA */
+ if (opts < 0 || fcntl(g_cm_events->fd,
F_SETFL, opts | O_NONBLOCK) < 0) {
dapl_dbg_log (DAPL_DBG_TYPE_ERR,
" dapl_ib_init: ERR with uCMA FD\n" );
@@ -741,7 +750,7 @@ void dapli_thread(void *arg)
dapl_dbg_log (DAPL_DBG_TYPE_UTIL,
" ib_thread(%d,0x%x): ENTER: pipe %d ucma %d\n",
- getpid(), g_ib_thread, g_ib_pipe[0], rdma_get_fd());
+ getpid(), g_ib_thread, g_ib_pipe[0], g_cm_events->fd);
/* Poll across pipe, CM, AT never changes */
dapl_os_lock( &g_hca_lock );
@@ -749,7 +758,7 @@ void dapli_thread(void *arg)
ufds[0].fd = g_ib_pipe[0]; /* pipe */
ufds[0].events = POLLIN;
- ufds[1].fd = rdma_get_fd(); /* uCMA */
+ ufds[1].fd = g_cm_events->fd; /* uCMA */
ufds[1].events = POLLIN;
while (g_ib_thread_state == IB_THREAD_RUN) {
Index: dapl/openib_cma/dapl_ib_cm.c
===================================================================
--- dapl/openib_cma/dapl_ib_cm.c (revision 6942)
+++ dapl/openib_cma/dapl_ib_cm.c (working copy)
@@ -59,6 +59,8 @@
#include <sys/poll.h>
#include <signal.h>
+extern struct rdma_event_channel *g_cm_events;
+
/* local prototypes */
static struct dapl_cm_id * dapli_req_recv(struct dapl_cm_id *conn,
struct rdma_cm_event *event);
@@ -614,7 +616,7 @@ dapls_ib_setup_conn_listener(IN DAPL_IA
dapl_os_lock_init(&conn->lock);
/* create CM_ID, bind to local device, create QP */
- if (rdma_create_id(&conn->cm_id, (void*)conn)) {
+ if (rdma_create_id(g_cm_events, &conn->cm_id, (void*)conn)) {
dapl_os_free(conn, sizeof(*conn));
return(dapl_convert_errno(errno,"setup_listener"));
}
@@ -1067,7 +1069,7 @@ void dapli_cma_event_cb(void)
dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " cm_event()\n");
/* process one CM event, fairness */
- if(!rdma_get_cm_event(&event)) {
+ if(!rdma_get_cm_event(g_cm_events, &event)) {
struct dapl_cm_id *conn;
/* set proper conn from cm_id context*/
Index: dapl/openib_cma/dapl_ib_qp.c
===================================================================
--- dapl/openib_cma/dapl_ib_qp.c (revision 6942)
+++ dapl/openib_cma/dapl_ib_qp.c (working copy)
@@ -35,6 +35,8 @@
#include "dapl.h"
#include "dapl_adapter_util.h"
+extern struct rdma_event_channel *g_cm_events;
+
/*
* dapl_ib_qp_alloc
*
@@ -128,7 +130,7 @@ DAT_RETURN dapls_ib_qp_alloc(IN DAPL_IA
dapl_os_lock_init(&conn->lock);
/* create CM_ID, bind to local device, create QP */
- if (rdma_create_id(&cm_id, (void*)conn)) {
+ if (rdma_create_id(g_cm_events, &cm_id, (void*)conn)) {
dapl_os_free(conn, sizeof(*conn));
return(dapl_convert_errno(errno, "create_qp"));
}
_______________________________________________
openib-general mailing list
openib-general@openib.org
http://openib.org/mailman/listinfo/openib-general
To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
