revno: 37
revision-id: [EMAIL PROTECTED]
parent: [EMAIL PROTECTED]
committer: Peter Somogyi [EMAIL PROTECTED]
branch nick: ctdb
timestamp: Tue 2006-12-12 19:09:16 +0100
message:
Raw implementation done.
Let's start compilation...
modified:
ib/ibwrapper.c ibwrapper.c-20061204130028-0125b4f5a72f4b11
ib/ibwrapper.h ibwrapper.h-20061204130028-32755c6266dd3c49
ib/ibwrapper_internal.h
ibwrapper_internal.h-20061204130028-47f0a7e658b16ca2
=== modified file 'ib/ibwrapper.c'
--- a/ib/ibwrapper.c2006-12-11 18:56:15 +
+++ b/ib/ibwrapper.c2006-12-12 18:09:16 +
@@ -42,33 +42,34 @@
#define IBW_LASTERR_BUFSIZE 512
static char ibw_lasterr[IBW_LASTERR_BUFSIZE];
+static void ibw_event_handler_verbs(struct event_context *ev,
+ struct fd_event *fde, uint16_t flags, void *private_data);
+
static int ibw_init_memory(ibw_conn *conn)
{
ibw_ctx_priv *pctx = talloc_get_type(conn-ctx-internal, ibw_ctx_priv);
ibw_conn_priv *pconn = talloc_get_type(conn-internal, ibw_conn_priv);
- int i, num_msg;
+ int i;
ibw_wr *p;
- /* didn't find any reason to split send recv buffer handling */
- num_msg = pctx-opts.max_recv_wr + pctx-opts.max_send_wr;
-
- pconn-buf = memalign(pctx-page_size, pctx-opts.max_msg_size);
+ pconn-buf = memalign(pctx-page_size, pctx-max_msg_size);
if (!pconn-buf) {
sprintf(ibw_lasterr, couldn't allocate work buf\n);
return -1;
}
- pconn-mr = ibv_reg_mr(pctx-pd, pconn-buf, pctx-opts.bufsize,
IBV_ACCESS_LOCAL_WRITE);
+ pconn-mr = ibv_reg_mr(pctx-pd, pconn-buf,
+ pctx-qsize * pctx-max_msg_size, IBV_ACCESS_LOCAL_WRITE);
if (!pconn-mr) {
- sprintf(ibw_lasterr, Couldn't allocate mr\n);
+ sprintf(ibw_lasterr, couldn't allocate mr\n);
return -1;
}
- pconn-wr_index = talloc_size(pconn, num_msg * sizeof(ibw_wr *));
+ pconn-wr_index = talloc_size(pconn, pctx-qsize * sizeof(ibw_wr *));
- for(i=0; inum_msg; i++) {
+ for(i=0; ipctx-qsize; i++) {
p = pconn-wr_index[i] = talloc_zero(pconn, ibw_wr);
- p-msg = pconn-buf + (i * pconn-opts.max_msg_size);
+ p-msg = pconn-buf + (i * pctx-max_msg_size);
p-wr_id = i;
DLIST_ADD(pconn-mr_list_avail, p);
@@ -82,17 +83,6 @@
ibw_ctx *pctx = talloc_get_type(ctx-internal, ibw_ctx_priv);
assert(pctx!=NULL);
- if (pctx-verbs_channel) {
- ibv_destroy_comp_channel(pctx-verbs_channel);
- pctx-verbs_channel = NULL;
- }
-
- if (pctx-verbs_channel_event) {
- /* TODO: do we have to do this here? */
- talloc_free(pctx-verbs_channel_event);
- pctx-verbs_channel_event = NULL;
- }
-
if (pctx-pd) {
ibv_dealloc_pd(pctx-pd);
pctx-pd = NULL;
@@ -149,6 +139,15 @@
ibv_destroy_cq(pconn-cq);
pconn-cq = NULL;
}
+ if (pconn-verbs_channel) {
+ ibv_destroy_comp_channel(pconn-verbs_channel);
+ pconn-verbs_channel = NULL;
+ }
+ if (pconn-verbs_channel_event) {
+ /* TODO: do we have to do this here? */
+ talloc_free(pconn-verbs_channel_event);
+ pconn-verbs_channel_event = NULL;
+ }
if (pconn-cm_id) {
rdma_destroy_id(pctx-cm_id);
pctx-cm_id = NULL;
@@ -195,30 +194,44 @@
struct ibv_qp_init_attr init_attr;
int rc;
+ /* init mr */
if (ibw_init_memory(conn))
return -1;
- pctx-cq = ibv_create_cq(conn-cm_id-verbs, pctx-opts.max_send_wr +
pctx-opts.max_recv_wr,
- ctx, ctx-verbs_channel, 0);
+ /* init verbs */
+ pconn-verbs_channel = ibv_create_comp_channel(pconn-cm_id-verbs);
+ if (!pconn-verbs_channel) {
+ sprintf(ibw_lasterr, ibv_create_comp_channel failed %d\n,
errno);
+ goto cleanup;
+ }
+ DEBUG(10, created channel %p\n, pconn-channel);
+
+ pconn-verbs_channel_event = event_add_fd(pctx-ectx, conn,
+ pconn-verbs_channel-fd, EVENT_FD_READ,
ibw_event_handler_verbs, conn);
+
+ /* init cq */
+ pconn-cq = ibv_create_cq(conn-cm_id-verbs, pctx-qsize,
+ conn, pconn-verbs_channel, 0);
if (cq==NULL) {
sprintf(ibw_lasterr, ibv_create_cq failed\n);
return -1;
}
- rc = ibv_req_notify_cq(pctx-cq, 0);
+ rc = ibv_req_notify_cq(pconn-cq, 0);
if (rc) {
sprintf(ibw_lasterr, ibv_req_notify_cq failed with %d\n, rc);
return rc;
}
+ /* init qp */
memset(init_attr, 0, sizeof(init_attr));
