revno: 37
revision-id: [EMAIL PROTECTED]
parent: [EMAIL PROTECTED]
committer: Peter Somogyi <[EMAIL PROTECTED]>
branch nick: ctdb
timestamp: Tue 2006-12-12 19:09:16 +0100
message:
Raw implementation done.
Let's start compilation...
modified:
ib/ibwrapper.c ibwrapper.c-20061204130028-0125b4f5a72f4b11
ib/ibwrapper.h ibwrapper.h-20061204130028-32755c6266dd3c49
ib/ibwrapper_internal.h
ibwrapper_internal.h-20061204130028-47f0a7e658b16ca2
=== modified file 'ib/ibwrapper.c'
--- a/ib/ibwrapper.c2006-12-11 18:56:15 +
+++ b/ib/ibwrapper.c2006-12-12 18:09:16 +
@@ -42,33 +42,34 @@
#define IBW_LASTERR_BUFSIZE 512
static char ibw_lasterr[IBW_LASTERR_BUFSIZE];
+static void ibw_event_handler_verbs(struct event_context *ev,
+ struct fd_event *fde, uint16_t flags, void *private_data);
+
static int ibw_init_memory(ibw_conn *conn)
{
ibw_ctx_priv *pctx = talloc_get_type(conn->ctx->internal, ibw_ctx_priv);
ibw_conn_priv *pconn = talloc_get_type(conn->internal, ibw_conn_priv);
- int i, num_msg;
+ int i;
ibw_wr *p;
- /* didn't find any reason to split send & recv buffer handling */
- num_msg = pctx->opts.max_recv_wr + pctx->opts.max_send_wr;
-
- pconn->buf = memalign(pctx->page_size, pctx->opts.max_msg_size);
+ pconn->buf = memalign(pctx->page_size, pctx->max_msg_size);
if (!pconn->buf) {
sprintf(ibw_lasterr, "couldn't allocate work buf\n");
return -1;
}
- pconn->mr = ibv_reg_mr(pctx->pd, pconn->buf, pctx->opts.bufsize,
IBV_ACCESS_LOCAL_WRITE);
+ pconn->mr = ibv_reg_mr(pctx->pd, pconn->buf,
+ pctx->qsize * pctx->max_msg_size, IBV_ACCESS_LOCAL_WRITE);
if (!pconn->mr) {
- sprintf(ibw_lasterr, "Couldn't allocate mr\n");
+ sprintf(ibw_lasterr, "couldn't allocate mr\n");
return -1;
}
- pconn->wr_index = talloc_size(pconn, num_msg * sizeof(ibw_wr *));
+ pconn->wr_index = talloc_size(pconn, pctx->qsize * sizeof(ibw_wr *));
- for(i=0; iqsize; i++) {
p = pconn->wr_index[i] = talloc_zero(pconn, ibw_wr);
- p->msg = pconn->buf + (i * pconn->opts.max_msg_size);
+ p->msg = pconn->buf + (i * pctx->max_msg_size);
p->wr_id = i;
DLIST_ADD(pconn->mr_list_avail, p);
@@ -82,17 +83,6 @@
ibw_ctx *pctx = talloc_get_type(ctx->internal, ibw_ctx_priv);
assert(pctx!=NULL);
- if (pctx->verbs_channel) {
- ibv_destroy_comp_channel(pctx->verbs_channel);
- pctx->verbs_channel = NULL;
- }
-
- if (pctx->verbs_channel_event) {
- /* TODO: do we have to do this here? */
- talloc_free(pctx->verbs_channel_event);
- pctx->verbs_channel_event = NULL;
- }
-
if (pctx->pd) {
ibv_dealloc_pd(pctx->pd);
pctx->pd = NULL;
@@ -149,6 +139,15 @@
ibv_destroy_cq(pconn->cq);
pconn->cq = NULL;
}
+ if (pconn->verbs_channel) {
+ ibv_destroy_comp_channel(pconn->verbs_channel);
+ pconn->verbs_channel = NULL;
+ }
+ if (pconn->verbs_channel_event) {
+ /* TODO: do we have to do this here? */
+ talloc_free(pconn->verbs_channel_event);
+ pconn->verbs_channel_event = NULL;
+ }
if (pconn->cm_id) {
rdma_destroy_id(pctx->cm_id);
pctx->cm_id = NULL;
@@ -195,30 +194,44 @@
struct ibv_qp_init_attr init_attr;
int rc;
+ /* init mr */
if (ibw_init_memory(conn))
return -1;
- pctx->cq = ibv_create_cq(conn->cm_id->verbs, pctx->opts.max_send_wr +
pctx->opts.max_recv_wr,
- ctx, ctx->verbs_channel, 0);
+ /* init verbs */
+ pconn->verbs_channel = ibv_create_comp_channel(pconn->cm_id->verbs);
+ if (!pconn->verbs_channel) {
+ sprintf(ibw_lasterr, "ibv_create_comp_channel failed %d\n",
errno);
+ goto cleanup;
+ }
+ DEBUG(10, "created channel %p\n", pconn->channel);
+
+ pconn->verbs_channel_event = event_add_fd(pctx->ectx, conn,
+ pconn->verbs_channel->fd, EVENT_FD_READ,
ibw_event_handler_verbs, conn);
+
+ /* init cq */
+ pconn->cq = ibv_create_cq(conn->cm_id->verbs, pctx->qsize,
+ conn, pconn->verbs_channel, 0);
if (cq==NULL) {
sprintf(ibw_lasterr, "ibv_create_cq failed\n");
return -1;
}
- rc = ibv_req_notify_cq(pctx->cq, 0);
+ rc = ibv_req_notify_cq(pconn->cq, 0);
if (rc) {
sprintf(ibw_lasterr, "ibv_req_notify_cq failed with %d\n", rc);
return rc;
}
+ /* init qp */