Re: Rev 37: Raw implementation done. in http://samba.org/~tridge/psomogyi/

[Stefan (metze) Metzmacher]

-BEGIN PGP SIGNED MESSAGE-
Hash: SHA1

Hi Peter,

do you really need typedef's here?

using 'struct ' in the code is in most cases more readable.

metze
>  typedef struct _ibw_conn_priv {
> + struct ibv_comp_channel *verbs_channel;
> + struct fd_event *verbs_channel_event;
> +
>   struct rdma_cm_id *cm_id; /* client's cm id */
>   int is_accepted;
>  
>   struct ibv_cq   *cq; /* qp is in cm_id */
>   struct ibv_mr *mr;
> - char *buf; /* fixed size (opts.bufsize) buffer for send/recv */
> + char *buf; /* fixed size (qsize * opts.max_msg_size) buffer for 
> send/recv */
>   ibw_wr *wr_list_avail;
>   ibw_wr *wr_list_used;
> - ibw_wr **wr_index; /* array[0..(max_send_wr + max_recv_wr)-1] of 
> (ibw_wr *) */
> + ibw_wr **wr_index; /* array[0..(qsize-1)] of (ibw_wr *) */
>  } ibw_conn_priv;
>  
> 

-BEGIN PGP SIGNATURE-
Version: GnuPG v1.4.2 (GNU/Linux)
Comment: Using GnuPG with SUSE - http://enigmail.mozdev.org

iD8DBQFFfv3zm70gjA5TCD8RApmPAJ9wvNErq22jI7g4GSLQdWc1cERdEwCgqnWs
T/HlKIlGnfzcGthuVDuWJG0=
=BFP3
-END PGP SIGNATURE-

Rev 37: Raw implementation done. in http://samba.org/~tridge/psomogyi/

[psomogyi]

revno: 37
revision-id: [EMAIL PROTECTED]
parent: [EMAIL PROTECTED]
committer: Peter Somogyi <[EMAIL PROTECTED]>
branch nick: ctdb
timestamp: Tue 2006-12-12 19:09:16 +0100
message:
  Raw implementation done.
  Let's start compilation...
modified:
  ib/ibwrapper.c ibwrapper.c-20061204130028-0125b4f5a72f4b11
  ib/ibwrapper.h ibwrapper.h-20061204130028-32755c6266dd3c49
  ib/ibwrapper_internal.h
ibwrapper_internal.h-20061204130028-47f0a7e658b16ca2
=== modified file 'ib/ibwrapper.c'
--- a/ib/ibwrapper.c2006-12-11 18:56:15 +
+++ b/ib/ibwrapper.c2006-12-12 18:09:16 +
@@ -42,33 +42,34 @@
 #define IBW_LASTERR_BUFSIZE 512
 static char ibw_lasterr[IBW_LASTERR_BUFSIZE];
 
+static void ibw_event_handler_verbs(struct event_context *ev,
+   struct fd_event *fde, uint16_t flags, void *private_data);
+
 static int ibw_init_memory(ibw_conn *conn)
 {
ibw_ctx_priv *pctx = talloc_get_type(conn->ctx->internal, ibw_ctx_priv);
ibw_conn_priv *pconn = talloc_get_type(conn->internal, ibw_conn_priv);
 
-   int i, num_msg;
+   int i;
ibw_wr  *p;
 
-   /* didn't find any reason to split send & recv buffer handling */
-   num_msg = pctx->opts.max_recv_wr + pctx->opts.max_send_wr;
-
-   pconn->buf = memalign(pctx->page_size, pctx->opts.max_msg_size);
+   pconn->buf = memalign(pctx->page_size, pctx->max_msg_size);
if (!pconn->buf) {
sprintf(ibw_lasterr, "couldn't allocate work buf\n");
return -1;
}
-   pconn->mr = ibv_reg_mr(pctx->pd, pconn->buf, pctx->opts.bufsize, 
IBV_ACCESS_LOCAL_WRITE);
+   pconn->mr = ibv_reg_mr(pctx->pd, pconn->buf,
+   pctx->qsize * pctx->max_msg_size, IBV_ACCESS_LOCAL_WRITE);
if (!pconn->mr) {
-   sprintf(ibw_lasterr, "Couldn't allocate mr\n");
+   sprintf(ibw_lasterr, "couldn't allocate mr\n");
return -1;
}
 
-   pconn->wr_index = talloc_size(pconn, num_msg * sizeof(ibw_wr *));
+   pconn->wr_index = talloc_size(pconn, pctx->qsize * sizeof(ibw_wr *));
 
-   for(i=0; iqsize; i++) {
p = pconn->wr_index[i] = talloc_zero(pconn, ibw_wr);
-   p->msg = pconn->buf + (i * pconn->opts.max_msg_size);
+   p->msg = pconn->buf + (i * pctx->max_msg_size);
p->wr_id = i;
 
DLIST_ADD(pconn->mr_list_avail, p);
@@ -82,17 +83,6 @@
ibw_ctx *pctx = talloc_get_type(ctx->internal, ibw_ctx_priv);
assert(pctx!=NULL);
 
-   if (pctx->verbs_channel) {
-   ibv_destroy_comp_channel(pctx->verbs_channel);
-   pctx->verbs_channel = NULL;
-   }
-
-   if (pctx->verbs_channel_event) {
-   /* TODO: do we have to do this here? */
-   talloc_free(pctx->verbs_channel_event);
-   pctx->verbs_channel_event = NULL;
-   }
-
if (pctx->pd) {
ibv_dealloc_pd(pctx->pd);
pctx->pd = NULL;
@@ -149,6 +139,15 @@
ibv_destroy_cq(pconn->cq);
pconn->cq = NULL;
}
+   if (pconn->verbs_channel) {
+   ibv_destroy_comp_channel(pconn->verbs_channel);
+   pconn->verbs_channel = NULL;
+   }
+   if (pconn->verbs_channel_event) {
+   /* TODO: do we have to do this here? */
+   talloc_free(pconn->verbs_channel_event);
+   pconn->verbs_channel_event = NULL;
+   }
if (pconn->cm_id) {
rdma_destroy_id(pctx->cm_id);
pctx->cm_id = NULL;
@@ -195,30 +194,44 @@
struct ibv_qp_init_attr init_attr;
int rc;
 
+   /* init mr */
if (ibw_init_memory(conn))
return -1;
 
-   pctx->cq = ibv_create_cq(conn->cm_id->verbs, pctx->opts.max_send_wr + 
pctx->opts.max_recv_wr,
-   ctx, ctx->verbs_channel, 0);
+   /* init verbs */
+   pconn->verbs_channel = ibv_create_comp_channel(pconn->cm_id->verbs);
+   if (!pconn->verbs_channel) {
+   sprintf(ibw_lasterr, "ibv_create_comp_channel failed %d\n", 
errno);
+   goto cleanup;
+   }
+   DEBUG(10, "created channel %p\n", pconn->channel);
+
+   pconn->verbs_channel_event = event_add_fd(pctx->ectx, conn,
+   pconn->verbs_channel->fd, EVENT_FD_READ, 
ibw_event_handler_verbs, conn);
+
+   /* init cq */
+   pconn->cq = ibv_create_cq(conn->cm_id->verbs, pctx->qsize,
+   conn, pconn->verbs_channel, 0);
if (cq==NULL) {
sprintf(ibw_lasterr, "ibv_create_cq failed\n");
return -1;
}
 
-   rc = ibv_req_notify_cq(pctx->cq, 0);
+   rc = ibv_req_notify_cq(pconn->cq, 0);
if (rc) {
sprintf(ibw_lasterr, "ibv_req_notify_cq failed with %d\n", rc);
return rc;
}
 
+   /* init qp */