------------------------------------------------------------ revno: 66 revision-id: [EMAIL PROTECTED] parent: [EMAIL PROTECTED] committer: Peter Somogyi <[EMAIL PROTECTED]> branch nick: ctdb timestamp: Wed 2007-02-14 18:58:20 +0100 message: ib: a trivial approach of integration TODO: bugfix (not yet complete, something happens differently than expected at tcpip when connection is refused in the beginning) modified: common/ctdb.c ctdb.c-20061127094323-t50f58d65iaao5of-2 ib/ibw_ctdb.c ibw_ctdb.c-20070102171255-7krov7858dqza466-1 ib/ibw_ctdb_init.c ibw_ctdb_init.c-20070102171305-cn2z4k7ibx8141d5-1 ib/ibwrapper.c ibwrapper.c-20061204130028-0125b4f5a72f4b11 ib/ibwrapper_test.c ibwrapper_test.c-20061214171730-h11a2z5ed6pt66hj-1 === modified file 'common/ctdb.c' --- a/common/ctdb.c 2007-02-09 01:45:58 +0000 +++ b/common/ctdb.c 2007-02-14 17:58:20 +0000 @@ -32,10 +32,19 @@ int ctdb_set_transport(struct ctdb_context *ctdb, const char *transport) { int ctdb_tcp_init(struct ctdb_context *ctdb); +#ifdef USE_INFINIBAND + int ctdb_ibw_init(struct ctdb_context *ctdb); +#endif /* USE_INFINIBAND */ if (strcmp(transport, "tcp") == 0) { return ctdb_tcp_init(ctdb); } +#ifdef USE_INFINIBAND + if (strcmp(transport, "ib") == 0) { + return ctdb_ibw_init(ctdb); + } +#endif /* USE_INFINIBAND */ + ctdb_set_error(ctdb, "Unknown transport '%s'\n", transport); return -1; }
=== modified file 'ib/ibw_ctdb.c' --- a/ib/ibw_ctdb.c 2007-01-02 17:16:39 +0000 +++ b/ib/ibw_ctdb.c 2007-02-14 17:58:20 +0000 @@ -63,19 +63,24 @@ break; case IBWC_CONNECTED: { /* after ibw_accept or ibw_connect */ struct ctdb_node *node = talloc_get_type(conn->conn_userdata, struct ctdb_node); - if (node!=NULL) /* after ibw_connect */ + if (node!=NULL) { /* after ibw_connect */ + node->private = (void *)conn; node->ctdb->upcalls->node_connected(node); - else { /* after ibw_accept */ + } else { /* after ibw_accept */ /* NOP in CTDB case */ } } break; - case IBWC_DISCONNECTED: /* after ibw_disconnect */ + case IBWC_DISCONNECTED: { /* after ibw_disconnect */ /* TODO: have a CTDB upcall */ - break; + struct ctdb_node *node = talloc_get_type(conn->conn_userdata, struct ctdb_node); + if (node!=NULL) + node->ctdb->upcalls->node_dead(node); + talloc_free(conn); + } break; case IBWC_ERROR: { - struct ctdb_node *node = talloc_get_type(conn->conn_userdata, struct ctdb_node); +/* struct ctdb_node *node = talloc_get_type(conn->conn_userdata, struct ctdb_node); if (node!=NULL) - node->ctdb->upcalls->node_connected(node); + node->ctdb->upcalls->node_dead(node);*/ } break; default: assert(0); === modified file 'ib/ibw_ctdb_init.c' --- a/ib/ibw_ctdb_init.c 2007-01-25 13:22:03 +0000 +++ b/ib/ibw_ctdb_init.c 2007-02-14 17:58:20 +0000 @@ -29,9 +29,6 @@ #include "ibwrapper.h" #include "ibw_ctdb.h" -/* not nice; temporary workaround for the current implementation... */ -static void *last_key = NULL; - static int ctdb_ibw_listen(struct ctdb_context *ctdb, int backlog) { struct ibw_ctx *ictx = talloc_get_type(ctdb->private, struct ibw_ctx); @@ -60,6 +57,7 @@ { struct sockaddr_in sock_out; + memset(&sock_out, 0, sizeof(struct sockaddr_in)); inet_pton(AF_INET, node->address.address, &sock_out.sin_addr); sock_out.sin_port = htons(node->address.port); sock_out.sin_family = PF_INET; @@ -112,37 +110,36 @@ { struct ibw_conn *conn = talloc_get_type(node->private, struct ibw_conn); int rc; - - rc = ibw_send(conn, data, last_key, length); - last_key = NULL; + void *buf, *key; + + assert(length>=sizeof(uint32_t)); + + if (conn==NULL) { + DEBUG(0, ("ctdb_ibw_queue_pkt: conn is NULL\n")); + return -1; + } + + if (ibw_alloc_send_buf(conn, &buf, &key, length)) { + DEBUG(0, ("queue_pkt/ibw_alloc_send_buf failed\n")); + return -1; + } + + memcpy(buf, data, length); + rc = ibw_send(conn, buf, key, length); return rc; } +/* + * transport packet allocator - allows transport to control memory for packets + */ +static void *ctdb_ibw_allocate_pkt(struct ctdb_context *ctdb, size_t size) +{ + /* TODO: use ibw_alloc_send_buf instead... */ + return talloc_size(ctdb, size); +} + #ifdef __NOTDEF__ -/* - * transport packet allocator - allows transport to control memory for packets - */ -static void *ctdb_ibw_allocate_pkt(struct ctdb_node *node, size_t size) -{ - struct ibw_conn *conn = NULL; - void *buf = NULL; - - if (ibw_alloc_send_buf(conn, &buf, &last_key, size)) - return NULL; - - return buf; -} - -static void ctdb_ibw_dealloc_pkt(struct ctdb_node *node, void *data) -{ - if (last_key) { - struct ibw_conn *conn = talloc_get_type(node->private, struct ibw_conn); - - assert(conn!=NULL); - ibw_cancel_send_buf(conn, data, last_key); - } /* else ibw_send is already using it and will free it after completion */ -} static int ctdb_ibw_stop(struct ctdb_context *cctx) { @@ -158,9 +155,8 @@ .start = ctdb_ibw_start, .add_node = ctdb_ibw_add_node, .queue_pkt = ctdb_ibw_queue_pkt, -// .allocate_pkt = ctdb_ibw_allocate_pkt, + .allocate_pkt = ctdb_ibw_allocate_pkt, -// .dealloc_pkt = ctdb_ibw_dealloc_pkt, // .stop = ctdb_ibw_stop }; @@ -171,6 +167,7 @@ { struct ibw_ctx *ictx; + DEBUG(10, ("ctdb_ibw_init invoked...\n")); ictx = ibw_init( NULL, //struct ibw_initattr *attr, /* TODO */ 0, //int nattr, /* TODO */ @@ -186,5 +183,7 @@ ctdb->methods = &ctdb_ibw_methods; ctdb->private = ictx; + + DEBUG(10, ("ctdb_ibw_init succeeded.\n")); return 0; } === modified file 'ib/ibwrapper.c' --- a/ib/ibwrapper.c 2007-02-08 18:06:14 +0000 +++ b/ib/ibwrapper.c 2007-02-14 17:58:20 +0000 @@ -420,6 +420,7 @@ switch (event->event) { case RDMA_CM_EVENT_ADDR_RESOLVED: + DEBUG(11, ("RDMA_CM_EVENT_ADDR_RESOLVED\n")); /* continuing from ibw_connect ... */ rc = rdma_resolve_route(cma_id, 2000); if (rc) { @@ -430,6 +431,7 @@ break; case RDMA_CM_EVENT_ROUTE_RESOLVED: + DEBUG(11, ("RDMA_CM_EVENT_ROUTE_RESOLVED\n")); /* after RDMA_CM_EVENT_ADDR_RESOLVED: */ assert(cma_id->context!=NULL); conn = talloc_get_type(cma_id->context, struct ibw_conn); @@ -441,6 +443,7 @@ break; case RDMA_CM_EVENT_CONNECT_REQUEST: + DEBUG(11, ("RDMA_CM_EVENT_CONNECT_REQUEST\n")); ctx->state = IBWS_CONNECT_REQUEST; conn = ibw_conn_new(ctx); pconn = talloc_get_type(conn->internal, struct ibw_conn_priv); @@ -481,14 +484,23 @@ break; case RDMA_CM_EVENT_ADDR_ERROR: + sprintf(ibw_lasterr, "RDMA_CM_EVENT_ADDR_ERROR, error %d\n", event->status); + goto error; case RDMA_CM_EVENT_ROUTE_ERROR: + sprintf(ibw_lasterr, "RDMA_CM_EVENT_ROUTE_ERROR, error %d\n", event->status); + goto error; case RDMA_CM_EVENT_CONNECT_ERROR: + sprintf(ibw_lasterr, "RDMA_CM_EVENT_CONNECT_ERROR, error %d\n", event->status); + goto error; case RDMA_CM_EVENT_UNREACHABLE: + sprintf(ibw_lasterr, "RDMA_CM_EVENT_UNREACHABLE, error %d\n", event->status); + goto error; case RDMA_CM_EVENT_REJECTED: - sprintf(ibw_lasterr, "cma event %d, error %d\n", event->event, event->status); + sprintf(ibw_lasterr, "RDMA_CM_EVENT_REJECTED, error %d\n", event->status); goto error; case RDMA_CM_EVENT_DISCONNECTED: + DEBUG(11, ("RDMA_CM_EVENT_DISCONNECTED\n")); if (cma_id!=pctx->cm_id) { DEBUG(0, ("client DISCONNECT event cm_id=%p\n", cma_id)); conn = talloc_get_type(cma_id->context, struct ibw_conn); @@ -944,7 +956,7 @@ int rc; DEBUG(10, ("ibw_bind: addr=%s, port=%u\n", - inet_ntoa(my_addr->sin_addr), my_addr->sin_port)); + inet_ntoa(my_addr->sin_addr), ntohs(my_addr->sin_port))); rc = rdma_bind_addr(pctx->cm_id, (struct sockaddr *) my_addr); if (rc) { sprintf(ibw_lasterr, "rdma_bind_addr error %d\n", rc); @@ -1008,7 +1020,8 @@ conn = ibw_conn_new(ctx); conn->conn_userdata = conn_userdata; pconn = talloc_get_type(conn->internal, struct ibw_conn_priv); - DEBUG(10, ("ibw_connect: addr=%s, port=%u\n", inet_ntoa(serv_addr->sin_addr), serv_addr->sin_port)); + DEBUG(10, ("ibw_connect: addr=%s, port=%u\n", inet_ntoa(serv_addr->sin_addr), + ntohs(serv_addr->sin_port))); /* init cm */ rc = rdma_create_id(pctx->cm_channel, &pconn->cm_id, conn, RDMA_PS_TCP); @@ -1023,6 +1036,7 @@ if (rc) { sprintf(ibw_lasterr, "rdma_resolve_addr error %d\n", rc); DEBUG(0, (ibw_lasterr)); + talloc_free(conn); return -1; } === modified file 'ib/ibwrapper_test.c' --- a/ib/ibwrapper_test.c 2007-02-12 15:47:12 +0000 +++ b/ib/ibwrapper_test.c 2007-02-14 17:58:20 +0000 @@ -477,7 +477,7 @@ p = tcx->addrs + i; p->sin_family = AF_INET; p->sin_addr.s_addr = inet_addr(attrs[i].name); - p->sin_port = atoi(attrs[i].value); + p->sin_port = htons(atoi(attrs[i].value)); } return 0;