Change the winverbs interface to accept remote RDMA addresses in network byte order. This keeps it consistent with all other remote/wire data fields.
Unfortunately, this means that the remote address must be swizzled between winverbs and the HCA libraries, which take the address in host order, but the other fields in network order. Longer term, the HCA libraries can provide optimized interfaces for winverbs, like posting a single work request. For now, to avoid compatibility issues later, define the winverbs interface using what's most reasonable. Update libibverbs accordingly. Signed-off-by: Sean Hefty <[email protected]> --- diff -up -r -X \mshefty\scm\winof\trunk\docs\dontdiff.txt -I '\$Id:' trunk\core\winverbs/user/wv_base.h branches\winverbs\core\winverbs/user/wv_base.h --- trunk\core\winverbs/user/wv_base.h 2008-11-05 16:43:19.529313500 -0800 +++ branches\winverbs\core\winverbs/user/wv_base.h 2009-06-09 11:22:24.193591900 -0700 @@ -70,7 +70,6 @@ protected: #if defined(_WIN64) #define WvConvertSgl(pSgl, nSge) ((ib_local_ds_t *) pSgl) - #define WvConvertSends(pSend) ((ib_send_wr_t *) pSend) #else static inline ib_local_ds_t *WvConvertSgl(WV_SGE* pSgl, SIZE_T nSge) { @@ -81,16 +80,6 @@ protected: } return (ib_local_ds_t *) pSgl; } - - static inline ib_send_wr_t *WvConvertSends(WV_SEND_REQUEST *pSend) - { - WV_SEND_REQUEST *preq; - - for (preq = pSend; preq != NULL; preq = preq->pNext) { - WvConvertSgl(preq->pSgl, preq->nSge); - } - return (ib_send_wr_t *) pSend; - } #endif (_WIN64) #endif // _WV_BASE_H_ diff -up -r -X \mshefty\scm\winof\trunk\docs\dontdiff.txt -I '\$Id:' trunk\core\winverbs/user/wv_qp.cpp branches\winverbs\core\winverbs/user/wv_qp.cpp --- trunk\core\winverbs/user/wv_qp.cpp 2009-03-25 12:11:46.356325900 -0700 +++ branches\winverbs\core\winverbs/user/wv_qp.cpp 2009-06-09 11:48:04.545241500 -0700 @@ -421,18 +421,31 @@ PostReceive(UINT64 WrId, WV_SGE* pSgl, S STDMETHODIMP CWVQueuePair:: PostSend(WV_SEND_REQUEST *pSend, WV_SEND_REQUEST **ppFailed) { - ib_send_wr_t *wr; + WV_SEND_REQUEST *wr; ib_api_status_t stat; HRESULT hr; - stat = m_pVerbs->post_send(m_hVerbsQp, WvConvertSends(pSend), - (ppFailed == NULL) ? &wr : (ib_send_wr_t **) ppFailed); + for (wr = pSend; wr != NULL; wr = wr->pNext) { + if (wr->Opcode != WvSend) { + wr->Wr.Rdma.RemoteAddress = _byteswap_uint64(wr->Wr.Rdma.RemoteAddress); + } + WvConvertSgl(wr->pSgl, wr->nSge); + } + + stat = m_pVerbs->post_send(m_hVerbsQp, (ib_send_wr_t *) pSend, + (ib_send_wr_t **) (ppFailed ? ppFailed : &wr)); if (stat == IB_SUCCESS) { hr = WV_SUCCESS; } else { hr = WvConvertIbStatus(stat); } + for (wr = pSend; wr != NULL; wr = wr->pNext) { + if (wr->Opcode != WvSend) { + wr->Wr.Rdma.RemoteAddress = _byteswap_uint64(wr->Wr.Rdma.RemoteAddress); + } + } + return hr; } Only in branches\winverbs\inc\kernel\rdma: _ntstatus_.h diff -up -r -X \mshefty\scm\winof\trunk\docs\dontdiff.txt -I '\$Id:' trunk\inc\user\rdma/winverbs.h branches\winverbs\inc\user\rdma/winverbs.h --- trunk\inc\user\rdma/winverbs.h 2009-06-02 13:23:44.246541100 -0700 +++ branches\winverbs\inc\user\rdma/winverbs.h 2009-06-09 11:18:51.345933700 -0700 @@ -291,14 +291,14 @@ typedef struct _WV_SEND_REQUEST { struct { - UINT64 RemoteAddress; + NET64 RemoteAddress; NET32 Rkey; } Rdma; struct { - UINT64 RemoteAddress; + NET64 RemoteAddress; NET32 Rkey; NET64 Compare; NET64 Exchange; @@ -307,7 +307,7 @@ typedef struct _WV_SEND_REQUEST struct { - UINT64 RemoteAddress; + NET64 RemoteAddress; NET32 Rkey; NET64 Add; UINT64 Reserved; @@ -739,9 +739,9 @@ DECLARE_INTERFACE_(IWVSharedReceiveQueue #undef INTERFACE #define INTERFACE IWVQueuePair -// {a847c13c-e617-489c-b0ab-2da73eb0adfd} -DEFINE_GUID(IID_IWVQueuePair, 0xa847c13c, 0xe617, 0x489c, - 0xb0, 0xab, 0x2d, 0xa7, 0x3e, 0xb0, 0xad, 0xfd); +// {35C64226-6A1F-4c8d-9465-C6FEE8053CDD} +DEFINE_GUID(IID_IWVQueuePair, 0x35c64226, 0x6a1f, 0x4c8d, + 0x94, 0x65, 0xc6, 0xfe, 0xe8, 0x5, 0x3c, 0xdd); DECLARE_INTERFACE_(IWVQueuePair, IWVOverlapped) { @@ -802,9 +802,9 @@ DECLARE_INTERFACE_(IWVQueuePair, IWVOver #undef INTERFACE #define INTERFACE IWVConnectQueuePair -// {A791309C-4244-4194-AD63-7D372CADE5E0} -DEFINE_GUID(IID_IWVConnectQueuePair, 0xa791309c, 0x4244, 0x4194, - 0xad, 0x63, 0x7d, 0x37, 0x2c, 0xad, 0xe5, 0xe0); +// {DF905570-AEEB-4114-B30E-4DC3EB5A9AD6} +DEFINE_GUID(IID_IWVConnectQueuePair, 0xdf905570, 0xaeeb, 0x4114, + 0xb3, 0xe, 0x4d, 0xc3, 0xeb, 0x5a, 0x9a, 0xd6); DECLARE_INTERFACE_(IWVConnectQueuePair, IWVQueuePair) { diff -up -r -X \mshefty\scm\winof\trunk\docs\dontdiff.txt -I '\$Id:' trunk\ulp\libibverbs/src/verbs.cpp branches\winverbs\ulp\libibverbs/src/verbs.cpp --- trunk\ulp\libibverbs/src/verbs.cpp 2009-04-08 13:33:19.307495000 -0700 +++ branches\winverbs\ulp\libibverbs/src/verbs.cpp 2009-06-09 11:45:51.798639900 -0700 @@ -735,6 +735,7 @@ int ibv_post_send(struct ibv_qp *qp, str } if (cur_wr->opcode != 0) { + cur_wr->wr.rdma.remote_addr = htonll(cur_wr->wr.rdma.remote_addr); cur_wr->wr.rdma.rkey = htonl(cur_wr->wr.rdma.rkey); } } @@ -744,6 +745,7 @@ int ibv_post_send(struct ibv_qp *qp, str for (cur_wr = wr; cur_wr != NULL; cur_wr = cur_wr->next) { if (cur_wr->opcode != 0) { cur_wr->wr.rdma.rkey = ntohl(cur_wr->wr.rdma.rkey); + cur_wr->wr.rdma.remote_addr = htonll(cur_wr->wr.rdma.remote_addr); } if ((cur_wr->send_flags & WV_SEND_IMMEDIATE) != 0) { _______________________________________________ ofw mailing list [email protected] http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ofw
