Hey Roland/IB experts,
I'm playing around with XRC using mlx4, and I'm hitting an error.
Attached is the program to setup an XRC connection and exchange some
data. It is based on the upstream XRC sample program submitted by
Intel. But I back-ported it to the OFED-1.5.4.1 XRC API. The QPs seem
to get setup ok, with the target xrc QP in RTR and the initiator QP in
RTS. Then the server side (initiator QP) posts a send to the client's
XRC-SRQ. On ingress, the client mlx driver gets a type
MLX4_EVENT_TYPE_WQ_ACCESS_ERROR (0x11) subtype 0 event which flows up to
the app via the async event channel as an IBV_EVENT_QP_ACCESS_ERR event.
I'm wondering where to look for the problem? I think I have
dest_qp_num, sq_psn and rq_psn setup correctly, so I'm not sure what I'm
doing wrong. Any thoughts/ideas is much appreciated!
Here's the client output:
[root@hpc-hn1 libibverbs-1.1.4]# ibv_xsrq_pingpong -d mlx4_0 192.168.174.52
local: LID 0001, QPN RECV 98004b SEND 18004c, PSN 5b6d99, SRQN 0042
remote: LID 0002, QPN RECV d4004a SEND 54004b, PSN d7ba7a, SRQN 0042
XRC event_type IBV_EVENT_QP_ACCESS_ERR (0x3), arg 0x98004b
XRC RCV QP: rq_psn 0xd7ba7a sq_psn 0 dest_qp_num 0x54004b access_flags 0x6
XRC event_type IBV_EVENT_QP_LAST_WQE_REACHED (0x10), arg 0x98004b
XRC RCV QP: rq_psn 0xd7ba7a sq_psn 0 dest_qp_num 0x54004b access_flags 0x6
Server output:
[root@hpc-cn2 libibverbs-1.1.4]# ibv_xsrq_pingpong -d mlx4_0
remote: LID 0001, QPN RECV 98004b SEND 18004c, PSN 5b6d99, SRQN 0042
local: LID 0002, QPN RECV d4004a SEND 54004b, PSN d7ba7a, SRQN 0042
/*
* Copyright (c) 2005 Topspin Communications. All rights reserved.
* Copyright (c) 2011 Intel Corporation, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
*copyright notice, this list of conditions and the following
*disclaimer.
*
* - Redistributions in binary form must reproduce the above
*copyright notice, this list of conditions and the following
*disclaimer in the documentation and/or other materials
*provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#if HAVE_CONFIG_H
# include
#endif /* HAVE_CONFIG_H */
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include "pingpong.h"
static const char *event_name_str(enum ibv_event_type event_type)
{
switch (event_type) {
case IBV_EVENT_DEVICE_FATAL:
return "IBV_EVENT_DEVICE_FATAL";
case IBV_EVENT_PORT_ACTIVE:
return "IBV_EVENT_PORT_ACTIVE";
case IBV_EVENT_PORT_ERR:
return "IBV_EVENT_PORT_ERR";
case IBV_EVENT_LID_CHANGE:
return "IBV_EVENT_LID_CHANGE";
case IBV_EVENT_PKEY_CHANGE:
return "IBV_EVENT_PKEY_CHANGE";
case IBV_EVENT_SM_CHANGE:
return "IBV_EVENT_SM_CHANGE";
case IBV_EVENT_CLIENT_REREGISTER:
return "IBV_EVENT_CLIENT_REREGISTER";
case IBV_EVENT_GID_CHANGE:
return "IBV_EVENT_GID_CHANGE";
case IBV_EVENT_QP_ACCESS_ERR:
return "IBV_EVENT_QP_ACCESS_ERR";
case IBV_EVENT_SRQ_ERR:
return "IBV_EVENT_SRQ_ERR";
case IBV_EVENT_SRQ_LIMIT_REACHED:
return "IBV_EVENT_SRQ_LIMIT_REACHED";
case IBV_EVENT_QP_LAST_WQE_REACHED:
return "IBV_EVENT_QP_LAST_WQE_REACHED";
case IBV_EVENT_COMM_EST:
case IBV_EVENT_SQ_DRAINED:
case IBV_EVENT_PATH_MIG:
case IBV_EVENT_PATH_MIG_ERR:
case IBV_EVENT_CQ_ERR:
case IBV_EVENT_QP_FATAL:
case IBV_EVENT_QP_REQ_ERR:
default:
return "unexpected";
}
}
#define MSG_FORMAT "%04x:%06x:%06x:%06x:%04x"
#define MSG_SIZE 31
#define MSG_SSCAN "%x:%x:%x:%x:%x"
#define ADDR_FORMAT \
"%8s: LID %04x, QPN RECV %06x SEND %06x, PSN %06x, SRQN %04x\n"
static int page_size;
struct pingpong_dest
