Provide recorded mbox send/receive history and a dump of it together with the AEQ state so mailbox timeouts and response mismatches can be diagnosed.
Signed-off-by: Jiacheng Ye <[email protected]> --- drivers/net/hinic3/base/hinic3_eqs.c | 59 +++++++++++++++++++++++++++ drivers/net/hinic3/base/hinic3_eqs.h | 2 + drivers/net/hinic3/base/hinic3_mbox.c | 53 ++++++++++++++++++++++++ drivers/net/hinic3/base/hinic3_mbox.h | 22 ++++++++++ 4 files changed, 136 insertions(+) diff --git a/drivers/net/hinic3/base/hinic3_eqs.c b/drivers/net/hinic3/base/hinic3_eqs.c index 1fc78adc85..e961eda374 100644 --- a/drivers/net/hinic3/base/hinic3_eqs.c +++ b/drivers/net/hinic3/base/hinic3_eqs.c @@ -587,6 +587,65 @@ hinic3_dump_aeq_info(struct hinic3_hwdev *hwdev) } } +void hinic3_dump_aeq_mbox_info(struct hinic3_hwdev *hwdev) +{ + struct save_mbox_info *save_mbox = NULL; + struct hinic3_eq *aeq = NULL; + struct hinic3_aeq_elem *aeqe_pos = NULL; + struct rte_pci_device *pci_dev = NULL; + struct rte_eth_dev *eth_dev = NULL; + uint32_t i, pos; + uint8_t src, size, wrapped, seq_id, seg_len, msg_id, mod; + uint8_t data[HINIC3_AEQE_DATA_SIZE]; + uint16_t src_func_idx, cmd; + uint32_t aeqe_desc; + uint64_t mbox_header; + enum hinic3_aeq_type event; + + eth_dev = &rte_eth_devices[hwdev->port_id]; + pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev); + save_mbox = hwdev->func_to_func->save_mbox; + aeq = &hwdev->aeqs->aeq[HINIC3_MBOX_RSP_MSG_AEQ]; + + PMD_DRV_LOG(ERR, "Sending the latest mbox messages:"); + for (i = 0; i < HINIC3_MBOX_SAVE_NUM; i++) { + pos = (hwdev->func_to_func->save_mbox->start + i) % HINIC3_MBOX_SAVE_NUM; + PMD_DRV_LOG(ERR, "send_msg_id: %u, cmd: %u, mod: %u, port: %u, func id: %u, bus: %u, devid: %u", + save_mbox->send_info[pos].send_msg_id, save_mbox->send_info[pos].cmd, + save_mbox->send_info[pos].mod, save_mbox->send_info[pos].port, + save_mbox->send_info[pos].func_id, save_mbox->send_info[pos].bus, + save_mbox->send_info[pos].devid); + } + + PMD_DRV_LOG(ERR, "dump aeqe info:"); + PMD_DRV_LOG(ERR, "port: %u, eq cid: %u, func id: %u, bus: %u, devid: %u, wrapped: %u", + hwdev->port_id, aeq->cons_idx, pci_dev->addr.function, pci_dev->addr.bus, + pci_dev->addr.devid, aeq->wrapped); + for (i = 0; i < aeq->eq_len; i++) { + /* Parsing the data field. */ + aeqe_pos = GET_AEQ_ELEM(aeq, i); + aeqe_desc = rte_be_to_cpu_32(aeqe_pos->desc); + memcpy(data, aeqe_pos->aeqe_data, HINIC3_AEQE_DATA_SIZE); + hinic3_be32_to_cpu(data, HINIC3_AEQE_DATA_SIZE); + mbox_header = *((uint64_t *)data); + seq_id = HINIC3_MSG_HEADER_GET(mbox_header, SEQID); + seg_len = HINIC3_MSG_HEADER_GET(mbox_header, SEG_LEN); + src_func_idx = HINIC3_MSG_HEADER_GET(mbox_header, SRC_GLB_FUNC_IDX); + msg_id = HINIC3_MSG_HEADER_GET(mbox_header, MSG_ID); + mod = HINIC3_MSG_HEADER_GET(mbox_header, MODULE); + cmd = HINIC3_MSG_HEADER_GET(mbox_header, CMD); + /* Parsing the desc field. */ + event = EQ_ELEM_DESC_GET(aeqe_desc, TYPE); + src = EQ_ELEM_DESC_GET(aeqe_desc, SRC); + size = EQ_ELEM_DESC_GET(aeqe_desc, SIZE); + wrapped = EQ_ELEM_DESC_GET(aeqe_desc, WRAPPED); + + PMD_DRV_LOG(ERR, "index: %d, msg_id: %u, mod: %u, cmd: %u, seq_id: %u, seg_len: %u," + "src_func_idx: %u, event: %u, src: %u, size: %u, wrapped: %u", i, msg_id, + mod, cmd, seq_id, seg_len, src_func_idx, event, src, size, wrapped); + } +} + static int aeq_elem_handler(struct hinic3_eq *eq, uint32_t aeqe_desc, struct hinic3_aeq_elem *aeqe_pos, void *param) diff --git a/drivers/net/hinic3/base/hinic3_eqs.h b/drivers/net/hinic3/base/hinic3_eqs.h index 365c9cbf39..38de79a705 100644 --- a/drivers/net/hinic3/base/hinic3_eqs.h +++ b/drivers/net/hinic3/base/hinic3_eqs.h @@ -85,6 +85,8 @@ void hinic3_aeqs_free(struct hinic3_hwdev *hwdev); void hinic3_dump_aeq_info(struct hinic3_hwdev *hwdev); +void hinic3_dump_aeq_mbox_info(struct hinic3_hwdev *hwdev); + int hinic3_aeq_poll_msg(struct hinic3_eq *eq, uint32_t timeout, void *param); void hinic3_dev_handle_aeq_event(struct hinic3_hwdev *hwdev, void *param); diff --git a/drivers/net/hinic3/base/hinic3_mbox.c b/drivers/net/hinic3/base/hinic3_mbox.c index c4ca2ed7ba..03aa5cfbb2 100644 --- a/drivers/net/hinic3/base/hinic3_mbox.c +++ b/drivers/net/hinic3/base/hinic3_mbox.c @@ -711,6 +711,35 @@ send_tlp_mbox_seg(struct hinic3_mbox *func_to_func, uint64_t header, uint16_t ds return 0; } +static void hinic3_record_mbox_info(struct hinic3_mbox *func_to_func, + enum hinic3_mod_type mod, uint16_t cmd, + uint8_t msg_id) +{ + struct rte_pci_device *pci_dev = NULL; + struct rte_eth_dev *eth_dev = NULL; + struct mbox_send_info send_mbox; + + eth_dev = &rte_eth_devices[func_to_func->hwdev->port_id]; + pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev); + send_mbox.cmd = cmd; + send_mbox.mod = mod; + send_mbox.port = func_to_func->hwdev->port_id; + send_mbox.send_msg_id = msg_id; + send_mbox.func_id = pci_dev->addr.function; + send_mbox.devid = pci_dev->addr.devid; + send_mbox.bus = pci_dev->addr.bus; + + uint8_t pos = (func_to_func->save_mbox->start + func_to_func->save_mbox->count) % + HINIC3_MBOX_SAVE_NUM; + func_to_func->save_mbox->send_info[pos] = send_mbox; + + if (func_to_func->save_mbox->count < HINIC3_MBOX_SAVE_NUM) + func_to_func->save_mbox->count++; + else + func_to_func->save_mbox->start = (func_to_func->save_mbox->start + 1) % + HINIC3_MBOX_SAVE_NUM; +} + static int send_mbox_to_func(struct hinic3_mbox *func_to_func, enum hinic3_mod_type mod, struct hinic3_handler_info *handler_info, struct mbox_msg_info *msg_info) @@ -724,6 +753,9 @@ send_mbox_to_func(struct hinic3_mbox *func_to_func, enum hinic3_mod_type mod, rsp_aeq_id = HINIC3_MBOX_RSP_MSG_AEQ; + hinic3_record_mbox_info(func_to_func, mod, handler_info->cmd, + msg_info->msg_id); + /* Set the header message. */ header = HINIC3_MSG_HEADER_SET(handler_info->in_size, MSG_LEN) | HINIC3_MSG_HEADER_SET(mod, MODULE) | @@ -782,6 +814,9 @@ send_tlp_mbox_to_func(struct hinic3_mbox *func_to_func, enum hinic3_mod_type mod rsp_aeq_id = HINIC3_MBOX_RSP_MSG_AEQ; + hinic3_record_mbox_info(func_to_func, mod, handler_info->cmd, + msg_info->msg_id); + /* Set the header message. */ header = HINIC3_MSG_HEADER_SET(MBOX_TLP_HEADER_SZ, MSG_LEN) | HINIC3_MSG_HEADER_SET(MBOX_TLP_HEADER_SZ, SEG_LEN) | @@ -873,6 +908,7 @@ hinic3_mbox_to_func(struct hinic3_mbox *func_to_func, enum hinic3_mod_type mod, goto send_err; } + func_to_func->mbox_send_cnt++; /* Wait for the response message. */ time = timeout ? timeout : HINIC3_MBOX_COMP_TIME; aeq = &func_to_func->hwdev->aeqs->aeq[HINIC3_MBOX_RSP_MSG_AEQ]; @@ -880,15 +916,18 @@ hinic3_mbox_to_func(struct hinic3_mbox *func_to_func, enum hinic3_mod_type mod, if (err) { set_mbox_to_func_event(func_to_func, EVENT_TIMEOUT); PMD_DRV_LOG(ERR, "Send mailbox message time out"); + hinic3_dump_aeq_mbox_info(func_to_func->hwdev); err = -ETIMEDOUT; goto send_err; } + func_to_func->mbox_ack_cnt++; /* Check whether mod and command of the rsp message match the sent message. */ if (mod != mbox_for_resp->mod || handler_info->cmd != mbox_for_resp->cmd) { PMD_DRV_LOG(ERR, "Invalid response mbox message, mod: 0x%x, cmd: 0x%x, expect mod: 0x%x, cmd: 0x%x", mbox_for_resp->mod, mbox_for_resp->cmd, mod, handler_info->cmd); + hinic3_dump_aeq_mbox_info(func_to_func->hwdev); err = -EFAULT; goto send_err; } @@ -1129,6 +1168,16 @@ hinic3_func_to_func_init(struct hinic3_hwdev *hwdev) hwdev->func_to_func = func_to_func; func_to_func->hwdev = hwdev; + + struct save_mbox_info *save_mbox = rte_zmalloc("save_mbox_info", + sizeof(struct save_mbox_info), 0); + if (!save_mbox) { + err = -ENOMEM; + goto alloc_save_mbox_err; + } + + func_to_func->save_mbox = save_mbox; + rte_spinlock_init(&func_to_func->mbox_lock); /* Alloc the memory required by the mailbox. */ @@ -1170,6 +1219,9 @@ hinic3_func_to_func_init(struct hinic3_hwdev *hwdev) free_mbox_info(func_to_func->mbox_send); alloc_mbox_for_send_err: + rte_free(save_mbox); +alloc_save_mbox_err: + hwdev->func_to_func = NULL; rte_free(func_to_func); return err; @@ -1184,5 +1236,6 @@ hinic3_func_to_func_free(struct hinic3_hwdev *hwdev) free_mbox_tlp_buffer(func_to_func); free_mbox_info(func_to_func->mbox_resp); free_mbox_info(func_to_func->mbox_send); + rte_free(func_to_func->save_mbox); rte_free(func_to_func); } diff --git a/drivers/net/hinic3/base/hinic3_mbox.h b/drivers/net/hinic3/base/hinic3_mbox.h index df6de062e2..c0ee700ae6 100644 --- a/drivers/net/hinic3/base/hinic3_mbox.h +++ b/drivers/net/hinic3/base/hinic3_mbox.h @@ -15,6 +15,8 @@ #define HINIC3_MAX_PF_FUNCS 32 +#define HINIC3_MBOX_SAVE_NUM 20 + /* Message header define. */ #define HINIC3_MSG_HEADER_SRC_GLB_FUNC_IDX_SHIFT 0 #define HINIC3_MSG_HEADER_STATUS_SHIFT 13 @@ -150,8 +152,25 @@ enum hinic3_mbox_cb_state { HINIC3_PPF_TO_PF_MBOX_CB_RUNNING }; +struct mbox_send_info { + uint16_t cmd; + enum hinic3_mod_type mod; + uint8_t send_msg_id; + uint8_t port; + uint8_t func_id; + uint8_t devid; + uint8_t bus; +}; + +struct save_mbox_info { + struct mbox_send_info send_info[HINIC3_MBOX_SAVE_NUM]; + uint8_t start; + uint8_t count; +}; + struct hinic3_mbox { struct hinic3_hwdev *hwdev; + struct save_mbox_info *save_mbox; struct hinic3_send_mbox send_mbox; @@ -163,6 +182,9 @@ struct hinic3_mbox { enum mbox_event_state event_flag; /* Lock for mbox event flag. */ rte_spinlock_t mbox_lock; + + uint64_t mbox_send_cnt; + uint64_t mbox_ack_cnt; }; int hinic3_mbox_func_aeqe_handler(struct hinic3_hwdev *hwdev, uint8_t *header, -- 2.33.0
