From: Jun Yang <[email protected]> The event queue previously stored a single owning event port and each DPCON kept a single channel_index, so a queue could be linked to only one port and the static dequeue channel could be added to only one DPIO at a time. This prevented linking the same event queue from several ports.
Track the DPCON static dequeue channel per DPIO by replacing the single channel_index with ch_idx[]/dpio_idx[] arrays and a ch_idx_num count (bounded by DPAA2_DPCON_MAX_CH_IDX_NUM). On link, add a channel for the linking DPIO and record its index; on unlink, look up the channel by DPIO index and remove it, compacting the arrays. Replace the exclusive evq_info->event_port ownership with a per-queue link_num reference count and a per-port evq_map[] that resolves an event queue id to its evq_info during enqueue. This drops the old per-enqueue owner check in favour of a direct map lookup. Add range checks on event queue id in queue setup and Rx queue attach, verify the queue index matches its event_queue_id, reject linking a queue configured as single link when already linked, and return specific error codes. Lower a couple of noisy messages from ERR/WARN to WARN/DEBUG. Update the scheduler DPCI receive and drain paths to the new per-DPIO channel index tracking. Signed-off-by: Jun Yang <[email protected]> --- drivers/bus/fslmc/portal/dpaa2_hw_pvt.h | 6 +- drivers/event/dpaa2/dpaa2_eventdev.c | 207 ++++++++++++++---------- drivers/event/dpaa2/dpaa2_eventdev.h | 3 +- drivers/event/dpaa2/dpaa2_scheduler.c | 10 +- 4 files changed, 140 insertions(+), 86 deletions(-) diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h b/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h index a0b1d2abfa..f67f2283db 100644 --- a/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h +++ b/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h @@ -283,6 +283,8 @@ struct dpaa2_dpci_dev { struct dpaa2_queue *tx_queue; }; +#define DPAA2_DPCON_MAX_CH_IDX_NUM 16 + struct dpaa2_dpcon_dev { TAILQ_ENTRY(dpaa2_dpcon_dev) next; struct fsl_mc_io dpcon; @@ -291,7 +293,9 @@ struct dpaa2_dpcon_dev { uint32_t dpcon_id; uint16_t qbman_ch_id; uint8_t num_priorities; - uint8_t channel_index; + uint8_t ch_idx[DPAA2_DPCON_MAX_CH_IDX_NUM]; + uint16_t dpio_idx[DPAA2_DPCON_MAX_CH_IDX_NUM]; + uint8_t ch_idx_num; }; /* DPCON channel pool, shared by the net rx-interrupt path and the event PMD */ diff --git a/drivers/event/dpaa2/dpaa2_eventdev.c b/drivers/event/dpaa2/dpaa2_eventdev.c index cd2b00c3c9..4c20e26870 100644 --- a/drivers/event/dpaa2/dpaa2_eventdev.c +++ b/drivers/event/dpaa2/dpaa2_eventdev.c @@ -47,14 +47,21 @@ dpaa2_eventdev_attach_eth_rxq(const struct rte_eth_dev *dev, const struct rte_event_eth_rx_adapter_queue_conf *queue_conf) { int ret; - uint8_t ev_qid = queue_conf->ev.queue_id; - uint16_t idx = priv->evq_info[ev_qid].dpni_rxq_num; - struct dpaa2_dpcon_dev *dpcon = priv->evq_info[ev_qid].dpcon; + uint16_t ev_qid = queue_conf->ev.queue_id, idx; + struct dpaa2_dpcon_dev *dpcon; + + if (ev_qid >= priv->max_event_queues) { + DPAA2_EVENTDEV_ERR("Invalid event queue ID(%d) >= %d", + ev_qid, priv->max_event_queues); + return -EINVAL; + } + idx = priv->evq_info[ev_qid].dpni_rxq_num; if (idx >= DPAA2_EVENT_MAX_QUEUE_FLOWS) { DPAA2_EVENTDEV_ERR("Too many flows to attach eventq%d", ev_qid); return -ENOMEM; } + dpcon = priv->evq_info[ev_qid].dpcon; ret = dpaa2_eth_eventq_attach(dev, rxq_id, dpcon, queue_conf); if (ret) { DPAA2_EVENTDEV_ERR("Event queue attach failed: err(%d)", ret); @@ -114,8 +121,6 @@ dpaa2_eventdev_enqueue_burst(void *port, const struct rte_event ev[], { struct dpaa2_port *dpaa2_portal = port; struct dpaa2_dpio_dev *dpio_dev; - struct rte_eventdev *eventdev = dpaa2_portal->eventdev; - struct dpaa2_eventdev *priv = eventdev->data->dev_private; struct dpaa2_eventq *evq_info; uint32_t queue_id, retry_count, loop, frames_to_send, i; struct qbman_swp *swp; @@ -145,13 +150,13 @@ dpaa2_eventdev_enqueue_burst(void *port, const struct rte_event ev[], event = &ev[num_tx + loop]; queue_id = event->queue_id; if (unlikely(queue_id >= DPAA2_EVENT_MAX_QUEUES || - priv->evq_info[queue_id].event_port != dpaa2_portal)) { - DPAA2_EVENTDEV_ERR("Event queue%d is not handled by port%d", + !dpaa2_portal->evq_map[queue_id])) { + DPAA2_EVENTDEV_ERR("Event queue%d not linked to port%d", queue_id, dpaa2_portal->port_id); ret = -EINVAL; goto send_partial; } - evq_info = &priv->evq_info[queue_id]; + evq_info = dpaa2_portal->evq_map[queue_id]; if (unlikely(event->sched_type >= DPAA2_EVENTQ_LINK_CONF_MAX || !evq_info->dpci_txqs[event->sched_type])) { DPAA2_EVENTDEV_ERR("Event sched_type(%d) is not supported", @@ -606,20 +611,32 @@ dpaa2_eventdev_queue_setup(struct rte_eventdev *dev, uint8_t queue_id, const struct rte_event_queue_conf *queue_conf) { struct dpaa2_eventdev *priv = dev->data->dev_private; - struct dpaa2_eventq *evq_info = &priv->evq_info[queue_id]; + struct dpaa2_eventq *evq_info; EVENTDEV_INIT_FUNC_TRACE(); + if (queue_id >= priv->max_event_queues) { + DPAA2_EVENTDEV_ERR("Invalid queue ID(%d) >= %d", queue_id, + priv->max_event_queues); + return -EINVAL; + } + evq_info = &priv->evq_info[queue_id]; + if (queue_id != evq_info->event_queue_id) { + DPAA2_EVENTDEV_ERR("Queue index(%d) != event queue ID(%d)?", + queue_id, evq_info->event_queue_id); + return -EACCES; + } + switch (queue_conf->schedule_type) { case RTE_SCHED_TYPE_PARALLEL: case RTE_SCHED_TYPE_ATOMIC: break; default: - DPAA2_EVENTDEV_ERR("Schedule type is not supported."); - return -1; + DPAA2_EVENTDEV_ERR("Schedule type(%d) is not supported.", + queue_conf->schedule_type); + return -ENOTSUP; } evq_info->event_queue_cfg = queue_conf->event_queue_cfg; - evq_info->event_queue_id = queue_id; return 0; } @@ -657,40 +674,83 @@ dpaa2_eventdev_port_unlink(struct rte_eventdev *dev, void *port, struct dpaa2_dpio_dev *dpio_dev = NULL; struct dpaa2_eventq *evq_info; struct qbman_swp *swp; - uint8_t total_queues[DPAA2_EVENT_MAX_QUEUES]; struct dpaa2_eventq *evq_infos[DPAA2_EVENT_MAX_QUEUES]; + uint16_t pos[DPAA2_DPCON_MAX_CH_IDX_NUM], ch_idx; EVENTDEV_INIT_FUNC_TRACE(); RTE_SET_USED(dev); - if (!queues) { - queues = total_queues; + dpio_dev = dpaa2_portal->dpio_dev; + if (!dpio_dev) + return 0; + + swp = dpio_dev->sw_portal; + if (!queues) nb_unlinks = dpaa2_portal->num_linked_evq; - for (i = 0; i < nb_unlinks; i++) - queues[i] = dpaa2_portal->evq_info[i]->event_queue_id; + + for (i = 0; i < nb_unlinks; i++) { + if (!queues) { + evq_info = dpaa2_portal->evq_info[i]; + goto map_check; + } + evq_info = NULL; + for (j = 0; j < dpaa2_portal->num_linked_evq; j++) { + if (queues[i] == dpaa2_portal->evq_info[j]->event_queue_id) { + evq_info = dpaa2_portal->evq_info[j]; + break; + } + } + if (!evq_info) { + DPAA2_EVENTDEV_WARN("Event port%d doesn't handle queues[%d](%d)", + dpaa2_portal->port_id, i, queues[i]); + return -EINVAL; + } +map_check: + if (dpaa2_portal->evq_map[evq_info->event_queue_id] != evq_info) { + DPAA2_EVENTDEV_ERR("Event queue mapping[%d] mismatch.", + evq_info->event_queue_id); + return -EINVAL; + } + for (j = 0; j < evq_info->dpcon->ch_idx_num; j++) { + if (evq_info->dpcon->dpio_idx[j] == dpio_dev->index) + break; + } + if (j == evq_info->dpcon->ch_idx_num) { + DPAA2_EVENTDEV_ERR("Event queue.%d was NOT attached to dpio.%d.", + evq_info->event_queue_id, dpio_dev->index); + return -EINVAL; + } + pos[i] = j; + evq_infos[i] = evq_info; } for (i = 0; i < nb_unlinks; i++) { + evq_info = evq_infos[i]; + ch_idx = evq_info->dpcon->ch_idx[pos[i]]; + qbman_swp_push_set(swp, ch_idx, 0); + ret = dpio_remove_static_dequeue_channel(dpio_dev->dpio, + 0, dpio_dev->token, evq_info->dpcon->dpcon_id); + if (ret) { + DPAA2_EVENTDEV_ERR("Failed(%d) to remove channel (dpio.%d/dpcon.%d)", + ret, dpio_dev->index, evq_info->dpcon->dpcon_id); + break; + } + memmove(&evq_info->dpcon->ch_idx[pos[i]], + &evq_info->dpcon->ch_idx[pos[i] + 1], + evq_info->dpcon->ch_idx_num - (pos[i] + 1)); + memmove(&evq_info->dpcon->dpio_idx[pos[i]], + &evq_info->dpcon->dpio_idx[pos[i] + 1], + (evq_info->dpcon->ch_idx_num - (pos[i] + 1)) * sizeof(uint16_t)); + evq_info->dpcon->ch_idx_num--; for (j = 0; j < dpaa2_portal->num_linked_evq; j++) { - evq_info = dpaa2_portal->evq_info[j]; - if (evq_info && evq_info->event_queue_id == queues[i]) { - dpio_dev = dpaa2_portal->dpio_dev; - swp = dpio_dev->sw_portal; - qbman_swp_push_set(swp, evq_info->dpcon->channel_index, 0); - ret = dpio_remove_static_dequeue_channel(dpio_dev->dpio, - 0, dpio_dev->token, evq_info->dpcon->dpcon_id); - if (ret) - DPAA2_EVENTDEV_WARN("Failed(%d) to remove channel", ret); - evq_info->event_port = NULL; + if (dpaa2_portal->evq_info[j] == evq_info) { dpaa2_portal->evq_info[j] = NULL; - num++; break; } } - if (j == dpaa2_portal->num_linked_evq) { - DPAA2_EVENTDEV_WARN("%s: Event port%d doesn't handle queue[%d]:%d", - __func__, dpaa2_portal->port_id, i, queues[i]); - } + dpaa2_portal->evq_map[evq_info->event_queue_id] = NULL; + evq_info->link_num--; + num++; } for (i = 0; i < dpaa2_portal->num_linked_evq; i++) { @@ -714,75 +774,60 @@ dpaa2_eventdev_port_link(struct rte_eventdev *dev, void *port, uint16_t nb_links) { struct dpaa2_eventdev *priv = dev->data->dev_private; - struct dpaa2_port *dpaa2_portal = port, *ev_port; + struct dpaa2_port *dpaa2_portal = port; struct dpaa2_dpio_dev *dpio_dev = dpaa2_portal->dpio_dev; struct dpaa2_eventq *evq_info; - int ret = 0; uint16_t i; - uint8_t channel_index; + uint8_t ch_idx; + int ret = 0; EVENTDEV_INIT_FUNC_TRACE(); RTE_SET_USED(priorities); - if (dpaa2_portal->num_linked_evq > 0) { - ret = rte_event_port_unlink(dev->data->dev_id, - dpaa2_portal->port_id, NULL, 0); - if (ret) - return ret; - } - for (i = 0; i < nb_links; i++) { + if (queues[i] >= priv->max_event_queues) { + DPAA2_EVENTDEV_ERR("Event port%d: queues[%d](%d) >= %d", + dpaa2_portal->port_id, i, queues[i], priv->max_event_queues); + break; + } evq_info = &priv->evq_info[queues[i]]; - if (!evq_info->valid) { - DPAA2_EVENTDEV_WARN("%s: Event port%d.queue%d is not valid!", - __func__, dpaa2_portal->port_id, queues[i]); - return -EINVAL; + if (evq_info->link_num > 0 && + (evq_info->event_queue_cfg & RTE_EVENT_QUEUE_CFG_SINGLE_LINK)) { + DPAA2_EVENTDEV_ERR("Event queue[%d] is configured as single link", + queues[i]); + break; } - if (evq_info->event_port) { - ev_port = evq_info->event_port; - DPAA2_EVENTDEV_WARN("%s: queue%d is occupied by event port%d", - __func__, queues[i], ev_port->port_id); - return -ENODEV; + if (evq_info->dpcon->ch_idx_num >= DPAA2_DPCON_MAX_CH_IDX_NUM) { + DPAA2_EVENTDEV_ERR("Too many channel to be added with dpcon(ID=%d)", + evq_info->dpcon->dpcon_id); + break; + } + if (dpaa2_portal->evq_map[queues[i]]) { + DPAA2_EVENTDEV_ERR("Event port%d: queue%d already mapped", + dpaa2_portal->port_id, queues[i]); + break; } - } - - for (i = 0; i < nb_links; i++) { - evq_info = &priv->evq_info[queues[i]]; ret = dpio_add_static_dequeue_channel(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token, - evq_info->dpcon->dpcon_id, &channel_index); + evq_info->dpcon->dpcon_id, &ch_idx); if (ret) { DPAA2_EVENTDEV_ERR("Event port%d.queue%d static dequeue config failed(%d)", dpaa2_portal->port_id, queues[i], ret); - goto err; + break; } - - qbman_swp_push_set(dpio_dev->sw_portal, channel_index, 1); - evq_info->dpcon->channel_index = channel_index; - dpaa2_portal->evq_info[dpaa2_portal->num_linked_evq] = evq_info; - evq_info->event_port = dpaa2_portal; dpaa2_portal->num_linked_evq++; - } + evq_info->dpcon->ch_idx[evq_info->dpcon->ch_idx_num] = ch_idx; + evq_info->dpcon->dpio_idx[evq_info->dpcon->ch_idx_num] = dpio_dev->index; + evq_info->dpcon->ch_idx_num++; + evq_info->link_num++; + dpaa2_portal->evq_map[queues[i]] = evq_info; -err: - if (!ret) - return nb_links; - - nb_links = i; - for (i = 0; i < nb_links; i++) { - evq_info = &priv->evq_info[queues[i]]; - evq_info->event_port = NULL; - qbman_swp_push_set(dpio_dev->sw_portal, evq_info->dpcon->channel_index, 0); - if (dpio_remove_static_dequeue_channel(dpio_dev->dpio, - 0, dpio_dev->token, evq_info->dpcon->dpcon_id)) { - DPAA2_EVENTDEV_ERR("Event port%d.queue%d remove failed!", - dpaa2_portal->port_id, queues[i]); - } - dpaa2_portal->num_linked_evq--; + qbman_swp_push_set(dpio_dev->sw_portal, ch_idx, 1); } - return ret; + + return i; } static int @@ -795,7 +840,7 @@ dpaa2_eventdev_port_setup(struct rte_eventdev *dev, uint8_t port_id, EVENTDEV_INIT_FUNC_TRACE(); if (dev->data->ports[port_id]) { - DPAA2_EVENTDEV_WARN("Event port%d exists!", port_id); + DPAA2_EVENTDEV_DEBUG("Event port%d exists!", port_id); dpaa2_eventdev_port_release(dev->data->ports[port_id]); } @@ -1231,7 +1276,7 @@ dpaa2_eventdev_create(const char *name, struct rte_vdev_device *vdev) /** TO DO: order schedule.*/ evq_info->valid = true; - evq_info->event_port = NULL; + evq_info->event_queue_id = priv->max_event_queues; priv->max_event_queues++; if (priv->max_event_queues >= DPAA2_EVENT_MAX_QUEUES) diff --git a/drivers/event/dpaa2/dpaa2_eventdev.h b/drivers/event/dpaa2/dpaa2_eventdev.h index adeb22bda6..c0ce42c83f 100644 --- a/drivers/event/dpaa2/dpaa2_eventdev.h +++ b/drivers/event/dpaa2/dpaa2_eventdev.h @@ -54,7 +54,7 @@ enum { #define DPAA2_EVENTQ_LINK_CONF_MAX 4 /** Max schedule type + 1.*/ struct dpaa2_eventq { int valid; - void *event_port; + uint8_t link_num; /* DPcon device */ struct dpaa2_dpcon_dev *dpcon; /* Attached DPCI device */ @@ -80,6 +80,7 @@ struct dpaa2_port { RTE_ATOMIC(uint32_t) flushing; struct dpaa2_eventq *evq_info[DPAA2_EVENT_MAX_QUEUES]; + struct dpaa2_eventq *evq_map[DPAA2_EVENT_MAX_QUEUES]; uint8_t port_id; uint8_t num_linked_evq; uint64_t timeout_us; diff --git a/drivers/event/dpaa2/dpaa2_scheduler.c b/drivers/event/dpaa2/dpaa2_scheduler.c index 6a4ec04aae..9e58b4253f 100644 --- a/drivers/event/dpaa2/dpaa2_scheduler.c +++ b/drivers/event/dpaa2/dpaa2_scheduler.c @@ -196,16 +196,18 @@ dpaa2_scheduler_dpci_recv(struct dpaa2_sch_dev *sch_dev, ret = dpio_add_static_dequeue_channel(dpio_dev->dpio, CMD_PRI_LOW, dpio_dev->token, dpcon_dev->dpcon_id, - &dpcon_dev->channel_index); + &dpcon_dev->ch_idx[dpcon_dev->ch_idx_num]); if (ret) { DPAA2_EVENTDEV_ERR("Failure(%d) adding dpcon%d to static dq channel", ret, dpcon_dev->dpcon_id); return 0; } + dpcon_dev->dpio_idx[dpcon_dev->ch_idx_num] = dpio_dev->index; - qbman_swp_push_set(swp, dpcon_dev->channel_index, 1); + qbman_swp_push_set(swp, dpcon_dev->ch_idx[dpcon_dev->ch_idx_num], 1); sch_dev->linked[cpu] = sch_dev->port_queue_num; sch_dev->dpio_dev[cpu] = dpio_dev; + dpcon_dev->ch_idx_num++; start_dq: while (num_pkts < nb_pkts) { @@ -562,7 +564,9 @@ dpaa2_scheduler_rx_core_push_drain(struct dpaa2_sch_dev *sch_dev, if (drain_num > 0) DPAA2_EVENTDEV_WARN("Drain %d buffer(s) from core%d", drain_num, core); - qbman_swp_push_set(dpio_dev->sw_portal, dpcon_dev->channel_index, 0); + qbman_swp_push_set(dpio_dev->sw_portal, + dpcon_dev->ch_idx[dpcon_dev->ch_idx_num - 1], 0); + dpcon_dev->ch_idx_num--; if (attach_num) (*attach_num)++; ret = dpio_remove_static_dequeue_channel(dpio_dev->dpio, -- 2.43.0
