event_register() added each Rx queue interrupt fd to the per-thread epoll set before the worker loop, while the queues were still disarmed. This works when the fd is assigned at queue setup and stays fixed, as on MSI-X NICs.
The dpaa2 PMD delivers Rx interrupts through a per-lcore QBMan portal and binds the fd when the interrupt is enabled, not at queue setup. Adding a queue to the epoll set before arming it then watches an fd that is not bound yet, so the lcore never wakes from rte_epoll_wait(). Register from rx_interrupt_wait(), after the queues are armed, and only once: keep the epoll entry installed for the lifetime of the loop, with the state in struct lcore_conf. NICs with a fixed per-queue fd are unaffected. The "RX interrupt won't enable" notice moves to the loop error path, where it is still logged once per lcore since intr_en latches off. Signed-off-by: Maxime Leroy <[email protected]> --- examples/l3fwd-power/main.c | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c index 70f76d202e..43a0509bd5 100644 --- a/examples/l3fwd-power/main.c +++ b/examples/l3fwd-power/main.c @@ -368,6 +368,7 @@ struct __rte_cache_aligned lcore_conf { struct rte_eth_dev_tx_buffer *tx_buffer[RTE_MAX_ETHPORTS]; lookup_struct_t * ipv4_lookup_struct; lookup_struct_t * ipv6_lookup_struct; + bool intr_registered; }; struct __rte_cache_aligned lcore_stats { @@ -953,6 +954,16 @@ rx_interrupt_wait(struct lcore_conf *qconf) if (ret != 0) return ret; + /* some PMDs expose the interrupt fd only once the queue is armed */ + if (!qconf->intr_registered) { + ret = event_register(qconf); + if (ret != 0) { + rx_intr_disable_all(qconf); + return ret; + } + qconf->intr_registered = true; + } + sleep_until_rx_interrupt(qconf->n_rx_queue, rte_lcore_id()); rx_intr_disable_all(qconf); return 0; @@ -970,7 +981,7 @@ static int main_intr_loop(__rte_unused void *dummy) struct lcore_rx_queue *rx_queue; uint32_t lcore_rx_idle_count = 0; uint32_t lcore_idle_hint = 0; - int intr_en = 0; + int intr_en = 1; int ret; const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / @@ -998,12 +1009,6 @@ static int main_intr_loop(__rte_unused void *dummy) lcore_id, portid, queueid); } - /* add into event wait list */ - if (event_register(qconf) == 0) - intr_en = 1; - else - RTE_LOG(INFO, L3FWD_POWER, "RX interrupt won't enable.\n"); - while (!is_done()) { stats[lcore_id].nb_iteration_looped++; @@ -1107,6 +1112,8 @@ static int main_intr_loop(__rte_unused void *dummy) if (ret == -EAGAIN) goto start_rx; if (ret != 0) { + RTE_LOG(INFO, L3FWD_POWER, + "RX interrupt won't enable.\n"); intr_en = 0; continue; } @@ -1260,7 +1267,7 @@ main_legacy_loop(__rte_unused void *dummy) enum freq_scale_hint_t lcore_scaleup_hint; uint32_t lcore_rx_idle_count = 0; uint32_t lcore_idle_hint = 0; - int intr_en = 0; + int intr_en = 1; int ret; const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US; @@ -1286,12 +1293,6 @@ main_legacy_loop(__rte_unused void *dummy) "rxqueueid=%" PRIu16 "\n", lcore_id, portid, queueid); } - /* add into event wait list */ - if (event_register(qconf) == 0) - intr_en = 1; - else - RTE_LOG(INFO, L3FWD_POWER, "RX interrupt won't enable.\n"); - while (!is_done()) { stats[lcore_id].nb_iteration_looped++; @@ -1428,6 +1429,8 @@ main_legacy_loop(__rte_unused void *dummy) if (ret == -EAGAIN) goto start_rx; if (ret != 0) { + RTE_LOG(INFO, L3FWD_POWER, + "RX interrupt won't enable.\n"); intr_en = 0; continue; } -- 2.43.0
