From: billyom <billy.o.mah...@intel.com> Previously if there is no available (non-isolated) pmd on the numa node for a port then the port is not polled at all. This can result in a non-operational system until such time as nics are physically repositioned. It is preferable to operate with a pmd on the 'wrong' numa node albeit with lower performance. Local pmds are still chosen when available.
Signed-off-by: Billy O'Mahony <billy.o.mah...@intel.com> --- lib/dpif-netdev.c | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c index 30907b7..6d57d8f 100644 --- a/lib/dpif-netdev.c +++ b/lib/dpif-netdev.c @@ -3070,10 +3070,13 @@ rr_numa_list_lookup(struct rr_numa_list *rr, int numa_id) } static void -rr_numa_list_populate(struct dp_netdev *dp, struct rr_numa_list *rr) +rr_numa_list_populate(struct dp_netdev *dp, struct rr_numa_list *rr, + int *all_numa_ids, unsigned all_numa_ids_sz, + int *num_ids_written) { struct dp_netdev_pmd_thread *pmd; struct rr_numa *numa; + unsigned idx = 0; hmap_init(&rr->numas); @@ -3091,7 +3094,11 @@ rr_numa_list_populate(struct dp_netdev *dp, struct rr_numa_list *rr) numa->n_pmds++; numa->pmds = xrealloc(numa->pmds, numa->n_pmds * sizeof *numa->pmds); numa->pmds[numa->n_pmds - 1] = pmd; + + all_numa_ids[idx % all_numa_ids_sz] = pmd->numa_id; + idx++; } + *num_ids_written = idx; } static struct dp_netdev_pmd_thread * @@ -3123,8 +3130,14 @@ rxq_scheduling(struct dp_netdev *dp, bool pinned) OVS_REQUIRES(dp->port_mutex) { struct dp_netdev_port *port; struct rr_numa_list rr; + int all_numa_ids [64]; + int all_numa_ids_sz = sizeof all_numa_ids / sizeof all_numa_ids[0]; + unsigned all_numa_ids_idx = 0; + int all_numa_ids_max_idx = 0; + int num_numa_ids = 0; - rr_numa_list_populate(dp, &rr); + rr_numa_list_populate(dp, &rr, all_numa_ids, all_numa_ids_sz, &num_numa_ids); + all_numa_ids_max_idx = MIN(num_numa_ids - 1, all_numa_ids_sz - 1); HMAP_FOR_EACH (port, node, &dp->ports) { struct rr_numa *numa; @@ -3155,10 +3168,24 @@ rxq_scheduling(struct dp_netdev *dp, bool pinned) OVS_REQUIRES(dp->port_mutex) } } else if (!pinned && q->core_id == OVS_CORE_UNSPEC) { if (!numa) { + if (all_numa_ids_max_idx < 0) { + VLOG_ERR("There are no pmd threads. " + "Is pmd-cpu-mask set to zero?"); + continue; + } VLOG_WARN("There's no available (non isolated) pmd thread " "on numa node %d. Queue %d on port \'%s\' will " - "not be polled.", - numa_id, qid, netdev_get_name(port->netdev)); + "be assigned to a pmd on numa node %d. Expect " + "reduced performance.", + numa_id, qid, netdev_get_name(port->netdev), + all_numa_ids[all_numa_ids_idx]); + numa_id = all_numa_ids[all_numa_ids_idx]; + numa = rr_numa_list_lookup(&rr, numa_id); + q->pmd = rr_numa_get_pmd(numa); + all_numa_ids_idx++; + if (all_numa_ids_idx > all_numa_ids_max_idx) { + all_numa_ids_idx = 0; + } } else { q->pmd = rr_numa_get_pmd(numa); } -- 2.7.4 _______________________________________________ dev mailing list d...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-dev