Threads registered via rte_thread_register() are assigned a valid
lcore_id by eal_lcore_non_eal_allocate(), but their core_index in
lcore_config is left at -1. This value was set during rte_eal_cpu_init()
for lcores with ROLE_OFF (undetected CPUs) and is never updated when the
lcore is later allocated to a non-EAL thread.
As a result, rte_lcore_index() returns -1 for registered non-EAL
threads. Libraries that use rte_lcore_index() to select per-lcore
caches fall back to a shared global path when it returns -1, causing
severe contention under concurrent access from multiple registered
threads.
A concrete example is the mlx5 indexed memory pool (mlx5_ipool), which
uses rte_lcore_index() in mlx5_ipool_malloc_cache() to select a per-core
cache slot. When core_index is -1, all registered threads are funneled
into a single shared slot protected by a spinlock. In testing with VPP
(which registers worker threads via rte_thread_register()), this caused
async flow rule insertion throughput to drop from ~6.4M rules/sec to
~1.2M rules/sec with 4 workers -- a 5x regression attributable entirely
to spinlock contention in the ipool allocator.
Fix by tracking currently allocated core_index values in a bitset and
assigning non-EAL threads the first free index. Keep the bitset in sync
when initial EAL lcores are configured, when EAL core-list parsing
remaps lcores, and when non-EAL registration fails or releases an lcore.
Fixes: 5c307ba2a5b1 ("eal: register non-EAL threads as lcores")
Signed-off-by: Maxime Peim <[email protected]>
---
v2:
- Track allocated core_index values with a bitset instead of deriving
the next non-EAL index from lcore_count, avoiding duplicate indices
after non-EAL lcore release.
- Keep the bitset in sync when default EAL lcores are discovered, when
EAL lcore options remap the active set, and when non-EAL lcore
registration rolls back or releases an lcore.
lib/eal/common/eal_common_lcore.c | 19 ++++++++++++++++++-
lib/eal/common/eal_common_options.c | 5 +++++
lib/eal/common/eal_private.h | 3 +++
3 files changed, 26 insertions(+), 1 deletion(-)
diff --git a/lib/eal/common/eal_common_lcore.c
b/lib/eal/common/eal_common_lcore.c
index 39411f9370..b5f59a6380 100644
--- a/lib/eal/common/eal_common_lcore.c
+++ b/lib/eal/common/eal_common_lcore.c
@@ -6,8 +6,9 @@
#include <stdlib.h>
#include <string.h>
-#include <rte_common.h>
+#include <rte_bitset.h>
#include <rte_branch_prediction.h>
+#include <rte_common.h>
#include <rte_errno.h>
#include <rte_lcore.h>
#include <rte_log.h>
@@ -184,6 +185,9 @@ rte_eal_cpu_init(void)
/* By default, lcore 1:1 map to cpu id */
CPU_SET(lcore_id, &lcore_config[lcore_id].cpuset);
+ /* This is the first time we discover the lcores, so the bitset
should be zeroed */
+ rte_bitset_set(config->core_indices, count);
+
/* By default, each detected core is enabled */
config->lcore_role[lcore_id] = ROLE_RTE;
lcore_config[lcore_id].core_role = ROLE_RTE;
@@ -373,11 +377,20 @@ eal_lcore_non_eal_allocate(void)
struct lcore_callback *callback;
struct lcore_callback *prev;
unsigned int lcore_id;
+ int core_index = -1;
rte_rwlock_write_lock(&lcore_lock);
+ core_index = rte_bitset_find_first_clear(cfg->core_indices,
RTE_MAX_LCORE);
+ if (core_index == -1) {
+ EAL_LOG(DEBUG, "No core_index available.");
+ lcore_id = RTE_MAX_LCORE;
+ goto out;
+ }
for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
if (cfg->lcore_role[lcore_id] != ROLE_OFF)
continue;
+ rte_bitset_set(cfg->core_indices, core_index);
+ lcore_config[lcore_id].core_index = core_index;
cfg->lcore_role[lcore_id] = ROLE_NON_EAL;
cfg->lcore_count++;
break;
@@ -399,6 +412,8 @@ eal_lcore_non_eal_allocate(void)
}
EAL_LOG(DEBUG, "Initialization refused for lcore %u.",
lcore_id);
+ rte_bitset_clear(cfg->core_indices,
lcore_config[lcore_id].core_index);
+ lcore_config[lcore_id].core_index = -1;
cfg->lcore_role[lcore_id] = ROLE_OFF;
cfg->lcore_count--;
lcore_id = RTE_MAX_LCORE;
@@ -420,6 +435,8 @@ eal_lcore_non_eal_release(unsigned int lcore_id)
goto out;
TAILQ_FOREACH(callback, &lcore_callbacks, next)
callback_uninit(callback, lcore_id);
+ rte_bitset_clear(cfg->core_indices, lcore_config[lcore_id].core_index);
+ lcore_config[lcore_id].core_index = -1;
cfg->lcore_role[lcore_id] = ROLE_OFF;
cfg->lcore_count--;
out:
diff --git a/lib/eal/common/eal_common_options.c
b/lib/eal/common/eal_common_options.c
index 290386dc63..8f35fb376d 100644
--- a/lib/eal/common/eal_common_options.c
+++ b/lib/eal/common/eal_common_options.c
@@ -891,6 +891,7 @@ eal_parse_service_coremask(const char *coremask)
if (coremask[i] != '0')
return -1;
+ rte_bitset_clear_all(cfg->core_indices, RTE_MAX_LCORE);
for (; idx < RTE_MAX_LCORE; idx++)
lcore_config[idx].core_index = -1;
@@ -917,6 +918,7 @@ update_lcore_config(const rte_cpuset_t *cpuset, bool remap,
uint16_t remap_base)
int ret = 0;
/* set everything to disabled first, then set up values */
+ rte_bitset_clear_all(cfg->core_indices, RTE_MAX_LCORE);
for (i = 0; i < RTE_MAX_LCORE; i++) {
cfg->lcore_role[i] = ROLE_OFF;
lcore_config[i].core_index = -1;
@@ -946,6 +948,7 @@ update_lcore_config(const rte_cpuset_t *cpuset, bool remap,
uint16_t remap_base)
continue;
}
+ rte_bitset_set(cfg->core_indices, count);
cfg->lcore_role[lcore_id] = ROLE_RTE;
lcore_config[lcore_id].core_index = count;
CPU_ZERO(&lcore_config[lcore_id].cpuset);
@@ -1368,6 +1371,7 @@ eal_parse_lcores(const char *lcores)
CPU_ZERO(&cpuset);
/* Reset lcore config */
+ rte_bitset_clear_all(cfg->core_indices, RTE_MAX_LCORE);
for (idx = 0; idx < RTE_MAX_LCORE; idx++) {
cfg->lcore_role[idx] = ROLE_OFF;
lcore_config[idx].core_index = -1;
@@ -1432,6 +1436,7 @@ eal_parse_lcores(const char *lcores)
set_count--;
if (cfg->lcore_role[idx] != ROLE_RTE) {
+ rte_bitset_set(cfg->core_indices, count);
lcore_config[idx].core_index = count;
cfg->lcore_role[idx] = ROLE_RTE;
count++;
diff --git a/lib/eal/common/eal_private.h b/lib/eal/common/eal_private.h
index e032dd10c9..2b7a4fddbc 100644
--- a/lib/eal/common/eal_private.h
+++ b/lib/eal/common/eal_private.h
@@ -11,6 +11,7 @@
#include <sys/queue.h>
#include <dev_driver.h>
+#include <rte_bitset.h>
#include <rte_lcore.h>
#include <rte_log.h>
#include <rte_memory.h>
@@ -44,6 +45,8 @@ extern struct lcore_config lcore_config[RTE_MAX_LCORE];
* The global RTE configuration structure.
*/
struct rte_config {
+ RTE_BITSET_DECLARE(core_indices,
+ RTE_MAX_LCORE); /**< bitset of currently allocated
core_indices */
uint32_t main_lcore; /**< Id of the main lcore */
uint32_t lcore_count; /**< Number of available logical cores. */
uint32_t numa_node_count; /**< Number of detected NUMA nodes. */
--
2.43.0
