From: "McDaniel, Timothy" <timothy.mcdan...@intel.com>
Signed-off-by: McDaniel, Timothy <timothy.mcdan...@intel.com>
---
drivers/event/dlb/Makefile | 1 +
drivers/event/dlb/dlb.c | 517 +++++++++++++++++++++++++++++++++++++++++
drivers/event/dlb/meson.build | 3 +-
3 files changed, 520 insertions(+), 1 deletion(-)
create mode 100644 drivers/event/dlb/dlb.c
diff --git a/drivers/event/dlb/Makefile b/drivers/event/dlb/Makefile
index 203d54f..af92771 100644
--- a/drivers/event/dlb/Makefile
+++ b/drivers/event/dlb/Makefile
@@ -20,6 +20,7 @@ LIBABIVER := 1
EXPORT_MAP := rte_pmd_dlb_event_version.map
# library source files
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_DLB_EVENTDEV) += dlb.c
SRCS-$(CONFIG_RTE_LIBRTE_PMD_DLB_EVENTDEV) += dlb_selftest.c
SRCS-$(CONFIG_RTE_LIBRTE_PMD_DLB_EVENTDEV) += dlb_xstats.c
SRCS-$(CONFIG_RTE_LIBRTE_PMD_DLB_EVENTDEV) += rte_pmd_dlb.c
diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c
new file mode 100644
index 0000000..c635deb
--- /dev/null
+++ b/drivers/event/dlb/dlb.c
@@ -0,0 +1,517 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <sys/fcntl.h>
+#include <errno.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+#include <pthread.h>
+#include <nmmintrin.h>
+
+#include <rte_common.h>
+#include <rte_config.h>
+#include <rte_debug.h>
+#include <rte_log.h>
+#include <rte_dev.h>
+#include <rte_mbuf.h>
+#include <rte_ring.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+#include <rte_malloc.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+#include <rte_string_fns.h>
+#include <rte_prefetch.h>
+
+#include "dlb_priv.h"
+#include "dlb_iface.h"
+#include "dlb_inline_fns.h"
+
+/*
+ * Resources exposed to eventdev. Some values overridden at runtime using
+ * values returned by the DLB kernel driver.
+ */
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
+#error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
+#endif
+static struct rte_event_dev_info evdev_dlb_default_info = {
+ .driver_name = "", /* probe will set */
+ .min_dequeue_timeout_ns = DLB_MIN_DEQUEUE_TIMEOUT_NS,
+ .max_dequeue_timeout_ns = DLB_MAX_DEQUEUE_TIMEOUT_NS,
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV < DLB_MAX_NUM_LDB_QUEUES)
+ .max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
+#else
+ .max_event_queues = DLB_MAX_NUM_LDB_QUEUES,
+#endif
+ .max_event_queue_flows = DLB_MAX_NUM_FLOWS,
+ .max_event_queue_priority_levels = DLB_QID_PRIORITIES,
+ .max_event_priority_levels = DLB_QID_PRIORITIES,
+ .max_event_ports = DLB_MAX_NUM_LDB_PORTS,
+ .max_event_port_dequeue_depth = DLB_MAX_CQ_DEPTH,
+ .max_event_port_enqueue_depth = DLB_MAX_ENQUEUE_DEPTH,
+ .max_event_port_links = DLB_MAX_NUM_QIDS_PER_LDB_CQ,
+ .max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+ .max_single_link_event_port_queue_pairs = DLB_MAX_NUM_DIR_PORTS,
+ .event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS |
+ RTE_EVENT_DEV_CAP_EVENT_QOS |
+ RTE_EVENT_DEV_CAP_BURST_MODE |
+ RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED |
+ RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE |
+ RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES),
+};
+
+/* These functions will vary based on processor capabilities */
+static struct dlb_port_low_level_io_functions qm_mmio_fns;
+
+static int
+dlb_hw_query_resources(struct dlb_eventdev *dlb)
+{
+ struct dlb_hw_dev *handle = &dlb->qm_instance;
+ struct dlb_hw_resource_info *dlb_info = &handle->info;
+ int ret;
+
+ ret = dlb_iface_get_num_resources(handle,
+ &dlb->hw_rsrc_query_results);
+ if (ret) {
+ DLB_LOG_ERR("ioctl get dlb num resources, err=%d\n",
+ ret);
+ return ret;
+ }
+
+ /* Complete filling in device resource info returned to evdev app,
+ * overriding any default values.
+ * The capabilities (CAPs) were set at compile time.
+ */
+
+ evdev_dlb_default_info.max_event_queues =
+ dlb->hw_rsrc_query_results.num_ldb_queues;
+
+ evdev_dlb_default_info.max_event_ports =
+ dlb->hw_rsrc_query_results.num_ldb_ports;
+
+ evdev_dlb_default_info.max_num_events =
+ dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+ /* Save off values used when creating the scheduling domain. */
+
+ handle->info.num_sched_domains =
+ dlb->hw_rsrc_query_results.num_sched_domains;
+
+ handle->info.hw_rsrc_max.nb_events_limit =
+ dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+ handle->info.hw_rsrc_max.num_queues =
+ dlb->hw_rsrc_query_results.num_ldb_queues +
+ dlb->hw_rsrc_query_results.num_dir_ports;
+
+ handle->info.hw_rsrc_max.num_ldb_queues =
+ dlb->hw_rsrc_query_results.num_ldb_queues;
+
+ handle->info.hw_rsrc_max.num_ldb_ports =
+ dlb->hw_rsrc_query_results.num_ldb_ports;
+
+ handle->info.hw_rsrc_max.num_dir_ports =
+ dlb->hw_rsrc_query_results.num_dir_ports;
+
+ handle->info.hw_rsrc_max.reorder_window_size =
+ dlb->hw_rsrc_query_results.num_hist_list_entries;
+
+ rte_memcpy(dlb_info, &handle->info.hw_rsrc_max, sizeof(*dlb_info));
+
+ return 0;
+}
+
+/* Wrapper for string to int conversion. Substituted for atoi(...), which is
+ * unsafe.
+ */
+#define RTE_BASE_10 10
+int dlb_string_to_int(int *result, const char *str)
+{
+ long ret;
+
+ if (str == NULL || result == NULL)
+ return -EINVAL;
+
+ errno = 0;
+ ret = strtol(str, NULL, RTE_BASE_10);
+ if (errno)
+ return -errno;
+
+ /* long int and int may be different width for some architectures */
+ if (ret < INT_MIN || ret > INT_MAX)
+ return -EINVAL;
+
+ *result = ret;
+ return 0;
+}
+
+int
+set_numa_node(const char *key __rte_unused, const char *value, void *opaque)
+{
+ int *socket_id = opaque;
+ int ret;
+
+ ret = dlb_string_to_int(socket_id, value);
+ if (ret < 0)
+ return ret;
+
+ if (*socket_id > RTE_MAX_NUMA_NODES)
+ return -EINVAL;
+
+ return 0;
+}
+
+int
+set_max_num_events(const char *key __rte_unused,
+ const char *value,
+ void *opaque)
+{
+ int *max_num_events = opaque;
+ int ret;
+
+ if (value == NULL || opaque == NULL) {
+ DLB_LOG_ERR("NULL pointer\n");
+ return -EINVAL;
+ }
+
+ ret = dlb_string_to_int(max_num_events, value);
+ if (ret < 0)
+ return ret;
+
+ if (*max_num_events < 0 || *max_num_events > DLB_MAX_NUM_LDB_CREDITS) {
+ DLB_LOG_ERR("dlb: max_num_events must be between 0 and %d\n",
+ DLB_MAX_NUM_LDB_CREDITS);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+int
+set_num_dir_credits(const char *key __rte_unused,
+ const char *value,
+ void *opaque)
+{
+ int *num_dir_credits = opaque;
+ int ret;
+
+ if (value == NULL || opaque == NULL) {
+ DLB_LOG_ERR("NULL pointer\n");
+ return -EINVAL;
+ }
+
+ ret = dlb_string_to_int(num_dir_credits, value);
+ if (ret < 0)
+ return ret;
+
+ if (*num_dir_credits < 0 ||
+ *num_dir_credits > DLB_MAX_NUM_DIR_CREDITS) {
+ DLB_LOG_ERR("dlb: num_dir_credits must be between 0 and %d\n",
+ DLB_MAX_NUM_DIR_CREDITS);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int
+set_dev_id(const char *key __rte_unused,
+ const char *value,
+ void *opaque)
+{
+ int *dev_id = opaque;
+ int ret;
+
+ if (value == NULL || opaque == NULL) {
+ DLB_LOG_ERR("NULL pointer\n");
+ return -EINVAL;
+ }
+
+ ret = dlb_string_to_int(dev_id, value);
+ if (ret < 0)
+ return ret;
+
+ return 0;
+}
+
+static int
+set_defer_sched(const char *key __rte_unused,
+ const char *value,
+ void *opaque)
+{
+ int *defer_sched = opaque;
+
+ if (value == NULL || opaque == NULL) {
+ DLB_LOG_ERR("NULL pointer\n");
+ return -EINVAL;
+ }
+
+ if (strncmp(value, "on", 2) != 0) {
+ DLB_LOG_ERR("Invalid defer_sched argument \"%s\" (expected
\"on\")\n",
+ value);
+ return -EINVAL;
+ }
+
+ *defer_sched = 1;
+
+ return 0;
+}
+
+static int
+set_num_atm_inflights(const char *key __rte_unused,
+ const char *value,
+ void *opaque)
+{
+ int *num_atm_inflights = opaque;
+ int ret;
+
+ if (value == NULL || opaque == NULL) {
+ DLB_LOG_ERR("NULL pointer\n");
+ return -EINVAL;
+ }
+
+ ret = dlb_string_to_int(num_atm_inflights, value);
+ if (ret < 0)
+ return ret;
+
+ if (*num_atm_inflights < 0 ||
+ *num_atm_inflights > DLB_MAX_NUM_ATM_INFLIGHTS) {
+ DLB_LOG_ERR("dlb: atm_inflights must be between 0 and %d\n",
+ DLB_MAX_NUM_ATM_INFLIGHTS);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+
+static void
+dlb_qm_mmio_fn_init(void)
+{
+ /* Process-local function pointers for performing low level port i/o */
+
+ if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_MOVDIR64B))
+ qm_mmio_fns.pp_enqueue_four = dlb_movdir64b;
+ else
+ qm_mmio_fns.pp_enqueue_four = dlb_movntdq;
+}
+
+int
+dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+ const char *name,
+ struct dlb_devargs *dlb_args,
+ bool is_vdev)
+{
+ struct dlb_eventdev *dlb;
+ int err, i;
+
+ dlb = dev->data->dev_private;
+
+ dlb->event_dev = dev; /* backlink */
+ dlb->is_vdev = is_vdev; /* vdev or pf */
+
+ evdev_dlb_default_info.driver_name = name;
+
+ dlb->max_num_events_override = dlb_args->max_num_events;
+ dlb->num_dir_credits_override = dlb_args->num_dir_credits_override;
+ dlb->qm_instance.device_path_id = dlb_args->dev_id;
+ dlb->defer_sched = dlb_args->defer_sched;
+ dlb->num_atm_inflights_per_queue = dlb_args->num_atm_inflights;
+
+ /* Open the interface.
+ * For vdev mode, this means open the dlb kernel module.
+ */
+ err = dlb_iface_open(&dlb->qm_instance, name);
+ if (err < 0) {
+ DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+ err);
+ return err;
+ }
+
+ err = dlb_iface_get_device_version(&dlb->qm_instance, &dlb->revision);
+ if (err < 0) {
+ DLB_LOG_ERR("dlb: failed to get the device version, err=%d\n",
+ err);
+ return err;
+ }
+
+ err = dlb_hw_query_resources(dlb);
+ if (err) {
+ DLB_LOG_ERR("get resources err=%d for %s\n",
+ err, name);
+ return err;
+ }
+
+ err = dlb_iface_get_cq_poll_mode(&dlb->qm_instance, &dlb->poll_mode);
+ if (err < 0) {
+ DLB_LOG_ERR("dlb: failed to get the poll mode, err=%d\n",
+ err);
+ return err;
+ }
+
+ /* Complete xtstats runtime initialization */
+ err = dlb_xstats_init(dlb);
+ if (err) {
+ DLB_LOG_ERR("dlb: failed to init xstats, err=%d\n", err);
+ return err;
+ }
+
+ /* Initialize each port's token pop mode */
+ for (i = 0; i < DLB_MAX_NUM_PORTS; i++)
+ dlb->ev_ports[i].qm_port.token_pop_mode = AUTO_POP;
+
+ rte_spinlock_init(&dlb->qm_instance.resource_lock);
+
+ dlb_qm_mmio_fn_init();
+
+ dlb_iface_low_level_io_init(dlb);
+
+ dlb_entry_points_init(dev);
+
+ return 0;
+}
+
+int
+dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+ const char *name,
+ bool is_vdev)
+{
+ struct dlb_eventdev *dlb;
+ int err;
+
+RTE_SET_USED(is_vdev);
+
+ dlb = dev->data->dev_private;
+
+ evdev_dlb_default_info.driver_name = name;
+
+ err = dlb_iface_open(&dlb->qm_instance, name);
+ if (err < 0) {
+ DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+ err);
+ return err;
+ }
+
+ err = dlb_hw_query_resources(dlb);
+ if (err) {
+ DLB_LOG_ERR("get resources err=%d for %s\n",
+ err, name);
+ return err;
+ }
+
+ dlb_qm_mmio_fn_init();
+
+ dlb_iface_low_level_io_init(dlb);
+
+ dlb_entry_points_init(dev);
+
+ return 0;
+}
+
+int
+dlb_parse_params(const char *params,
+ const char *name,
+ struct dlb_devargs *dlb_args)
+{
+ int ret = 0;
+ static const char * const args[] = { NUMA_NODE_ARG,
+ DLB_MAX_NUM_EVENTS,
+ DLB_NUM_DIR_CREDITS,
+ DEV_ID_ARG,
+ DLB_DEFER_SCHED_ARG,
+ DLB_NUM_ATM_INFLIGHTS_ARG,
+ NULL };
+
+ if (params && params[0] != '\0') {
+ struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
+
+ if (!kvlist) {
+ RTE_LOG(INFO, PMD,
+ "Ignoring unsupported parameters when creating
device '%s'\n",
+ name);
+ } else {
+ int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
+ set_numa_node,
+ &dlb_args->socket_id);
+ if (ret != 0) {
+ DLB_LOG_ERR("%s: Error parsing numa node
parameter",
+ name);
+ rte_kvargs_free(kvlist);
+ return ret;
+ }
+
+ ret = rte_kvargs_process(kvlist, DLB_MAX_NUM_EVENTS,
+ set_max_num_events,
+ &dlb_args->max_num_events);
+ if (ret != 0) {
+ DLB_LOG_ERR("%s: Error parsing max_num_events
parameter",
+ name);
+ rte_kvargs_free(kvlist);
+ return ret;
+ }
+
+ ret = rte_kvargs_process(kvlist,
+ DLB_NUM_DIR_CREDITS,
+ set_num_dir_credits,
+ &dlb_args->num_dir_credits_override);
+ if (ret != 0) {
+ DLB_LOG_ERR("%s: Error parsing num_dir_credits
parameter",
+ name);
+ rte_kvargs_free(kvlist);
+ return ret;
+ }
+
+ ret = rte_kvargs_process(kvlist, DEV_ID_ARG,
+ set_dev_id,
+ &dlb_args->dev_id);
+ if (ret != 0) {
+ DLB_LOG_ERR("%s: Error parsing dev_id
parameter",
+ name);
+ rte_kvargs_free(kvlist);
+ return ret;
+ }
+
+ ret = rte_kvargs_process(kvlist, DLB_DEFER_SCHED_ARG,
+ set_defer_sched,
+ &dlb_args->defer_sched);
+ if (ret != 0) {
+ DLB_LOG_ERR("%s: Error parsing defer_sched
parameter",
+ name);
+ rte_kvargs_free(kvlist);
+ return ret;
+ }
+
+ ret = rte_kvargs_process(kvlist,
+ DLB_NUM_ATM_INFLIGHTS_ARG,
+ set_num_atm_inflights,
+ &dlb_args->num_atm_inflights);
+ if (ret != 0) {
+ DLB_LOG_ERR("%s: Error parsing atm_inflights
parameter",
+ name);
+ rte_kvargs_free(kvlist);
+ return ret;
+ }
+
+ rte_kvargs_free(kvlist);
+ }
+ }
+ return ret;
+}
+
+/* declared extern in header, for access from other .c files */
+int eventdev_dlb_log_level;
+
+RTE_INIT(evdev_dlb_init_log)
+{
+ eventdev_dlb_log_level = rte_log_register("pmd.event.dlb");
+ if (eventdev_dlb_log_level >= 0)
+ rte_log_set_level(eventdev_dlb_log_level, RTE_LOG_NOTICE);
+}
diff --git a/drivers/event/dlb/meson.build b/drivers/event/dlb/meson.build
index 633e236..e45ea28 100644
--- a/drivers/event/dlb/meson.build
+++ b/drivers/event/dlb/meson.build
@@ -1,7 +1,8 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright(c) 2019-2020 Intel Corporation
-sources = files('dlb_selftest.c',
+sources = files('dlb.c',
+ 'dlb_selftest.c',
'dlb_xstats.c',
'rte_pmd_dlb.c',
'dlb_iface.c',
--
1.7.10
