Create a symmetric crypto device and supported cryptodev ops. Signed-off-by: Vikas Gupta <vikas.gu...@broadcom.com> Signed-off-by: Raveendra Padasalagi <raveendra.padasal...@broadcom.com> Reviewed-by: Ajit Khaparde <ajit.khapa...@broadcom.com> --- drivers/crypto/bcmfs/bcmfs_device.c | 15 ++ drivers/crypto/bcmfs/bcmfs_device.h | 9 + drivers/crypto/bcmfs/bcmfs_qp.c | 37 +++ drivers/crypto/bcmfs/bcmfs_qp.h | 16 ++ drivers/crypto/bcmfs/bcmfs_sym_pmd.c | 387 +++++++++++++++++++++++++++ drivers/crypto/bcmfs/bcmfs_sym_pmd.h | 38 +++ drivers/crypto/bcmfs/bcmfs_sym_req.h | 22 ++ drivers/crypto/bcmfs/meson.build | 3 +- 8 files changed, 526 insertions(+), 1 deletion(-) create mode 100644 drivers/crypto/bcmfs/bcmfs_sym_pmd.c create mode 100644 drivers/crypto/bcmfs/bcmfs_sym_pmd.h create mode 100644 drivers/crypto/bcmfs/bcmfs_sym_req.h
diff --git a/drivers/crypto/bcmfs/bcmfs_device.c b/drivers/crypto/bcmfs/bcmfs_device.c index bd2d64acf..c9263ec28 100644 --- a/drivers/crypto/bcmfs/bcmfs_device.c +++ b/drivers/crypto/bcmfs/bcmfs_device.c @@ -13,6 +13,7 @@ #include "bcmfs_logs.h" #include "bcmfs_qp.h" #include "bcmfs_vfio.h" +#include "bcmfs_sym_pmd.h" struct bcmfs_device_attr { const char name[BCMFS_MAX_PATH_LEN]; @@ -239,6 +240,7 @@ bcmfs_vdev_probe(struct rte_vdev_device *vdev) char out_dirname[BCMFS_MAX_PATH_LEN]; uint32_t fsdev_dev[BCMFS_MAX_NODES]; enum bcmfs_device_type dtype; + int err; int i = 0; int dev_idx; int count = 0; @@ -290,7 +292,20 @@ bcmfs_vdev_probe(struct rte_vdev_device *vdev) return -ENODEV; } + err = bcmfs_sym_dev_create(fsdev); + if (err) { + BCMFS_LOG(WARNING, + "Failed to create BCMFS SYM PMD for device %s", + fsdev->name); + goto pmd_create_fail; + } + return 0; + +pmd_create_fail: + fsdev_release(fsdev); + + return err; } static int diff --git a/drivers/crypto/bcmfs/bcmfs_device.h b/drivers/crypto/bcmfs/bcmfs_device.h index 96beb10fa..37907b91f 100644 --- a/drivers/crypto/bcmfs/bcmfs_device.h +++ b/drivers/crypto/bcmfs/bcmfs_device.h @@ -62,6 +62,15 @@ struct bcmfs_device { struct bcmfs_qp *qps_in_use[BCMFS_MAX_HW_QUEUES]; /* queue pair ops exported by symmetric crypto hw */ struct bcmfs_hw_queue_pair_ops *sym_hw_qp_ops; + /* a cryptodevice attached to bcmfs device */ + struct rte_cryptodev *cdev; + /* a rte_device to register with cryptodev */ + struct rte_device sym_rte_dev; + /* private info to keep with cryptodev */ + struct bcmfs_sym_dev_private *sym_dev; }; +/* stats exported by device */ + + #endif /* _BCMFS_DEV_H_ */ diff --git a/drivers/crypto/bcmfs/bcmfs_qp.c b/drivers/crypto/bcmfs/bcmfs_qp.c index ec1327b78..cb5ff6c61 100644 --- a/drivers/crypto/bcmfs/bcmfs_qp.c +++ b/drivers/crypto/bcmfs/bcmfs_qp.c @@ -344,3 +344,40 @@ bcmfs_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops) return deq; } + +void bcmfs_qp_stats_get(struct bcmfs_qp **qp, int num_qp, + struct bcmfs_qp_stats *stats) +{ + int i; + + if (stats == NULL) { + BCMFS_LOG(ERR, "invalid param: stats %p", + stats); + return; + } + + for (i = 0; i < num_qp; i++) { + if (qp[i] == NULL) { + BCMFS_LOG(DEBUG, "Uninitialised qp %d", i); + continue; + } + + stats->enqueued_count += qp[i]->stats.enqueued_count; + stats->dequeued_count += qp[i]->stats.dequeued_count; + stats->enqueue_err_count += qp[i]->stats.enqueue_err_count; + stats->dequeue_err_count += qp[i]->stats.dequeue_err_count; + } +} + +void bcmfs_qp_stats_reset(struct bcmfs_qp **qp, int num_qp) +{ + int i; + + for (i = 0; i < num_qp; i++) { + if (qp[i] == NULL) { + BCMFS_LOG(DEBUG, "Uninitialised qp %d", i); + continue; + } + memset(&qp[i]->stats, 0, sizeof(qp[i]->stats)); + } +} diff --git a/drivers/crypto/bcmfs/bcmfs_qp.h b/drivers/crypto/bcmfs/bcmfs_qp.h index e4b0c3f2f..fec58ca71 100644 --- a/drivers/crypto/bcmfs/bcmfs_qp.h +++ b/drivers/crypto/bcmfs/bcmfs_qp.h @@ -24,6 +24,13 @@ enum bcmfs_queue_type { BCMFS_RM_CPLQ }; +#define BCMFS_QP_IOBASE_XLATE(base, idx) \ + ((base) + ((idx) * BCMFS_HW_QUEUE_IO_ADDR_LEN)) + +/* Max pkts for preprocessing before submitting to h/w qp */ +#define BCMFS_MAX_REQS_BUFF 64 + +/* qp stats */ struct bcmfs_qp_stats { /* Count of all operations enqueued */ uint64_t enqueued_count; @@ -92,6 +99,10 @@ struct bcmfs_qp { struct bcmfs_qp_stats stats; /* h/w ops associated with qp */ struct bcmfs_hw_queue_pair_ops *ops; + /* bcmfs requests pool*/ + struct rte_mempool *sr_mp; + /* a temporary buffer to keep message pointers */ + struct bcmfs_qp_message *infl_msgs[BCMFS_MAX_REQS_BUFF]; } __rte_cache_aligned; @@ -123,4 +134,9 @@ bcmfs_qp_setup(struct bcmfs_qp **qp_addr, uint16_t queue_pair_id, struct bcmfs_qp_config *bcmfs_conf); +/* stats functions*/ +void bcmfs_qp_stats_get(struct bcmfs_qp **qp, int num_qp, + struct bcmfs_qp_stats *stats); +void bcmfs_qp_stats_reset(struct bcmfs_qp **qp, int num_qp); + #endif /* _BCMFS_QP_H_ */ diff --git a/drivers/crypto/bcmfs/bcmfs_sym_pmd.c b/drivers/crypto/bcmfs/bcmfs_sym_pmd.c new file mode 100644 index 000000000..0f96915f7 --- /dev/null +++ b/drivers/crypto/bcmfs/bcmfs_sym_pmd.c @@ -0,0 +1,387 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2020 Broadcom + * All rights reserved. + */ + +#include <rte_common.h> +#include <rte_dev.h> +#include <rte_errno.h> +#include <rte_malloc.h> +#include <rte_cryptodev_pmd.h> + +#include "bcmfs_device.h" +#include "bcmfs_logs.h" +#include "bcmfs_qp.h" +#include "bcmfs_sym_pmd.h" +#include "bcmfs_sym_req.h" + +uint8_t cryptodev_bcmfs_driver_id; + +static int bcmfs_sym_qp_release(struct rte_cryptodev *dev, + uint16_t queue_pair_id); + +static int +bcmfs_sym_dev_config(__rte_unused struct rte_cryptodev *dev, + __rte_unused struct rte_cryptodev_config *config) +{ + return 0; +} + +static int +bcmfs_sym_dev_start(__rte_unused struct rte_cryptodev *dev) +{ + return 0; +} + +static void +bcmfs_sym_dev_stop(__rte_unused struct rte_cryptodev *dev) +{ +} + +static int +bcmfs_sym_dev_close(struct rte_cryptodev *dev) +{ + int i, ret; + + for (i = 0; i < dev->data->nb_queue_pairs; i++) { + ret = bcmfs_sym_qp_release(dev, i); + if (ret < 0) + return ret; + } + + return 0; +} + +static void +bcmfs_sym_dev_info_get(struct rte_cryptodev *dev, + struct rte_cryptodev_info *dev_info) +{ + struct bcmfs_sym_dev_private *internals = dev->data->dev_private; + struct bcmfs_device *fsdev = internals->fsdev; + + if (dev_info != NULL) { + dev_info->driver_id = cryptodev_bcmfs_driver_id; + dev_info->feature_flags = dev->feature_flags; + dev_info->max_nb_queue_pairs = fsdev->max_hw_qps; + /* No limit of number of sessions */ + dev_info->sym.max_nb_sessions = 0; + } +} + +static void +bcmfs_sym_stats_get(struct rte_cryptodev *dev, + struct rte_cryptodev_stats *stats) +{ + struct bcmfs_qp_stats bcmfs_stats = {0}; + struct bcmfs_sym_dev_private *bcmfs_priv; + struct bcmfs_device *fsdev; + + if (stats == NULL || dev == NULL) { + BCMFS_LOG(ERR, "invalid ptr: stats %p, dev %p", stats, dev); + return; + } + bcmfs_priv = dev->data->dev_private; + fsdev = bcmfs_priv->fsdev; + + bcmfs_qp_stats_get(fsdev->qps_in_use, fsdev->max_hw_qps, &bcmfs_stats); + + stats->enqueued_count = bcmfs_stats.enqueued_count; + stats->dequeued_count = bcmfs_stats.dequeued_count; + stats->enqueue_err_count = bcmfs_stats.enqueue_err_count; + stats->dequeue_err_count = bcmfs_stats.dequeue_err_count; +} + +static void +bcmfs_sym_stats_reset(struct rte_cryptodev *dev) +{ + struct bcmfs_sym_dev_private *bcmfs_priv; + struct bcmfs_device *fsdev; + + if (dev == NULL) { + BCMFS_LOG(ERR, "invalid cryptodev ptr %p", dev); + return; + } + bcmfs_priv = dev->data->dev_private; + fsdev = bcmfs_priv->fsdev; + + bcmfs_qp_stats_reset(fsdev->qps_in_use, fsdev->max_hw_qps); +} + +static int +bcmfs_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) +{ + struct bcmfs_sym_dev_private *bcmfs_private = dev->data->dev_private; + struct bcmfs_qp *qp = (struct bcmfs_qp *) + (dev->data->queue_pairs[queue_pair_id]); + + BCMFS_LOG(DEBUG, "Release sym qp %u on device %d", + queue_pair_id, dev->data->dev_id); + + rte_mempool_free(qp->sr_mp); + + bcmfs_private->fsdev->qps_in_use[queue_pair_id] = NULL; + + return bcmfs_qp_release((struct bcmfs_qp **) + &dev->data->queue_pairs[queue_pair_id]); +} + +static void +spu_req_init(struct bcmfs_sym_request *sr, rte_iova_t iova __rte_unused) +{ + memset(sr, 0, sizeof(*sr)); +} + +static void +req_pool_obj_init(__rte_unused struct rte_mempool *mp, + __rte_unused void *opaque, void *obj, + __rte_unused unsigned int obj_idx) +{ + spu_req_init(obj, rte_mempool_virt2iova(obj)); +} + +static struct rte_mempool * +bcmfs_sym_req_pool_create(struct rte_cryptodev *cdev __rte_unused, + uint32_t nobjs, uint16_t qp_id, + int socket_id) +{ + char softreq_pool_name[RTE_RING_NAMESIZE]; + struct rte_mempool *mp; + + snprintf(softreq_pool_name, RTE_RING_NAMESIZE, "%s_%d", + "bcm_sym", qp_id); + + mp = rte_mempool_create(softreq_pool_name, + RTE_ALIGN_MUL_CEIL(nobjs, 64), + sizeof(struct bcmfs_sym_request), + 64, 0, NULL, NULL, req_pool_obj_init, NULL, + socket_id, 0); + if (mp == NULL) + BCMFS_LOG(ERR, "Failed to create req pool, qid %d, err %d", + qp_id, rte_errno); + + return mp; +} + +static int +bcmfs_sym_qp_setup(struct rte_cryptodev *cdev, uint16_t qp_id, + const struct rte_cryptodev_qp_conf *qp_conf, + int socket_id) +{ + int ret = 0; + struct bcmfs_qp *qp = NULL; + struct bcmfs_qp_config bcmfs_qp_conf; + + struct bcmfs_qp **qp_addr = + (struct bcmfs_qp **)&cdev->data->queue_pairs[qp_id]; + struct bcmfs_sym_dev_private *bcmfs_private = cdev->data->dev_private; + struct bcmfs_device *fsdev = bcmfs_private->fsdev; + + + /* If qp is already in use free ring memory and qp metadata. */ + if (*qp_addr != NULL) { + ret = bcmfs_sym_qp_release(cdev, qp_id); + if (ret < 0) + return ret; + } + + if (qp_id >= fsdev->max_hw_qps) { + BCMFS_LOG(ERR, "qp_id %u invalid for this device", qp_id); + return -EINVAL; + } + + bcmfs_qp_conf.nb_descriptors = qp_conf->nb_descriptors; + bcmfs_qp_conf.socket_id = socket_id; + bcmfs_qp_conf.max_descs_req = BCMFS_CRYPTO_MAX_HW_DESCS_PER_REQ; + bcmfs_qp_conf.iobase = BCMFS_QP_IOBASE_XLATE(fsdev->mmap_addr, qp_id); + bcmfs_qp_conf.ops = fsdev->sym_hw_qp_ops; + + ret = bcmfs_qp_setup(qp_addr, qp_id, &bcmfs_qp_conf); + if (ret != 0) + return ret; + + qp = (struct bcmfs_qp *)*qp_addr; + + qp->sr_mp = bcmfs_sym_req_pool_create(cdev, qp_conf->nb_descriptors, + qp_id, socket_id); + if (qp->sr_mp == NULL) + return -ENOMEM; + + /* store a link to the qp in the bcmfs_device */ + bcmfs_private->fsdev->qps_in_use[qp_id] = *qp_addr; + + cdev->data->queue_pairs[qp_id] = qp; + BCMFS_LOG(NOTICE, "queue %d setup done\n", qp_id); + + return 0; +} + +static struct rte_cryptodev_ops crypto_bcmfs_ops = { + /* Device related operations */ + .dev_configure = bcmfs_sym_dev_config, + .dev_start = bcmfs_sym_dev_start, + .dev_stop = bcmfs_sym_dev_stop, + .dev_close = bcmfs_sym_dev_close, + .dev_infos_get = bcmfs_sym_dev_info_get, + /* Stats Collection */ + .stats_get = bcmfs_sym_stats_get, + .stats_reset = bcmfs_sym_stats_reset, + /* Queue-Pair management */ + .queue_pair_setup = bcmfs_sym_qp_setup, + .queue_pair_release = bcmfs_sym_qp_release, +}; + +/** Enqueue burst */ +static uint16_t +bcmfs_sym_pmd_enqueue_op_burst(void *queue_pair, + struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + int i, j; + uint16_t enq = 0; + struct bcmfs_sym_request *sreq; + struct bcmfs_qp *qp = (struct bcmfs_qp *)queue_pair; + + if (nb_ops == 0) + return 0; + + if (nb_ops > BCMFS_MAX_REQS_BUFF) + nb_ops = BCMFS_MAX_REQS_BUFF; + + /* We do not process more than available space */ + if (nb_ops > (qp->nb_descriptors - qp->nb_pending_requests)) + nb_ops = qp->nb_descriptors - qp->nb_pending_requests; + + for (i = 0; i < nb_ops; i++) { + if (rte_mempool_get(qp->sr_mp, (void **)&sreq)) + goto enqueue_err; + + /* save rte_crypto_op */ + sreq->op = ops[i]; + + /* save context */ + qp->infl_msgs[i] = &sreq->msgs; + qp->infl_msgs[i]->ctx = (void *)sreq; + } + /* Send burst request to hw QP */ + enq = bcmfs_enqueue_op_burst(qp, (void **)qp->infl_msgs, i); + + for (j = enq; j < i; j++) + rte_mempool_put(qp->sr_mp, qp->infl_msgs[j]->ctx); + + return enq; + +enqueue_err: + for (j = 0; j < i; j++) + rte_mempool_put(qp->sr_mp, qp->infl_msgs[j]->ctx); + + return enq; +} + +static uint16_t +bcmfs_sym_pmd_dequeue_op_burst(void *queue_pair, + struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + int i; + uint16_t deq = 0; + unsigned int pkts = 0; + struct bcmfs_sym_request *sreq; + struct bcmfs_qp *qp = queue_pair; + + if (nb_ops > BCMFS_MAX_REQS_BUFF) + nb_ops = BCMFS_MAX_REQS_BUFF; + + deq = bcmfs_dequeue_op_burst(qp, (void **)qp->infl_msgs, nb_ops); + /* get rte_crypto_ops */ + for (i = 0; i < deq; i++) { + sreq = (struct bcmfs_sym_request *)qp->infl_msgs[i]->ctx; + + ops[pkts++] = sreq->op; + + rte_mempool_put(qp->sr_mp, sreq); + } + + return pkts; +} + +/* + * An rte_driver is needed in the registration of both the + * device and the driver with cryptodev. + */ +static const char bcmfs_sym_drv_name[] = RTE_STR(CRYPTODEV_NAME_BCMFS_SYM_PMD); +static const struct rte_driver cryptodev_bcmfs_sym_driver = { + .name = bcmfs_sym_drv_name, + .alias = bcmfs_sym_drv_name +}; + +int +bcmfs_sym_dev_create(struct bcmfs_device *fsdev) +{ + struct rte_cryptodev_pmd_init_params init_params = { + .name = "", + .socket_id = rte_socket_id(), + .private_data_size = sizeof(struct bcmfs_sym_dev_private) + }; + char name[RTE_CRYPTODEV_NAME_MAX_LEN]; + struct rte_cryptodev *cryptodev; + struct bcmfs_sym_dev_private *internals; + + snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s", + fsdev->name, "sym"); + + /* Populate subset device to use in cryptodev device creation */ + fsdev->sym_rte_dev.driver = &cryptodev_bcmfs_sym_driver; + fsdev->sym_rte_dev.numa_node = 0; + fsdev->sym_rte_dev.devargs = NULL; + + cryptodev = rte_cryptodev_pmd_create(name, + &fsdev->sym_rte_dev, + &init_params); + if (cryptodev == NULL) + return -ENODEV; + + fsdev->sym_rte_dev.name = cryptodev->data->name; + cryptodev->driver_id = cryptodev_bcmfs_driver_id; + cryptodev->dev_ops = &crypto_bcmfs_ops; + + cryptodev->enqueue_burst = bcmfs_sym_pmd_enqueue_op_burst; + cryptodev->dequeue_burst = bcmfs_sym_pmd_dequeue_op_burst; + + cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | + RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | + RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT; + + internals = cryptodev->data->dev_private; + internals->fsdev = fsdev; + fsdev->sym_dev = internals; + + internals->sym_dev_id = cryptodev->data->dev_id; + + BCMFS_LOG(DEBUG, "Created bcmfs-sym device %s as cryptodev instance %d", + cryptodev->data->name, internals->sym_dev_id); + return 0; +} + +int +bcmfs_sym_dev_destroy(struct bcmfs_device *fsdev) +{ + struct rte_cryptodev *cryptodev; + + if (fsdev == NULL) + return -ENODEV; + if (fsdev->sym_dev == NULL) + return 0; + + /* free crypto device */ + cryptodev = rte_cryptodev_pmd_get_dev(fsdev->sym_dev->sym_dev_id); + rte_cryptodev_pmd_destroy(cryptodev); + fsdev->sym_rte_dev.name = NULL; + fsdev->sym_dev = NULL; + + return 0; +} + +static struct cryptodev_driver bcmfs_crypto_drv; +RTE_PMD_REGISTER_CRYPTO_DRIVER(bcmfs_crypto_drv, + cryptodev_bcmfs_sym_driver, + cryptodev_bcmfs_driver_id); diff --git a/drivers/crypto/bcmfs/bcmfs_sym_pmd.h b/drivers/crypto/bcmfs/bcmfs_sym_pmd.h new file mode 100644 index 000000000..65d704609 --- /dev/null +++ b/drivers/crypto/bcmfs/bcmfs_sym_pmd.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2020 Broadcom + * All rights reserved. + */ + +#ifndef _BCMFS_SYM_PMD_H_ +#define _BCMFS_SYM_PMD_H_ + +#include <rte_cryptodev.h> + +#include "bcmfs_device.h" + +#define CRYPTODEV_NAME_BCMFS_SYM_PMD crypto_bcmfs + +#define BCMFS_CRYPTO_MAX_HW_DESCS_PER_REQ 16 + +extern uint8_t cryptodev_bcmfs_driver_id; + +/** private data structure for a BCMFS device. + * This BCMFS device is a device offering only symmetric crypto service, + * there can be one of these on each bcmfs_pci_device (VF). + */ +struct bcmfs_sym_dev_private { + /* The bcmfs device hosting the service */ + struct bcmfs_device *fsdev; + /* Device instance for this rte_cryptodev */ + uint8_t sym_dev_id; + /* BCMFS device symmetric crypto capabilities */ + const struct rte_cryptodev_capabilities *fsdev_capabilities; +}; + +int +bcmfs_sym_dev_create(struct bcmfs_device *fdev); + +int +bcmfs_sym_dev_destroy(struct bcmfs_device *fdev); + +#endif /* _BCMFS_SYM_PMD_H_ */ diff --git a/drivers/crypto/bcmfs/bcmfs_sym_req.h b/drivers/crypto/bcmfs/bcmfs_sym_req.h new file mode 100644 index 000000000..0f0b051f1 --- /dev/null +++ b/drivers/crypto/bcmfs/bcmfs_sym_req.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2020 Broadcom + * All rights reserved. + */ + +#ifndef _BCMFS_SYM_REQ_H_ +#define _BCMFS_SYM_REQ_H_ + +#include "bcmfs_dev_msg.h" + +/* + * This structure hold the supportive data required to process a + * rte_crypto_op + */ +struct bcmfs_sym_request { + /* bcmfs qp message for h/w queues to process */ + struct bcmfs_qp_message msgs; + /* crypto op */ + struct rte_crypto_op *op; +}; + +#endif /* _BCMFS_SYM_REQ_H_ */ diff --git a/drivers/crypto/bcmfs/meson.build b/drivers/crypto/bcmfs/meson.build index cd58bd5e2..d9a3d73e9 100644 --- a/drivers/crypto/bcmfs/meson.build +++ b/drivers/crypto/bcmfs/meson.build @@ -11,5 +11,6 @@ sources = files( 'bcmfs_qp.c', 'hw/bcmfs4_rm.c', 'hw/bcmfs5_rm.c', - 'hw/bcmfs_rm_common.c' + 'hw/bcmfs_rm_common.c', + 'bcmfs_sym_pmd.c' ) -- 2.17.1