This commit isolates implementations of common part in QAT PMD. When changing/expanding particular generation code, other generations code should left intact. Generation code in drivers common is invisible to other generations code.
Signed-off-by: Arek Kusztal <arkadiuszx.kusz...@intel.com> --- drivers/common/qat/dev/qat_dev_gen1.c | 245 ++++++++++ drivers/common/qat/dev/qat_dev_gen1.h | 52 +++ drivers/common/qat/dev/qat_dev_gen2.c | 38 ++ drivers/common/qat/dev/qat_dev_gen3.c | 76 +++ drivers/common/qat/dev/qat_dev_gen4.c | 258 +++++++++++ drivers/common/qat/meson.build | 4 + drivers/common/qat/qat_common.h | 2 + drivers/common/qat/qat_device.c | 117 ++--- drivers/common/qat/qat_device.h | 24 +- drivers/common/qat/qat_qp.c | 641 +++++++++----------------- drivers/common/qat/qat_qp.h | 45 +- 11 files changed, 983 insertions(+), 519 deletions(-) create mode 100644 drivers/common/qat/dev/qat_dev_gen1.c create mode 100644 drivers/common/qat/dev/qat_dev_gen1.h create mode 100644 drivers/common/qat/dev/qat_dev_gen2.c create mode 100644 drivers/common/qat/dev/qat_dev_gen3.c create mode 100644 drivers/common/qat/dev/qat_dev_gen4.c diff --git a/drivers/common/qat/dev/qat_dev_gen1.c b/drivers/common/qat/dev/qat_dev_gen1.c new file mode 100644 index 0000000000..4d60c2a051 --- /dev/null +++ b/drivers/common/qat/dev/qat_dev_gen1.c @@ -0,0 +1,245 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include "qat_device.h" +#include "qat_qp.h" +#include "adf_transport_access_macros.h" +#include "qat_dev_gen1.h" + +#include <stdint.h> + +#define ADF_ARB_REG_SLOT 0x1000 + +#define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \ + ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \ + (ADF_ARB_REG_SLOT * index), value) + +__extension__ +const struct qat_qp_hw_data qat_gen1_qps[QAT_MAX_SERVICES] + [ADF_MAX_QPS_ON_ANY_SERVICE] = { + /* queue pairs which provide an asymmetric crypto service */ + [QAT_SERVICE_ASYMMETRIC] = { + { + .service_type = QAT_SERVICE_ASYMMETRIC, + .hw_bundle_num = 0, + .tx_ring_num = 0, + .rx_ring_num = 8, + .tx_msg_size = 64, + .rx_msg_size = 32, + + }, { + .service_type = QAT_SERVICE_ASYMMETRIC, + .hw_bundle_num = 0, + .tx_ring_num = 1, + .rx_ring_num = 9, + .tx_msg_size = 64, + .rx_msg_size = 32, + } + }, + /* queue pairs which provide a symmetric crypto service */ + [QAT_SERVICE_SYMMETRIC] = { + { + .service_type = QAT_SERVICE_SYMMETRIC, + .hw_bundle_num = 0, + .tx_ring_num = 2, + .rx_ring_num = 10, + .tx_msg_size = 128, + .rx_msg_size = 32, + }, + { + .service_type = QAT_SERVICE_SYMMETRIC, + .hw_bundle_num = 0, + .tx_ring_num = 3, + .rx_ring_num = 11, + .tx_msg_size = 128, + .rx_msg_size = 32, + } + }, + /* queue pairs which provide a compression service */ + [QAT_SERVICE_COMPRESSION] = { + { + .service_type = QAT_SERVICE_COMPRESSION, + .hw_bundle_num = 0, + .tx_ring_num = 6, + .rx_ring_num = 14, + .tx_msg_size = 128, + .rx_msg_size = 32, + }, { + .service_type = QAT_SERVICE_COMPRESSION, + .hw_bundle_num = 0, + .tx_ring_num = 7, + .rx_ring_num = 15, + .tx_msg_size = 128, + .rx_msg_size = 32, + } + } +}; + +int +qat_qp_rings_per_service_gen1(struct qat_pci_device *qat_dev, + enum qat_service_type service) +{ + int i = 0, count = 0, max_ops_per_srv = 0; + const struct qat_qp_hw_data *sym_hw_qps = + qat_gen_config[qat_dev->qat_dev_gen] + .qp_hw_data[service]; + + max_ops_per_srv = ADF_MAX_QPS_ON_ANY_SERVICE; + for (i = 0, count = 0; i < max_ops_per_srv; i++) + if (sym_hw_qps[i].service_type == service) + count++; + return count; +} + +void +qat_qp_csr_build_ring_base_gen1(void *io_addr, + struct qat_queue *queue) +{ + uint64_t queue_base; + + queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr, + queue->queue_size); + WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number, + queue->hw_queue_number, queue_base); +} + +void +qat_qp_adf_arb_enable_gen1(const struct qat_queue *txq, + void *base_addr, rte_spinlock_t *lock) +{ + uint32_t arb_csr_offset = 0, value; + + rte_spinlock_lock(lock); + arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET + + (ADF_ARB_REG_SLOT * + txq->hw_bundle_number); + value = ADF_CSR_RD(base_addr, + arb_csr_offset); + value |= (0x01 << txq->hw_queue_number); + ADF_CSR_WR(base_addr, arb_csr_offset, value); + rte_spinlock_unlock(lock); +} + +void +qat_qp_adf_arb_disable_gen1(const struct qat_queue *txq, + void *base_addr, rte_spinlock_t *lock) +{ + uint32_t arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET + + (ADF_ARB_REG_SLOT * + txq->hw_bundle_number); + uint32_t value; + + rte_spinlock_lock(lock); + value = ADF_CSR_RD(base_addr, arb_csr_offset); + value &= ~(0x01 << txq->hw_queue_number); + ADF_CSR_WR(base_addr, arb_csr_offset, value); + rte_spinlock_unlock(lock); +} + +void +qat_qp_adf_configure_queues_gen1(struct qat_qp *qp) +{ + uint32_t q_tx_config, q_resp_config; + struct qat_queue *q_tx = &qp->tx_q, *q_rx = &qp->rx_q; + + q_tx_config = BUILD_RING_CONFIG(q_tx->queue_size); + q_resp_config = BUILD_RESP_RING_CONFIG(q_rx->queue_size, + ADF_RING_NEAR_WATERMARK_512, + ADF_RING_NEAR_WATERMARK_0); + WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, + q_tx->hw_bundle_number, q_tx->hw_queue_number, + q_tx_config); + WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, + q_rx->hw_bundle_number, q_rx->hw_queue_number, + q_resp_config); +} + +void +qat_qp_csr_write_tail_gen1(struct qat_qp *qp, struct qat_queue *q) +{ + WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number, + q->hw_queue_number, q->tail); +} + +void +qat_qp_csr_write_head_gen1(struct qat_qp *qp, struct qat_queue *q, + uint32_t new_head) +{ + WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number, + q->hw_queue_number, new_head); +} + +void +qat_qp_csr_setup_gen1(struct qat_pci_device *qat_dev, + void *io_addr, struct qat_qp *qp) +{ + qat_qp_csr_build_ring_base_gen1(io_addr, &qp->tx_q); + qat_qp_csr_build_ring_base_gen1(io_addr, &qp->rx_q); + qat_qp_adf_configure_queues_gen1(qp); + qat_qp_adf_arb_enable_gen1(&qp->tx_q, qp->mmap_bar_addr, + &qat_dev->arb_csr_lock); +} + +static struct qat_qp_hw_spec_funcs qat_qp_hw_spec_gen1 = { + .qat_qp_rings_per_service = qat_qp_rings_per_service_gen1, + .qat_qp_build_ring_base = qat_qp_csr_build_ring_base_gen1, + .qat_qp_adf_arb_enable = qat_qp_adf_arb_enable_gen1, + .qat_qp_adf_arb_disable = qat_qp_adf_arb_disable_gen1, + .qat_qp_adf_configure_queues = qat_qp_adf_configure_queues_gen1, + .qat_qp_csr_write_tail = qat_qp_csr_write_tail_gen1, + .qat_qp_csr_write_head = qat_qp_csr_write_head_gen1, + .qat_qp_csr_setup = qat_qp_csr_setup_gen1, +}; + +int qat_reset_ring_pairs_gen1( + struct qat_pci_device *qat_pci_dev __rte_unused) +{ + /* + * Ring pairs reset not supported on base, continue + */ + return 0; +} + +const struct +rte_mem_resource *qat_dev_get_transport_bar_gen1( + struct rte_pci_device *pci_dev) +{ + return &pci_dev->mem_resource[0]; +} + +int +qat_dev_get_misc_bar_gen1( + struct rte_mem_resource **mem_resource __rte_unused, + struct rte_pci_device *pci_dev __rte_unused) +{ + return -1; +} + +int +qat_dev_read_config_gen1(struct qat_pci_device *qat_dev __rte_unused) +{ + /* + * Base generations do not have configuration, + * but set this pointer anyway that we can + * distinguish higher generations faulty set to NULL + */ + return 0; +} + +static struct qat_dev_hw_spec_funcs qat_dev_hw_spec_gen1 = { + .qat_dev_reset_ring_pairs = qat_reset_ring_pairs_gen1, + .qat_dev_get_transport_bar = qat_dev_get_transport_bar_gen1, + .qat_dev_get_misc_bar = qat_dev_get_misc_bar_gen1, + .qat_dev_read_config = qat_dev_read_config_gen1, +}; + +RTE_INIT(qat_dev_gen_gen1_init) +{ + qat_qp_hw_spec[QAT_GEN1] = &qat_qp_hw_spec_gen1; + qat_dev_hw_spec[QAT_GEN1] = &qat_dev_hw_spec_gen1; + qat_gen_config[QAT_GEN1].dev_gen = QAT_GEN1; + qat_gen_config[QAT_GEN1].qp_hw_data = qat_gen1_qps; + qat_gen_config[QAT_GEN1].comp_num_im_bufs_required = + QAT_NUM_INTERM_BUFS_GEN1; +} diff --git a/drivers/common/qat/dev/qat_dev_gen1.h b/drivers/common/qat/dev/qat_dev_gen1.h new file mode 100644 index 0000000000..9bf4fcf01b --- /dev/null +++ b/drivers/common/qat/dev/qat_dev_gen1.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#ifndef _QAT_DEV_GEN_H_ +#define _QAT_DEV_GEN_H_ + +#include "qat_device.h" +#include "qat_qp.h" + +#include <stdint.h> + +extern const struct qat_qp_hw_data qat_gen1_qps[QAT_MAX_SERVICES] + [ADF_MAX_QPS_ON_ANY_SERVICE]; + +int +qat_qp_rings_per_service_gen1(struct qat_pci_device *qat_dev, + enum qat_service_type service); +void +qat_qp_csr_build_ring_base_gen1(void *io_addr, + struct qat_queue *queue); +void +qat_qp_adf_arb_enable_gen1(const struct qat_queue *txq, + void *base_addr, rte_spinlock_t *lock); +void +qat_qp_adf_arb_disable_gen1(const struct qat_queue *txq, + void *base_addr, rte_spinlock_t *lock); +void +qat_qp_adf_configure_queues_gen1(struct qat_qp *qp); +void +qat_qp_csr_write_tail_gen1(struct qat_qp *qp, struct qat_queue *q); +void +qat_qp_csr_write_head_gen1(struct qat_qp *qp, struct qat_queue *q, + uint32_t new_head); +void +qat_qp_csr_setup_gen1(struct qat_pci_device *qat_dev, + void *io_addr, struct qat_qp *qp); + +int +qat_reset_ring_pairs_gen1( + struct qat_pci_device *qat_pci_dev __rte_unused); +const struct +rte_mem_resource *qat_dev_get_transport_bar_gen1( + struct rte_pci_device *pci_dev); +int +qat_dev_get_misc_bar_gen1( + struct rte_mem_resource **mem_resource __rte_unused, + struct rte_pci_device *pci_dev __rte_unused); +int +qat_dev_read_config_gen1(struct qat_pci_device *qat_dev __rte_unused); + +#endif diff --git a/drivers/common/qat/dev/qat_dev_gen2.c b/drivers/common/qat/dev/qat_dev_gen2.c new file mode 100644 index 0000000000..ad1b643e00 --- /dev/null +++ b/drivers/common/qat/dev/qat_dev_gen2.c @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include "qat_device.h" +#include "qat_qp.h" +#include "adf_transport_access_macros.h" +#include "qat_dev_gen1.h" + +#include <stdint.h> + +static struct qat_qp_hw_spec_funcs qat_qp_hw_spec_gen2 = { + .qat_qp_rings_per_service = qat_qp_rings_per_service_gen1, + .qat_qp_build_ring_base = qat_qp_csr_build_ring_base_gen1, + .qat_qp_adf_arb_enable = qat_qp_adf_arb_enable_gen1, + .qat_qp_adf_arb_disable = qat_qp_adf_arb_disable_gen1, + .qat_qp_adf_configure_queues = qat_qp_adf_configure_queues_gen1, + .qat_qp_csr_write_tail = qat_qp_csr_write_tail_gen1, + .qat_qp_csr_write_head = qat_qp_csr_write_head_gen1, + .qat_qp_csr_setup = qat_qp_csr_setup_gen1, +}; + +static struct qat_dev_hw_spec_funcs qat_dev_hw_spec_gen2 = { + .qat_dev_reset_ring_pairs = qat_reset_ring_pairs_gen1, + .qat_dev_get_transport_bar = qat_dev_get_transport_bar_gen1, + .qat_dev_get_misc_bar = qat_dev_get_misc_bar_gen1, + .qat_dev_read_config = qat_dev_read_config_gen1, +}; + +RTE_INIT(qat_dev_gen_gen2_init) +{ + qat_qp_hw_spec[QAT_GEN2] = &qat_qp_hw_spec_gen2; + qat_dev_hw_spec[QAT_GEN2] = &qat_dev_hw_spec_gen2; + qat_gen_config[QAT_GEN2].dev_gen = QAT_GEN2; + qat_gen_config[QAT_GEN2].qp_hw_data = qat_gen1_qps; + qat_gen_config[QAT_GEN2].comp_num_im_bufs_required = + QAT_NUM_INTERM_BUFS_GEN2; +} diff --git a/drivers/common/qat/dev/qat_dev_gen3.c b/drivers/common/qat/dev/qat_dev_gen3.c new file mode 100644 index 0000000000..407d21576b --- /dev/null +++ b/drivers/common/qat/dev/qat_dev_gen3.c @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include "qat_device.h" +#include "qat_qp.h" +#include "adf_transport_access_macros.h" +#include "qat_dev_gen1.h" + +#include <stdint.h> + +__extension__ +const struct qat_qp_hw_data qat_gen3_qps[QAT_MAX_SERVICES] + [ADF_MAX_QPS_ON_ANY_SERVICE] = { + /* queue pairs which provide an asymmetric crypto service */ + [QAT_SERVICE_ASYMMETRIC] = { + { + .service_type = QAT_SERVICE_ASYMMETRIC, + .hw_bundle_num = 0, + .tx_ring_num = 0, + .rx_ring_num = 4, + .tx_msg_size = 64, + .rx_msg_size = 32, + } + }, + /* queue pairs which provide a symmetric crypto service */ + [QAT_SERVICE_SYMMETRIC] = { + { + .service_type = QAT_SERVICE_SYMMETRIC, + .hw_bundle_num = 0, + .tx_ring_num = 1, + .rx_ring_num = 5, + .tx_msg_size = 128, + .rx_msg_size = 32, + } + }, + /* queue pairs which provide a compression service */ + [QAT_SERVICE_COMPRESSION] = { + { + .service_type = QAT_SERVICE_COMPRESSION, + .hw_bundle_num = 0, + .tx_ring_num = 3, + .rx_ring_num = 7, + .tx_msg_size = 128, + .rx_msg_size = 32, + } + } +}; + +static struct qat_qp_hw_spec_funcs qat_qp_hw_spec_gen3 = { + .qat_qp_rings_per_service = qat_qp_rings_per_service_gen1, + .qat_qp_build_ring_base = qat_qp_csr_build_ring_base_gen1, + .qat_qp_adf_arb_enable = qat_qp_adf_arb_enable_gen1, + .qat_qp_adf_arb_disable = qat_qp_adf_arb_disable_gen1, + .qat_qp_adf_configure_queues = qat_qp_adf_configure_queues_gen1, + .qat_qp_csr_write_tail = qat_qp_csr_write_tail_gen1, + .qat_qp_csr_write_head = qat_qp_csr_write_head_gen1, + .qat_qp_csr_setup = qat_qp_csr_setup_gen1, +}; + +static struct qat_dev_hw_spec_funcs qat_dev_hw_spec_gen3 = { + .qat_dev_reset_ring_pairs = qat_reset_ring_pairs_gen1, + .qat_dev_get_transport_bar = qat_dev_get_transport_bar_gen1, + .qat_dev_get_misc_bar = qat_dev_get_misc_bar_gen1, + .qat_dev_read_config = qat_dev_read_config_gen1, +}; + +RTE_INIT(qat_dev_gen_gen3_init) +{ + qat_qp_hw_spec[QAT_GEN3] = &qat_qp_hw_spec_gen3; + qat_dev_hw_spec[QAT_GEN3] = &qat_dev_hw_spec_gen3; + qat_gen_config[QAT_GEN3].dev_gen = QAT_GEN3; + qat_gen_config[QAT_GEN3].qp_hw_data = qat_gen3_qps; + qat_gen_config[QAT_GEN3].comp_num_im_bufs_required = + QAT_NUM_INTERM_BUFS_GEN3; +} diff --git a/drivers/common/qat/dev/qat_dev_gen4.c b/drivers/common/qat/dev/qat_dev_gen4.c new file mode 100644 index 0000000000..6394e17dde --- /dev/null +++ b/drivers/common/qat/dev/qat_dev_gen4.c @@ -0,0 +1,258 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include <rte_dev.h> +#include <rte_pci.h> + +#include "qat_device.h" +#include "qat_qp.h" +#include "adf_transport_access_macros_gen4vf.h" +#include "adf_pf2vf_msg.h" +#include "qat_pf2vf.h" + +#include <stdint.h> + +static struct qat_pf2vf_dev qat_pf2vf_gen4 = { + .pf2vf_offset = ADF_4XXXIOV_PF2VM_OFFSET, + .vf2pf_offset = ADF_4XXXIOV_VM2PF_OFFSET, + .pf2vf_type_shift = ADF_PFVF_2X_MSGTYPE_SHIFT, + .pf2vf_type_mask = ADF_PFVF_2X_MSGTYPE_MASK, + .pf2vf_data_shift = ADF_PFVF_2X_MSGDATA_SHIFT, + .pf2vf_data_mask = ADF_PFVF_2X_MSGDATA_MASK, +}; + +static int +qat_query_svc(struct qat_pci_device *qat_dev, uint8_t *val) +{ + struct qat_pf2vf_msg pf2vf_msg; + + pf2vf_msg.msg_type = ADF_VF2PF_MSGTYPE_GET_SMALL_BLOCK_REQ; + pf2vf_msg.block_hdr = ADF_VF2PF_BLOCK_MSG_GET_RING_TO_SVC_REQ; + pf2vf_msg.msg_data = 2; + return qat_pf2vf_exch_msg(qat_dev, pf2vf_msg, 2, val); +} + +static int +qat_qp_rings_per_service_gen4(struct qat_pci_device *qat_dev, + enum qat_service_type service) +{ + int i = 0, count = 0, max_ops_per_srv = 0; + + max_ops_per_srv = QAT_GEN4_BUNDLE_NUM; + for (i = 0, count = 0; i < max_ops_per_srv; i++) + if (qat_dev->qp_gen4_data[i][0].service_type == service) + count++; + return count; +} + +static int +qat_dev_read_config_gen4(struct qat_pci_device *qat_dev) +{ + int i = 0; + uint16_t svc = 0; + + if (qat_query_svc(qat_dev, (uint8_t *)&svc)) + return -EFAULT; + for (; i < QAT_GEN4_BUNDLE_NUM; i++) { + struct qat_qp_hw_data *hw_data = + &qat_dev->qp_gen4_data[i][0]; + uint8_t svc1 = (svc >> (3 * i)) & 0x7; + enum qat_service_type service_type = QAT_SERVICE_INVALID; + + if (svc1 == QAT_SVC_SYM) { + service_type = QAT_SERVICE_SYMMETRIC; + QAT_LOG(DEBUG, + "Discovered SYMMETRIC service on bundle %d", + i); + } else if (svc1 == QAT_SVC_COMPRESSION) { + service_type = QAT_SERVICE_COMPRESSION; + QAT_LOG(DEBUG, + "Discovered COPRESSION service on bundle %d", + i); + } else if (svc1 == QAT_SVC_ASYM) { + service_type = QAT_SERVICE_ASYMMETRIC; + QAT_LOG(DEBUG, + "Discovered ASYMMETRIC service on bundle %d", + i); + } else { + QAT_LOG(ERR, + "Unrecognized service on bundle %d", + i); + return -EFAULT; + } + + memset(hw_data, 0, sizeof(*hw_data)); + hw_data->service_type = service_type; + if (service_type == QAT_SERVICE_ASYMMETRIC) { + hw_data->tx_msg_size = 64; + hw_data->rx_msg_size = 32; + } else if (service_type == QAT_SERVICE_SYMMETRIC || + service_type == + QAT_SERVICE_COMPRESSION) { + hw_data->tx_msg_size = 128; + hw_data->rx_msg_size = 32; + } + hw_data->tx_ring_num = 0; + hw_data->rx_ring_num = 1; + hw_data->hw_bundle_num = i; + } + return 0; +} + +static void +qat_qp_build_ring_base_gen4(void *io_addr, + struct qat_queue *queue) +{ + uint64_t queue_base; + + queue_base = BUILD_RING_BASE_ADDR_GEN4(queue->base_phys_addr, + queue->queue_size); + WRITE_CSR_RING_BASE_GEN4VF(io_addr, queue->hw_bundle_number, + queue->hw_queue_number, queue_base); +} + +static void +qat_qp_adf_arb_enable_gen4(const struct qat_queue *txq, + void *base_addr, rte_spinlock_t *lock) +{ + uint32_t arb_csr_offset = 0, value; + + rte_spinlock_lock(lock); + arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET + + (ADF_RING_BUNDLE_SIZE_GEN4 * + txq->hw_bundle_number); + value = ADF_CSR_RD(base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, + arb_csr_offset); + value |= (0x01 << txq->hw_queue_number); + ADF_CSR_WR(base_addr, arb_csr_offset, value); + rte_spinlock_unlock(lock); +} + +static void +qat_qp_adf_arb_disable_gen4(const struct qat_queue *txq, + void *base_addr, rte_spinlock_t *lock) +{ + uint32_t arb_csr_offset = 0, value; + + rte_spinlock_lock(lock); + arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET + + (ADF_RING_BUNDLE_SIZE_GEN4 * + txq->hw_bundle_number); + value = ADF_CSR_RD(base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, + arb_csr_offset); + value &= ~(0x01 << txq->hw_queue_number); + ADF_CSR_WR(base_addr, arb_csr_offset, value); + rte_spinlock_unlock(lock); +} + +static void +qat_qp_adf_configure_queues_gen4(struct qat_qp *qp) +{ + uint32_t q_tx_config, q_resp_config; + struct qat_queue *q_tx = &qp->tx_q, *q_rx = &qp->rx_q; + + q_tx_config = BUILD_RING_CONFIG(q_tx->queue_size); + q_resp_config = BUILD_RESP_RING_CONFIG(q_rx->queue_size, + ADF_RING_NEAR_WATERMARK_512, + ADF_RING_NEAR_WATERMARK_0); + + WRITE_CSR_RING_CONFIG_GEN4VF(qp->mmap_bar_addr, + q_tx->hw_bundle_number, q_tx->hw_queue_number, + q_tx_config); + WRITE_CSR_RING_CONFIG_GEN4VF(qp->mmap_bar_addr, + q_rx->hw_bundle_number, q_rx->hw_queue_number, + q_resp_config); +} + +static void +qat_qp_csr_write_tail_gen4(struct qat_qp *qp, struct qat_queue *q) +{ + WRITE_CSR_RING_TAIL_GEN4VF(qp->mmap_bar_addr, + q->hw_bundle_number, q->hw_queue_number, q->tail); +} + +static void +qat_qp_csr_write_head_gen4(struct qat_qp *qp, struct qat_queue *q, + uint32_t new_head) +{ + WRITE_CSR_RING_HEAD_GEN4VF(qp->mmap_bar_addr, + q->hw_bundle_number, q->hw_queue_number, new_head); +} + +static void +qat_qp_csr_setup_gen4(struct qat_pci_device *qat_dev, + void *io_addr, struct qat_qp *qp) +{ + qat_qp_build_ring_base_gen4(io_addr, &qp->tx_q); + qat_qp_build_ring_base_gen4(io_addr, &qp->rx_q); + qat_qp_adf_configure_queues_gen4(qp); + qat_qp_adf_arb_enable_gen4(&qp->tx_q, qp->mmap_bar_addr, + &qat_dev->arb_csr_lock); +} + +static struct qat_qp_hw_spec_funcs qat_qp_hw_spec_gen4 = { + .qat_qp_rings_per_service = qat_qp_rings_per_service_gen4, + .qat_qp_build_ring_base = qat_qp_build_ring_base_gen4, + .qat_qp_adf_arb_enable = qat_qp_adf_arb_enable_gen4, + .qat_qp_adf_arb_disable = qat_qp_adf_arb_disable_gen4, + .qat_qp_adf_configure_queues = qat_qp_adf_configure_queues_gen4, + .qat_qp_csr_write_tail = qat_qp_csr_write_tail_gen4, + .qat_qp_csr_write_head = qat_qp_csr_write_head_gen4, + .qat_qp_csr_setup = qat_qp_csr_setup_gen4, +}; + +static int +qat_reset_ring_pairs_gen4(struct qat_pci_device *qat_pci_dev) +{ + int ret = 0, i; + uint8_t data[4]; + struct qat_pf2vf_msg pf2vf_msg; + + pf2vf_msg.msg_type = ADF_VF2PF_MSGTYPE_RP_RESET; + pf2vf_msg.block_hdr = -1; + for (i = 0; i < QAT_GEN4_BUNDLE_NUM; i++) { + pf2vf_msg.msg_data = i; + ret = qat_pf2vf_exch_msg(qat_pci_dev, pf2vf_msg, 1, data); + if (ret) { + QAT_LOG(ERR, "QAT error when reset bundle no %d", + i); + return ret; + } + } + + return 0; +} + +static const struct +rte_mem_resource *qat_dev_get_transport_bar_gen4(struct rte_pci_device *pci_dev) +{ + return &pci_dev->mem_resource[0]; +} + +static int +qat_dev_get_misc_bar_gen4( + struct rte_mem_resource **mem_resource, + struct rte_pci_device *pci_dev) +{ + *mem_resource = &pci_dev->mem_resource[2]; + return 0; +} + +static struct qat_dev_hw_spec_funcs qat_dev_hw_spec_gen4 = { + .qat_dev_reset_ring_pairs = qat_reset_ring_pairs_gen4, + .qat_dev_get_transport_bar = qat_dev_get_transport_bar_gen4, + .qat_dev_get_misc_bar = qat_dev_get_misc_bar_gen4, + .qat_dev_read_config = qat_dev_read_config_gen4, +}; + +RTE_INIT(qat_dev_gen_4_init) +{ + qat_qp_hw_spec[QAT_GEN4] = &qat_qp_hw_spec_gen4; + qat_dev_hw_spec[QAT_GEN4] = &qat_dev_hw_spec_gen4; + qat_gen_config[QAT_GEN4].dev_gen = QAT_GEN4; + qat_gen_config[QAT_GEN4].qp_hw_data = NULL; + qat_gen_config[QAT_GEN4].comp_num_im_bufs_required = + QAT_NUM_INTERM_BUFS_GEN3; + qat_gen_config[QAT_GEN4].pf2vf_dev = &qat_pf2vf_gen4; +} diff --git a/drivers/common/qat/meson.build b/drivers/common/qat/meson.build index 053c219fed..532e0fabb3 100644 --- a/drivers/common/qat/meson.build +++ b/drivers/common/qat/meson.build @@ -50,6 +50,10 @@ sources += files( 'qat_device.c', 'qat_logs.c', 'qat_pf2vf.c', + 'dev/qat_dev_gen1.c', + 'dev/qat_dev_gen2.c', + 'dev/qat_dev_gen3.c', + 'dev/qat_dev_gen4.c' ) includes += include_directories( 'qat_adf', diff --git a/drivers/common/qat/qat_common.h b/drivers/common/qat/qat_common.h index 23715085f4..b15e980f0f 100644 --- a/drivers/common/qat/qat_common.h +++ b/drivers/common/qat/qat_common.h @@ -22,6 +22,8 @@ enum qat_device_gen { QAT_GEN4 }; +#define QAT_DEV_GEN_NO (QAT_GEN4 + 1) + enum qat_service_type { QAT_SERVICE_ASYMMETRIC = 0, QAT_SERVICE_SYMMETRIC, diff --git a/drivers/common/qat/qat_device.c b/drivers/common/qat/qat_device.c index 1b967cbcf7..030624b46d 100644 --- a/drivers/common/qat/qat_device.c +++ b/drivers/common/qat/qat_device.c @@ -13,42 +13,10 @@ #include "adf_pf2vf_msg.h" #include "qat_pf2vf.h" -/* pv2vf data Gen 4*/ -struct qat_pf2vf_dev qat_pf2vf_gen4 = { - .pf2vf_offset = ADF_4XXXIOV_PF2VM_OFFSET, - .vf2pf_offset = ADF_4XXXIOV_VM2PF_OFFSET, - .pf2vf_type_shift = ADF_PFVF_2X_MSGTYPE_SHIFT, - .pf2vf_type_mask = ADF_PFVF_2X_MSGTYPE_MASK, - .pf2vf_data_shift = ADF_PFVF_2X_MSGDATA_SHIFT, - .pf2vf_data_mask = ADF_PFVF_2X_MSGDATA_MASK, -}; - /* Hardware device information per generation */ -__extension__ -struct qat_gen_hw_data qat_gen_config[] = { - [QAT_GEN1] = { - .dev_gen = QAT_GEN1, - .qp_hw_data = qat_gen1_qps, - .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN1 - }, - [QAT_GEN2] = { - .dev_gen = QAT_GEN2, - .qp_hw_data = qat_gen1_qps, - /* gen2 has same ring layout as gen1 */ - .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN2 - }, - [QAT_GEN3] = { - .dev_gen = QAT_GEN3, - .qp_hw_data = qat_gen3_qps, - .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN3 - }, - [QAT_GEN4] = { - .dev_gen = QAT_GEN4, - .qp_hw_data = NULL, - .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN3, - .pf2vf_dev = &qat_pf2vf_gen4 - }, -}; + +struct qat_gen_hw_data qat_gen_config[QAT_DEV_GEN_NO]; +struct qat_dev_hw_spec_funcs *qat_dev_hw_spec[QAT_DEV_GEN_NO]; /* per-process array of device data */ struct qat_device_info qat_pci_devs[RTE_PMD_QAT_MAX_PCI_DEVICES]; @@ -126,44 +94,6 @@ qat_get_qat_dev_from_pci_dev(struct rte_pci_device *pci_dev) return qat_pci_get_named_dev(name); } -static int -qat_gen4_reset_ring_pair(struct qat_pci_device *qat_pci_dev) -{ - int ret = 0, i; - uint8_t data[4]; - struct qat_pf2vf_msg pf2vf_msg; - - pf2vf_msg.msg_type = ADF_VF2PF_MSGTYPE_RP_RESET; - pf2vf_msg.block_hdr = -1; - for (i = 0; i < QAT_GEN4_BUNDLE_NUM; i++) { - pf2vf_msg.msg_data = i; - ret = qat_pf2vf_exch_msg(qat_pci_dev, pf2vf_msg, 1, data); - if (ret) { - QAT_LOG(ERR, "QAT error when reset bundle no %d", - i); - return ret; - } - } - - return 0; -} - -int qat_query_svc(struct qat_pci_device *qat_dev, uint8_t *val) -{ - int ret = -(EINVAL); - struct qat_pf2vf_msg pf2vf_msg; - - if (qat_dev->qat_dev_gen == QAT_GEN4) { - pf2vf_msg.msg_type = ADF_VF2PF_MSGTYPE_GET_SMALL_BLOCK_REQ; - pf2vf_msg.block_hdr = ADF_VF2PF_BLOCK_MSG_GET_RING_TO_SVC_REQ; - pf2vf_msg.msg_data = 2; - ret = qat_pf2vf_exch_msg(qat_dev, pf2vf_msg, 2, val); - } - - return ret; -} - - static void qat_dev_parse_cmd(const char *str, struct qat_dev_cmd_param *qat_dev_cmd_param) { @@ -229,6 +159,8 @@ qat_pci_device_allocate(struct rte_pci_device *pci_dev, uint8_t qat_dev_id = 0; char name[QAT_DEV_NAME_MAX_LEN]; struct rte_devargs *devargs = pci_dev->device.devargs; + struct qat_dev_hw_spec_funcs *ops_hw = NULL; + struct rte_mem_resource *mem_resource; rte_pci_device_name(&pci_dev->addr, name, sizeof(name)); snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat"); @@ -300,24 +232,25 @@ qat_pci_device_allocate(struct rte_pci_device *pci_dev, return NULL; } - if (qat_dev->qat_dev_gen == QAT_GEN4) { - qat_dev->misc_bar_io_addr = pci_dev->mem_resource[2].addr; - if (qat_dev->misc_bar_io_addr == NULL) { + ops_hw = qat_dev_hw_spec[qat_dev->qat_dev_gen]; + RTE_FUNC_PTR_OR_ERR_RET(ops_hw->qat_dev_get_misc_bar, NULL); + if (ops_hw->qat_dev_get_misc_bar(&mem_resource, pci_dev) == 0) { + if (mem_resource->addr == NULL) { QAT_LOG(ERR, "QAT cannot get access to VF misc bar"); return NULL; } - } + qat_dev->misc_bar_io_addr = mem_resource->addr; + } else + qat_dev->misc_bar_io_addr = NULL; if (devargs && devargs->drv_str) qat_dev_parse_cmd(devargs->drv_str, qat_dev_cmd_param); - if (qat_dev->qat_dev_gen >= QAT_GEN4) { - if (qat_read_qp_config(qat_dev)) { - QAT_LOG(ERR, - "Cannot acquire ring configuration for QAT_%d", - qat_dev_id); - return NULL; - } + if (qat_read_qp_config(qat_dev)) { + QAT_LOG(ERR, + "Cannot acquire ring configuration for QAT_%d", + qat_dev_id); + return NULL; } rte_spinlock_init(&qat_dev->arb_csr_lock); @@ -392,6 +325,7 @@ static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, int sym_ret = 0, asym_ret = 0, comp_ret = 0; int num_pmds_created = 0; struct qat_pci_device *qat_pci_dev; + struct qat_dev_hw_spec_funcs *ops; struct qat_dev_cmd_param qat_dev_cmd_param[] = { { SYM_ENQ_THRESHOLD_NAME, 0 }, { ASYM_ENQ_THRESHOLD_NAME, 0 }, @@ -408,13 +342,14 @@ static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, if (qat_pci_dev == NULL) return -ENODEV; - if (qat_pci_dev->qat_dev_gen == QAT_GEN4) { - if (qat_gen4_reset_ring_pair(qat_pci_dev)) { - QAT_LOG(ERR, - "Cannot reset ring pairs, does pf driver supports pf2vf comms?" - ); - return -ENODEV; - } + ops = qat_dev_hw_spec[qat_pci_dev->qat_dev_gen]; + RTE_FUNC_PTR_OR_ERR_RET(ops->qat_dev_reset_ring_pairs, + -ENOTSUP); + if (ops->qat_dev_reset_ring_pairs(qat_pci_dev)) { + QAT_LOG(ERR, + "Cannot reset ring pairs, does pf driver supports pf2vf comms?" + ); + return -ENODEV; } sym_ret = qat_sym_dev_create(qat_pci_dev, qat_dev_cmd_param); diff --git a/drivers/common/qat/qat_device.h b/drivers/common/qat/qat_device.h index 228c057d1e..531aa663ca 100644 --- a/drivers/common/qat/qat_device.h +++ b/drivers/common/qat/qat_device.h @@ -21,6 +21,24 @@ #define COMP_ENQ_THRESHOLD_NAME "qat_comp_enq_threshold" #define MAX_QP_THRESHOLD_SIZE 32 +typedef int (*qat_dev_reset_ring_pairs_t) + (struct qat_pci_device *); +typedef const struct rte_mem_resource* (*qat_dev_get_transport_bar_t) + (struct rte_pci_device *); +typedef int (*qat_dev_get_misc_bar_t) + (struct rte_mem_resource **, struct rte_pci_device *); +typedef int (*qat_dev_read_config_t) + (struct qat_pci_device *); + +struct qat_dev_hw_spec_funcs { + qat_dev_reset_ring_pairs_t qat_dev_reset_ring_pairs; + qat_dev_get_transport_bar_t qat_dev_get_transport_bar; + qat_dev_get_misc_bar_t qat_dev_get_misc_bar; + qat_dev_read_config_t qat_dev_read_config; +}; + +extern struct qat_dev_hw_spec_funcs *qat_dev_hw_spec[]; + struct qat_dev_cmd_param { const char *name; uint16_t val; @@ -57,6 +75,9 @@ struct qat_device_info { */ }; +extern const struct qat_qp_hw_data qat_gen1_qps[][ADF_MAX_QPS_ON_ANY_SERVICE]; +extern const struct qat_qp_hw_data qat_gen3_qps[][ADF_MAX_QPS_ON_ANY_SERVICE]; + extern struct qat_device_info qat_pci_devs[]; struct qat_sym_dev_private; @@ -159,7 +180,4 @@ qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused, int qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused); -int -qat_query_svc(struct qat_pci_device *qat_pci_dev, uint8_t *ret); - #endif /* _QAT_DEVICE_H_ */ diff --git a/drivers/common/qat/qat_qp.c b/drivers/common/qat/qat_qp.c index 026ea5ee01..ff4d7fa95c 100644 --- a/drivers/common/qat/qat_qp.c +++ b/drivers/common/qat/qat_qp.c @@ -18,119 +18,14 @@ #include "qat_sym.h" #include "qat_asym.h" #include "qat_comp.h" -#include "adf_transport_access_macros.h" -#include "adf_transport_access_macros_gen4vf.h" #define QAT_CQ_MAX_DEQ_RETRIES 10 #define ADF_MAX_DESC 4096 #define ADF_MIN_DESC 128 -#define ADF_ARB_REG_SLOT 0x1000 -#define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C - -#define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \ - ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \ - (ADF_ARB_REG_SLOT * index), value) - -__extension__ -const struct qat_qp_hw_data qat_gen1_qps[QAT_MAX_SERVICES] - [ADF_MAX_QPS_ON_ANY_SERVICE] = { - /* queue pairs which provide an asymmetric crypto service */ - [QAT_SERVICE_ASYMMETRIC] = { - { - .service_type = QAT_SERVICE_ASYMMETRIC, - .hw_bundle_num = 0, - .tx_ring_num = 0, - .rx_ring_num = 8, - .tx_msg_size = 64, - .rx_msg_size = 32, - - }, { - .service_type = QAT_SERVICE_ASYMMETRIC, - .hw_bundle_num = 0, - .tx_ring_num = 1, - .rx_ring_num = 9, - .tx_msg_size = 64, - .rx_msg_size = 32, - } - }, - /* queue pairs which provide a symmetric crypto service */ - [QAT_SERVICE_SYMMETRIC] = { - { - .service_type = QAT_SERVICE_SYMMETRIC, - .hw_bundle_num = 0, - .tx_ring_num = 2, - .rx_ring_num = 10, - .tx_msg_size = 128, - .rx_msg_size = 32, - }, - { - .service_type = QAT_SERVICE_SYMMETRIC, - .hw_bundle_num = 0, - .tx_ring_num = 3, - .rx_ring_num = 11, - .tx_msg_size = 128, - .rx_msg_size = 32, - } - }, - /* queue pairs which provide a compression service */ - [QAT_SERVICE_COMPRESSION] = { - { - .service_type = QAT_SERVICE_COMPRESSION, - .hw_bundle_num = 0, - .tx_ring_num = 6, - .rx_ring_num = 14, - .tx_msg_size = 128, - .rx_msg_size = 32, - }, { - .service_type = QAT_SERVICE_COMPRESSION, - .hw_bundle_num = 0, - .tx_ring_num = 7, - .rx_ring_num = 15, - .tx_msg_size = 128, - .rx_msg_size = 32, - } - } -}; - -__extension__ -const struct qat_qp_hw_data qat_gen3_qps[QAT_MAX_SERVICES] - [ADF_MAX_QPS_ON_ANY_SERVICE] = { - /* queue pairs which provide an asymmetric crypto service */ - [QAT_SERVICE_ASYMMETRIC] = { - { - .service_type = QAT_SERVICE_ASYMMETRIC, - .hw_bundle_num = 0, - .tx_ring_num = 0, - .rx_ring_num = 4, - .tx_msg_size = 64, - .rx_msg_size = 32, - } - }, - /* queue pairs which provide a symmetric crypto service */ - [QAT_SERVICE_SYMMETRIC] = { - { - .service_type = QAT_SERVICE_SYMMETRIC, - .hw_bundle_num = 0, - .tx_ring_num = 1, - .rx_ring_num = 5, - .tx_msg_size = 128, - .rx_msg_size = 32, - } - }, - /* queue pairs which provide a compression service */ - [QAT_SERVICE_COMPRESSION] = { - { - .service_type = QAT_SERVICE_COMPRESSION, - .hw_bundle_num = 0, - .tx_ring_num = 3, - .rx_ring_num = 7, - .tx_msg_size = 128, - .rx_msg_size = 32, - } - } -}; +struct qat_qp_hw_spec_funcs* + qat_qp_hw_spec[QAT_DEV_GEN_NO]; static int qat_qp_check_queue_alignment(uint64_t phys_addr, uint32_t queue_size_bytes); @@ -139,66 +34,19 @@ static int qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue, struct qat_qp_config *, uint8_t dir); static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num, uint32_t *queue_size_for_csr); -static void adf_configure_queues(struct qat_qp *queue, +static int adf_configure_queues(struct qat_qp *queue, enum qat_device_gen qat_dev_gen); -static void adf_queue_arb_enable(enum qat_device_gen qat_dev_gen, +static int adf_queue_arb_enable(struct qat_pci_device *qat_dev, struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock); -static void adf_queue_arb_disable(enum qat_device_gen qat_dev_gen, +static int adf_queue_arb_disable(enum qat_device_gen qat_dev_gen, struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock); - -int qat_qps_per_service(struct qat_pci_device *qat_dev, - enum qat_service_type service) -{ - int i = 0, count = 0, max_ops_per_srv = 0; - - if (qat_dev->qat_dev_gen == QAT_GEN4) { - max_ops_per_srv = QAT_GEN4_BUNDLE_NUM; - for (i = 0, count = 0; i < max_ops_per_srv; i++) - if (qat_dev->qp_gen4_data[i][0].service_type == service) - count++; - } else { - const struct qat_qp_hw_data *sym_hw_qps = - qat_gen_config[qat_dev->qat_dev_gen] - .qp_hw_data[service]; - - max_ops_per_srv = ADF_MAX_QPS_ON_ANY_SERVICE; - for (i = 0, count = 0; i < max_ops_per_srv; i++) - if (sym_hw_qps[i].service_type == service) - count++; - } - - return count; -} - +static int qat_qp_build_ring_base(struct qat_pci_device *qat_dev, + void *io_addr, struct qat_queue *queue); static const struct rte_memzone * -queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size, - int socket_id) -{ - const struct rte_memzone *mz; - - mz = rte_memzone_lookup(queue_name); - if (mz != 0) { - if (((size_t)queue_size <= mz->len) && - ((socket_id == SOCKET_ID_ANY) || - (socket_id == mz->socket_id))) { - QAT_LOG(DEBUG, "re-use memzone already " - "allocated for %s", queue_name); - return mz; - } - - QAT_LOG(ERR, "Incompatible memzone already " - "allocated %s, size %u, socket %d. " - "Requested size %u, socket %u", - queue_name, (uint32_t)mz->len, - mz->socket_id, queue_size, socket_id); - return NULL; - } - - QAT_LOG(DEBUG, "Allocate memzone for %s, size %u on socket %u", - queue_name, queue_size, socket_id); - return rte_memzone_reserve_aligned(queue_name, queue_size, - socket_id, RTE_MEMZONE_IOVA_CONTIG, queue_size); -} + queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size, + int socket_id); +static int qat_qp_csr_setup(struct qat_pci_device *qat_dev, + void *io_addr, struct qat_qp *qp); int qat_qp_setup(struct qat_pci_device *qat_dev, struct qat_qp **qp_addr, @@ -209,8 +57,10 @@ int qat_qp_setup(struct qat_pci_device *qat_dev, struct rte_pci_device *pci_dev = qat_pci_devs[qat_dev->qat_dev_id].pci_dev; char op_cookie_pool_name[RTE_RING_NAMESIZE]; - enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen; uint32_t i; + struct qat_dev_hw_spec_funcs *ops_hw = + qat_dev_hw_spec[qat_dev->qat_dev_gen]; + void *io_addr; QAT_LOG(DEBUG, "Setup qp %u on qat pci device %d gen %d", queue_pair_id, qat_dev->qat_dev_id, qat_dev->qat_dev_gen); @@ -222,7 +72,10 @@ int qat_qp_setup(struct qat_pci_device *qat_dev, return -EINVAL; } - if (pci_dev->mem_resource[0].addr == NULL) { + RTE_FUNC_PTR_OR_ERR_RET(ops_hw->qat_dev_get_transport_bar, + -ENOTSUP); + io_addr = ops_hw->qat_dev_get_transport_bar(pci_dev)->addr; + if (io_addr == NULL) { QAT_LOG(ERR, "Could not find VF config space " "(UIO driver attached?)."); return -EINVAL; @@ -246,7 +99,7 @@ int qat_qp_setup(struct qat_pci_device *qat_dev, return -ENOMEM; } - qp->mmap_bar_addr = pci_dev->mem_resource[0].addr; + qp->mmap_bar_addr = io_addr; qp->enqueued = qp->dequeued = 0; if (qat_queue_create(qat_dev, &(qp->tx_q), qat_qp_conf, @@ -273,10 +126,6 @@ int qat_qp_setup(struct qat_pci_device *qat_dev, goto create_err; } - adf_configure_queues(qp, qat_dev_gen); - adf_queue_arb_enable(qat_dev_gen, &qp->tx_q, qp->mmap_bar_addr, - &qat_dev->arb_csr_lock); - snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE, "%s%d_cookies_%s_qp%hu", pci_dev->driver->driver.name, qat_dev->qat_dev_id, @@ -312,6 +161,8 @@ int qat_qp_setup(struct qat_pci_device *qat_dev, QAT_LOG(DEBUG, "QP setup complete: id: %d, cookiepool: %s", queue_pair_id, op_cookie_pool_name); + qat_qp_csr_setup(qat_dev, io_addr, qp); + *qp_addr = qp; return 0; @@ -323,80 +174,13 @@ int qat_qp_setup(struct qat_pci_device *qat_dev, return -EFAULT; } - -int qat_qp_release(enum qat_device_gen qat_dev_gen, struct qat_qp **qp_addr) -{ - struct qat_qp *qp = *qp_addr; - uint32_t i; - - if (qp == NULL) { - QAT_LOG(DEBUG, "qp already freed"); - return 0; - } - - QAT_LOG(DEBUG, "Free qp on qat_pci device %d", - qp->qat_dev->qat_dev_id); - - /* Don't free memory if there are still responses to be processed */ - if ((qp->enqueued - qp->dequeued) == 0) { - qat_queue_delete(&(qp->tx_q)); - qat_queue_delete(&(qp->rx_q)); - } else { - return -EAGAIN; - } - - adf_queue_arb_disable(qat_dev_gen, &(qp->tx_q), qp->mmap_bar_addr, - &qp->qat_dev->arb_csr_lock); - - for (i = 0; i < qp->nb_descriptors; i++) - rte_mempool_put(qp->op_cookie_pool, qp->op_cookies[i]); - - if (qp->op_cookie_pool) - rte_mempool_free(qp->op_cookie_pool); - - rte_free(qp->op_cookies); - rte_free(qp); - *qp_addr = NULL; - return 0; -} - - -static void qat_queue_delete(struct qat_queue *queue) -{ - const struct rte_memzone *mz; - int status = 0; - - if (queue == NULL) { - QAT_LOG(DEBUG, "Invalid queue"); - return; - } - QAT_LOG(DEBUG, "Free ring %d, memzone: %s", - queue->hw_queue_number, queue->memz_name); - - mz = rte_memzone_lookup(queue->memz_name); - if (mz != NULL) { - /* Write an unused pattern to the queue memory. */ - memset(queue->base_addr, 0x7F, queue->queue_size); - status = rte_memzone_free(mz); - if (status != 0) - QAT_LOG(ERR, "Error %d on freeing queue %s", - status, queue->memz_name); - } else { - QAT_LOG(DEBUG, "queue %s doesn't exist", - queue->memz_name); - } -} - static int qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue, struct qat_qp_config *qp_conf, uint8_t dir) { - uint64_t queue_base; - void *io_addr; const struct rte_memzone *qp_mz; struct rte_pci_device *pci_dev = qat_pci_devs[qat_dev->qat_dev_id].pci_dev; - enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen; int ret = 0; uint16_t desc_size = (dir == ADF_RING_DIR_TX ? qp_conf->hw->tx_msg_size : qp_conf->hw->rx_msg_size); @@ -456,19 +240,6 @@ qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue, * Write an unused pattern to the queue memory. */ memset(queue->base_addr, 0x7F, queue_size_bytes); - io_addr = pci_dev->mem_resource[0].addr; - - if (qat_dev_gen == QAT_GEN4) { - queue_base = BUILD_RING_BASE_ADDR_GEN4(queue->base_phys_addr, - queue->queue_size); - WRITE_CSR_RING_BASE_GEN4VF(io_addr, queue->hw_bundle_number, - queue->hw_queue_number, queue_base); - } else { - queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr, - queue->queue_size); - WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number, - queue->hw_queue_number, queue_base); - } QAT_LOG(DEBUG, "RING: Name:%s, size in CSR: %u, in bytes %u," " nb msgs %u, msg_size %u, modulo mask %u", @@ -484,202 +255,216 @@ qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue, return ret; } -int -qat_select_valid_queue(struct qat_pci_device *qat_dev, int qp_id, - enum qat_service_type service_type) +static const struct rte_memzone * +queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size, + int socket_id) { - if (qat_dev->qat_dev_gen == QAT_GEN4) { - int i = 0, valid_qps = 0; - - for (; i < QAT_GEN4_BUNDLE_NUM; i++) { - if (qat_dev->qp_gen4_data[i][0].service_type == - service_type) { - if (valid_qps == qp_id) - return i; - ++valid_qps; - } + const struct rte_memzone *mz; + + mz = rte_memzone_lookup(queue_name); + if (mz != 0) { + if (((size_t)queue_size <= mz->len) && + ((socket_id == SOCKET_ID_ANY) || + (socket_id == mz->socket_id))) { + QAT_LOG(DEBUG, + "re-use memzone already allocated for %s", + queue_name); + return mz; } + + QAT_LOG(ERR, + "Incompatible memzone already allocated %s, size %u, socket %d. Requested size %u, socket %u", + queue_name, (uint32_t)mz->len, + mz->socket_id, queue_size, socket_id); + return NULL; } - return -1; + + QAT_LOG(DEBUG, "Allocate memzone for %s, size %u on socket %u", + queue_name, queue_size, socket_id); + return rte_memzone_reserve_aligned(queue_name, queue_size, + socket_id, RTE_MEMZONE_IOVA_CONTIG, queue_size); } -int -qat_read_qp_config(struct qat_pci_device *qat_dev) +int qat_qp_release(enum qat_device_gen qat_dev_gen, struct qat_qp **qp_addr) { - int i = 0; - enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen; - - if (qat_dev_gen == QAT_GEN4) { - uint16_t svc = 0; - - if (qat_query_svc(qat_dev, (uint8_t *)&svc)) - return -(EFAULT); - for (; i < QAT_GEN4_BUNDLE_NUM; i++) { - struct qat_qp_hw_data *hw_data = - &qat_dev->qp_gen4_data[i][0]; - uint8_t svc1 = (svc >> (3 * i)) & 0x7; - enum qat_service_type service_type = QAT_SERVICE_INVALID; - - if (svc1 == QAT_SVC_SYM) { - service_type = QAT_SERVICE_SYMMETRIC; - QAT_LOG(DEBUG, - "Discovered SYMMETRIC service on bundle %d", - i); - } else if (svc1 == QAT_SVC_COMPRESSION) { - service_type = QAT_SERVICE_COMPRESSION; - QAT_LOG(DEBUG, - "Discovered COPRESSION service on bundle %d", - i); - } else if (svc1 == QAT_SVC_ASYM) { - service_type = QAT_SERVICE_ASYMMETRIC; - QAT_LOG(DEBUG, - "Discovered ASYMMETRIC service on bundle %d", - i); - } else { - QAT_LOG(ERR, - "Unrecognized service on bundle %d", - i); - return -(EFAULT); - } + int ret; + struct qat_qp *qp = *qp_addr; + uint32_t i; - memset(hw_data, 0, sizeof(*hw_data)); - hw_data->service_type = service_type; - if (service_type == QAT_SERVICE_ASYMMETRIC) { - hw_data->tx_msg_size = 64; - hw_data->rx_msg_size = 32; - } else if (service_type == QAT_SERVICE_SYMMETRIC || - service_type == - QAT_SERVICE_COMPRESSION) { - hw_data->tx_msg_size = 128; - hw_data->rx_msg_size = 32; - } - hw_data->tx_ring_num = 0; - hw_data->rx_ring_num = 1; - hw_data->hw_bundle_num = i; - } + if (qp == NULL) { + QAT_LOG(DEBUG, "qp already freed"); return 0; } - return -(EINVAL); -} -static int qat_qp_check_queue_alignment(uint64_t phys_addr, - uint32_t queue_size_bytes) -{ - if (((queue_size_bytes - 1) & phys_addr) != 0) - return -EINVAL; + QAT_LOG(DEBUG, "Free qp on qat_pci device %d", + qp->qat_dev->qat_dev_id); + + /* Don't free memory if there are still responses to be processed */ + if ((qp->enqueued - qp->dequeued) == 0) { + qat_queue_delete(&(qp->tx_q)); + qat_queue_delete(&(qp->rx_q)); + } else { + return -EAGAIN; + } + + ret = adf_queue_arb_disable(qat_dev_gen, &(qp->tx_q), qp->mmap_bar_addr, + &qp->qat_dev->arb_csr_lock); + if (ret) + return ret; + + for (i = 0; i < qp->nb_descriptors; i++) + rte_mempool_put(qp->op_cookie_pool, qp->op_cookies[i]); + + if (qp->op_cookie_pool) + rte_mempool_free(qp->op_cookie_pool); + + rte_free(qp->op_cookies); + rte_free(qp); + *qp_addr = NULL; return 0; } -static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num, - uint32_t *p_queue_size_for_csr) + +static void qat_queue_delete(struct qat_queue *queue) { - uint8_t i = ADF_MIN_RING_SIZE; + const struct rte_memzone *mz; + int status = 0; - for (; i <= ADF_MAX_RING_SIZE; i++) - if ((msg_size * msg_num) == - (uint32_t)ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) { - *p_queue_size_for_csr = i; - return 0; - } - QAT_LOG(ERR, "Invalid ring size %d", msg_size * msg_num); - return -EINVAL; + if (queue == NULL) { + QAT_LOG(DEBUG, "Invalid queue"); + return; + } + QAT_LOG(DEBUG, "Free ring %d, memzone: %s", + queue->hw_queue_number, queue->memz_name); + + mz = rte_memzone_lookup(queue->memz_name); + if (mz != NULL) { + /* Write an unused pattern to the queue memory. */ + memset(queue->base_addr, 0x7F, queue->queue_size); + status = rte_memzone_free(mz); + if (status != 0) + QAT_LOG(ERR, "Error %d on freeing queue %s", + status, queue->memz_name); + } else { + QAT_LOG(DEBUG, "queue %s doesn't exist", + queue->memz_name); + } } -static void -adf_queue_arb_enable(enum qat_device_gen qat_dev_gen, struct qat_queue *txq, +static int __rte_unused +adf_queue_arb_enable(struct qat_pci_device *qat_dev, struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock) { - uint32_t arb_csr_offset = 0, value; - - rte_spinlock_lock(lock); - if (qat_dev_gen == QAT_GEN4) { - arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET + - (ADF_RING_BUNDLE_SIZE_GEN4 * - txq->hw_bundle_number); - value = ADF_CSR_RD(base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, - arb_csr_offset); - } else { - arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET + - (ADF_ARB_REG_SLOT * - txq->hw_bundle_number); - value = ADF_CSR_RD(base_addr, - arb_csr_offset); - } - value |= (0x01 << txq->hw_queue_number); - ADF_CSR_WR(base_addr, arb_csr_offset, value); - rte_spinlock_unlock(lock); + struct qat_qp_hw_spec_funcs *ops = + qat_qp_hw_spec[qat_dev->qat_dev_gen]; + + RTE_FUNC_PTR_OR_ERR_RET(ops->qat_qp_adf_arb_enable, + -ENOTSUP); + ops->qat_qp_adf_arb_enable(txq, base_addr, lock); + return 0; } -static void adf_queue_arb_disable(enum qat_device_gen qat_dev_gen, +static int adf_queue_arb_disable(enum qat_device_gen qat_dev_gen, struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock) { - uint32_t arb_csr_offset = 0, value; - - rte_spinlock_lock(lock); - if (qat_dev_gen == QAT_GEN4) { - arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET + - (ADF_RING_BUNDLE_SIZE_GEN4 * - txq->hw_bundle_number); - value = ADF_CSR_RD(base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, - arb_csr_offset); - } else { - arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET + - (ADF_ARB_REG_SLOT * - txq->hw_bundle_number); - value = ADF_CSR_RD(base_addr, - arb_csr_offset); - } - value &= ~(0x01 << txq->hw_queue_number); - ADF_CSR_WR(base_addr, arb_csr_offset, value); - rte_spinlock_unlock(lock); + struct qat_qp_hw_spec_funcs *ops = + qat_qp_hw_spec[qat_dev_gen]; + + RTE_FUNC_PTR_OR_ERR_RET(ops->qat_qp_adf_arb_disable, + -ENOTSUP); + ops->qat_qp_adf_arb_disable(txq, base_addr, lock); + return 0; } -static void adf_configure_queues(struct qat_qp *qp, - enum qat_device_gen qat_dev_gen) +static int __rte_unused +qat_qp_build_ring_base(struct qat_pci_device *qat_dev, void *io_addr, + struct qat_queue *queue) { - uint32_t q_tx_config, q_resp_config; - struct qat_queue *q_tx = &qp->tx_q, *q_rx = &qp->rx_q; - - q_tx_config = BUILD_RING_CONFIG(q_tx->queue_size); - q_resp_config = BUILD_RESP_RING_CONFIG(q_rx->queue_size, - ADF_RING_NEAR_WATERMARK_512, - ADF_RING_NEAR_WATERMARK_0); - - if (qat_dev_gen == QAT_GEN4) { - WRITE_CSR_RING_CONFIG_GEN4VF(qp->mmap_bar_addr, - q_tx->hw_bundle_number, q_tx->hw_queue_number, - q_tx_config); - WRITE_CSR_RING_CONFIG_GEN4VF(qp->mmap_bar_addr, - q_rx->hw_bundle_number, q_rx->hw_queue_number, - q_resp_config); - } else { - WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, - q_tx->hw_bundle_number, q_tx->hw_queue_number, - q_tx_config); - WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, - q_rx->hw_bundle_number, q_rx->hw_queue_number, - q_resp_config); - } + struct qat_qp_hw_spec_funcs *ops = + qat_qp_hw_spec[qat_dev->qat_dev_gen]; + + RTE_FUNC_PTR_OR_ERR_RET(ops->qat_qp_build_ring_base, + -ENOTSUP); + ops->qat_qp_build_ring_base(io_addr, queue); + return 0; } -static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask) +int qat_qps_per_service(struct qat_pci_device *qat_dev, + enum qat_service_type service) { - return data & modulo_mask; + struct qat_qp_hw_spec_funcs *ops = + qat_qp_hw_spec[qat_dev->qat_dev_gen]; + + RTE_FUNC_PTR_OR_ERR_RET(ops->qat_qp_rings_per_service, + -ENOTSUP); + return ops->qat_qp_rings_per_service(qat_dev, service); +} + +int +qat_read_qp_config(struct qat_pci_device *qat_dev) +{ + struct qat_dev_hw_spec_funcs *ops_hw = + qat_dev_hw_spec[qat_dev->qat_dev_gen]; + + RTE_FUNC_PTR_OR_ERR_RET(ops_hw->qat_dev_read_config, + -ENOTSUP); + return ops_hw->qat_dev_read_config(qat_dev); +} + +static int __rte_unused +adf_configure_queues(struct qat_qp *qp, enum qat_device_gen qat_dev_gen) +{ + struct qat_qp_hw_spec_funcs *ops = + qat_qp_hw_spec[qat_dev_gen]; + + RTE_FUNC_PTR_OR_ERR_RET(ops->qat_qp_adf_configure_queues, + -ENOTSUP); + ops->qat_qp_adf_configure_queues(qp); + return 0; } static inline void txq_write_tail(enum qat_device_gen qat_dev_gen, - struct qat_qp *qp, struct qat_queue *q) { + struct qat_qp *qp, struct qat_queue *q) +{ + struct qat_qp_hw_spec_funcs *ops = + qat_qp_hw_spec[qat_dev_gen]; - if (qat_dev_gen == QAT_GEN4) { - WRITE_CSR_RING_TAIL_GEN4VF(qp->mmap_bar_addr, - q->hw_bundle_number, q->hw_queue_number, q->tail); - } else { - WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number, - q->hw_queue_number, q->tail); - } + /* + * Pointer check should be done during + * initialization + */ + ops->qat_qp_csr_write_tail(qp, q); } +static inline void +qat_qp_csr_write_head(enum qat_device_gen qat_dev_gen, struct qat_qp *qp, + struct qat_queue *q, uint32_t new_head) +{ + struct qat_qp_hw_spec_funcs *ops = + qat_qp_hw_spec[qat_dev_gen]; + + /* + * Pointer check should be done during + * initialization + */ + ops->qat_qp_csr_write_head(qp, q, new_head); +} + +static int +qat_qp_csr_setup(struct qat_pci_device *qat_dev, + void *io_addr, struct qat_qp *qp) +{ + struct qat_qp_hw_spec_funcs *ops = + qat_qp_hw_spec[qat_dev->qat_dev_gen]; + + RTE_FUNC_PTR_OR_ERR_RET(ops->qat_qp_csr_setup, + -ENOTSUP); + ops->qat_qp_csr_setup(qat_dev, io_addr, qp); + return 0; +} + + static inline void rxq_free_desc(enum qat_device_gen qat_dev_gen, struct qat_qp *qp, struct qat_queue *q) @@ -703,15 +488,35 @@ void rxq_free_desc(enum qat_device_gen qat_dev_gen, struct qat_qp *qp, q->nb_processed_responses = 0; q->csr_head = new_head; - /* write current head to CSR */ - if (qat_dev_gen == QAT_GEN4) { - WRITE_CSR_RING_HEAD_GEN4VF(qp->mmap_bar_addr, - q->hw_bundle_number, q->hw_queue_number, new_head); - } else { - WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number, - q->hw_queue_number, new_head); - } + qat_qp_csr_write_head(qat_dev_gen, qp, q, new_head); +} + +static int qat_qp_check_queue_alignment(uint64_t phys_addr, + uint32_t queue_size_bytes) +{ + if (((queue_size_bytes - 1) & phys_addr) != 0) + return -EINVAL; + return 0; +} + +static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num, + uint32_t *p_queue_size_for_csr) +{ + uint8_t i = ADF_MIN_RING_SIZE; + for (; i <= ADF_MAX_RING_SIZE; i++) + if ((msg_size * msg_num) == + (uint32_t)ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) { + *p_queue_size_for_csr = i; + return 0; + } + QAT_LOG(ERR, "Invalid ring size %d", msg_size * msg_num); + return -EINVAL; +} + +static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask) +{ + return data & modulo_mask; } uint16_t diff --git a/drivers/common/qat/qat_qp.h b/drivers/common/qat/qat_qp.h index e1627197fa..ffba3a3615 100644 --- a/drivers/common/qat/qat_qp.h +++ b/drivers/common/qat/qat_qp.h @@ -24,6 +24,8 @@ struct qat_pci_device; #define QAT_GEN4_BUNDLE_NUM 4 #define QAT_GEN4_QPS_PER_BUNDLE_NUM 1 +#define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C + /** * Structure with data needed for creation of queue pair. */ @@ -96,9 +98,6 @@ struct qat_qp { uint16_t min_enq_burst_threshold; } __rte_cache_aligned; -extern const struct qat_qp_hw_data qat_gen1_qps[][ADF_MAX_QPS_ON_ANY_SERVICE]; -extern const struct qat_qp_hw_data qat_gen3_qps[][ADF_MAX_QPS_ON_ANY_SERVICE]; - uint16_t qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops); @@ -129,11 +128,43 @@ qat_comp_process_response(void **op __rte_unused, uint8_t *resp __rte_unused, void *op_cookie __rte_unused, uint64_t *dequeue_err_count __rte_unused); -int -qat_select_valid_queue(struct qat_pci_device *qat_dev, int qp_id, - enum qat_service_type service_type); - int qat_read_qp_config(struct qat_pci_device *qat_dev); +typedef int (*qat_qp_rings_per_service_t) + (struct qat_pci_device *, enum qat_service_type); +typedef void (*qat_qp_build_ring_base_t) + (void *, struct qat_queue *); +typedef void (*qat_qp_adf_arb_enable_t) + (const struct qat_queue *, void *, + rte_spinlock_t *); +typedef void (*qat_qp_adf_arb_disable_t) + (const struct qat_queue *, void *, + rte_spinlock_t *); +typedef void (*qat_qp_adf_configure_queues_t)(struct qat_qp *); + +typedef void (*qat_qp_csr_write_tail_t)(struct qat_qp *qp, + struct qat_queue *q); + +typedef void (*qat_qp_csr_write_head_t)(struct qat_qp *qp, + struct qat_queue *q, + uint32_t new_head); + +typedef void (*qat_qp_csr_setup_t)(struct qat_pci_device*, + void *, struct qat_qp *); + +struct qat_qp_hw_spec_funcs { + qat_qp_rings_per_service_t qat_qp_rings_per_service; + qat_qp_build_ring_base_t qat_qp_build_ring_base; + qat_qp_adf_arb_enable_t qat_qp_adf_arb_enable; + qat_qp_adf_arb_disable_t qat_qp_adf_arb_disable; + qat_qp_adf_configure_queues_t qat_qp_adf_configure_queues; + qat_qp_csr_write_tail_t qat_qp_csr_write_tail; + qat_qp_csr_write_head_t qat_qp_csr_write_head; + qat_qp_csr_setup_t qat_qp_csr_setup; +}; + +extern struct +qat_qp_hw_spec_funcs *qat_qp_hw_spec[]; + #endif /* _QAT_QP_H_ */ -- 2.30.2