Signed-off-by: Geoff Thorpe <geoff.tho...@nxp.com>
Signed-off-by: Roy Pledge <roy.ple...@nxp.com>
Signed-off-by: Hemant Agrawal <hemant.agra...@nxp.com>
Signed-off-by: Shreyansh Jain <shreyansh.j...@nxp.com>
---
 drivers/bus/dpaa/Makefile                 |   1 +
 drivers/bus/dpaa/base/qbman/bman.c        | 394 +++++++++++++++++++++
 drivers/bus/dpaa/base/qbman/bman.h        | 550 ++++++++++++++++++++++++++++++
 drivers/bus/dpaa/base/qbman/bman_driver.c |  12 +
 drivers/bus/dpaa/base/qbman/dpaa_alloc.c  |  16 +
 5 files changed, 973 insertions(+)
 create mode 100644 drivers/bus/dpaa/base/qbman/bman.c
 create mode 100644 drivers/bus/dpaa/base/qbman/bman.h

diff --git a/drivers/bus/dpaa/Makefile b/drivers/bus/dpaa/Makefile
index 24dfa13..6d0c5ee 100644
--- a/drivers/bus/dpaa/Makefile
+++ b/drivers/bus/dpaa/Makefile
@@ -71,6 +71,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_DPAA_BUS) += \
        base/fman/of.c \
        base/fman/netcfg_layer.c \
        base/qbman/process.c \
+       base/qbman/bman.c \
        base/qbman/bman_driver.c \
        base/qbman/qman.c \
        base/qbman/qman_driver.c \
diff --git a/drivers/bus/dpaa/base/qbman/bman.c 
b/drivers/bus/dpaa/base/qbman/bman.c
new file mode 100644
index 0000000..a0bea62
--- /dev/null
+++ b/drivers/bus/dpaa/base/qbman/bman.c
@@ -0,0 +1,394 @@
+/*-
+ * This file is provided under a dual BSD/GPLv2 license. When using or
+ * redistributing this file, you may do so under either license.
+ *
+ *   BSD LICENSE
+ *
+ * Copyright 2008-2016 Freescale Semiconductor Inc.
+ * Copyright 2017 NXP.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of the above-listed copyright holders nor the
+ * names of any contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ *   GPL LICENSE SUMMARY
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "bman.h"
+#include <rte_branch_prediction.h>
+
+/* Compilation constants */
+#define RCR_THRESH     2       /* reread h/w CI when running out of space */
+#define IRQNAME                "BMan portal %d"
+#define MAX_IRQNAME    16      /* big enough for "BMan portal %d" */
+
+struct bman_portal {
+       struct bm_portal p;
+       /* 2-element array. pools[0] is mask, pools[1] is snapshot. */
+       struct bman_depletion *pools;
+       int thresh_set;
+       unsigned long irq_sources;
+       u32 slowpoll;   /* only used when interrupts are off */
+       /* When the cpu-affine portal is activated, this is non-NULL */
+       const struct bm_portal_config *config;
+       char irqname[MAX_IRQNAME];
+};
+
+static cpumask_t affine_mask;
+static DEFINE_SPINLOCK(affine_mask_lock);
+static DEFINE_PER_CPU(struct bman_portal, bman_affine_portal);
+
+static inline struct bman_portal *get_affine_portal(void)
+{
+       return &get_cpu_var(bman_affine_portal);
+}
+
+/*
+ * This object type refers to a pool, it isn't *the* pool. There may be
+ * more than one such object per BMan buffer pool, eg. if different users of
+ * the pool are operating via different portals.
+ */
+struct bman_pool {
+       struct bman_pool_params params;
+       /* Used for hash-table admin when using depletion notifications. */
+       struct bman_portal *portal;
+       struct bman_pool *next;
+#ifdef RTE_LIBRTE_DPAA_CHECKING
+       atomic_t in_use;
+#endif
+};
+
+static inline
+struct bman_portal *bman_create_portal(struct bman_portal *portal,
+                                      const struct bm_portal_config *c)
+{
+       struct bm_portal *p;
+       const struct bman_depletion *pools = &c->mask;
+       int ret;
+       u8 bpid = 0;
+
+       p = &portal->p;
+       /*
+        * prep the low-level portal struct with the mapped addresses from the
+        * config, everything that follows depends on it and "config" is more
+        * for (de)reference...
+        */
+       p->addr.ce = c->addr_virt[DPAA_PORTAL_CE];
+       p->addr.ci = c->addr_virt[DPAA_PORTAL_CI];
+       if (bm_rcr_init(p, bm_rcr_pvb, bm_rcr_cce)) {
+               pr_err("Bman RCR initialisation failed\n");
+               return NULL;
+       }
+       if (bm_mc_init(p)) {
+               pr_err("Bman MC initialisation failed\n");
+               goto fail_mc;
+       }
+       portal->pools = kmalloc(2 * sizeof(*pools), GFP_KERNEL);
+       if (!portal->pools)
+               goto fail_pools;
+       portal->pools[0] = *pools;
+       bman_depletion_init(portal->pools + 1);
+       while (bpid < bman_pool_max) {
+               /*
+                * Default to all BPIDs disabled, we enable as required at
+                * run-time.
+                */
+               bm_isr_bscn_mask(p, bpid, 0);
+               bpid++;
+       }
+       portal->slowpoll = 0;
+       /* Write-to-clear any stale interrupt status bits */
+       bm_isr_disable_write(p, 0xffffffff);
+       portal->irq_sources = 0;
+       bm_isr_enable_write(p, portal->irq_sources);
+       bm_isr_status_clear(p, 0xffffffff);
+       snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu);
+       if (request_irq(c->irq, NULL, 0, portal->irqname,
+                       portal)) {
+               pr_err("request_irq() failed\n");
+               goto fail_irq;
+       }
+
+       /* Need RCR to be empty before continuing */
+       ret = bm_rcr_get_fill(p);
+       if (ret) {
+               pr_err("Bman RCR unclean\n");
+               goto fail_rcr_empty;
+       }
+       /* Success */
+       portal->config = c;
+
+       bm_isr_disable_write(p, 0);
+       bm_isr_uninhibit(p);
+       return portal;
+fail_rcr_empty:
+       free_irq(c->irq, portal);
+fail_irq:
+       kfree(portal->pools);
+fail_pools:
+       bm_mc_finish(p);
+fail_mc:
+       bm_rcr_finish(p);
+       return NULL;
+}
+
+struct bman_portal *
+bman_create_affine_portal(const struct bm_portal_config *c)
+{
+       struct bman_portal *portal = get_affine_portal();
+
+       /*This function is called from the context which is already affine to
+        *CPU or in other words this in non-migratable to other CPUs.
+        */
+       portal = bman_create_portal(portal, c);
+       if (portal) {
+               spin_lock(&affine_mask_lock);
+               CPU_SET(c->cpu, &affine_mask);
+               spin_unlock(&affine_mask_lock);
+       }
+       return portal;
+}
+
+static inline
+void bman_destroy_portal(struct bman_portal *bm)
+{
+       const struct bm_portal_config *pcfg;
+
+       pcfg = bm->config;
+       bm_rcr_cce_update(&bm->p);
+       bm_rcr_cce_update(&bm->p);
+
+       free_irq(pcfg->irq, bm);
+
+       kfree(bm->pools);
+       bm_mc_finish(&bm->p);
+       bm_rcr_finish(&bm->p);
+       bm->config = NULL;
+}
+
+const struct
+bm_portal_config *bman_destroy_affine_portal(void)
+{
+       struct bman_portal *bm = get_affine_portal();
+       const struct bm_portal_config *pcfg;
+
+       pcfg = bm->config;
+       bman_destroy_portal(bm);
+       spin_lock(&affine_mask_lock);
+       CPU_CLR(pcfg->cpu, &affine_mask);
+       spin_unlock(&affine_mask_lock);
+       return pcfg;
+}
+
+int
+bman_get_portal_index(void)
+{
+       struct bman_portal *p = get_affine_portal();
+       return p->config->index;
+}
+
+static const u32 zero_thresholds[4] = {0, 0, 0, 0};
+
+struct bman_pool *bman_new_pool(const struct bman_pool_params *params)
+{
+       struct bman_pool *pool = NULL;
+       u32 bpid;
+
+       if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID) {
+               int ret = bman_alloc_bpid(&bpid);
+
+               if (ret)
+                       return NULL;
+       } else {
+               if (params->bpid >= bman_pool_max)
+                       return NULL;
+               bpid = params->bpid;
+       }
+       if (params->flags & BMAN_POOL_FLAG_THRESH) {
+               int ret = bm_pool_set(bpid, params->thresholds);
+
+               if (ret)
+                       goto err;
+       }
+
+       pool = kmalloc(sizeof(*pool), GFP_KERNEL);
+       if (!pool)
+               goto err;
+       pool->params = *params;
+#ifdef RTE_LIBRTE_DPAA_CHECKING
+       atomic_set(&pool->in_use, 1);
+#endif
+       if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID)
+               pool->params.bpid = bpid;
+
+       return pool;
+err:
+       if (params->flags & BMAN_POOL_FLAG_THRESH)
+               bm_pool_set(bpid, zero_thresholds);
+
+       if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID)
+               bman_release_bpid(bpid);
+       kfree(pool);
+
+       return NULL;
+}
+
+void bman_free_pool(struct bman_pool *pool)
+{
+       if (pool->params.flags & BMAN_POOL_FLAG_THRESH)
+               bm_pool_set(pool->params.bpid, zero_thresholds);
+       if (pool->params.flags & BMAN_POOL_FLAG_DYNAMIC_BPID)
+               bman_release_bpid(pool->params.bpid);
+       kfree(pool);
+}
+
+const struct bman_pool_params *bman_get_params(const struct bman_pool *pool)
+{
+       return &pool->params;
+}
+
+static void update_rcr_ci(struct bman_portal *p, int avail)
+{
+       if (avail)
+               bm_rcr_cce_prefetch(&p->p);
+       else
+               bm_rcr_cce_update(&p->p);
+}
+
+#define BMAN_BUF_MASK 0x0000fffffffffffful
+int bman_release(struct bman_pool *pool, const struct bm_buffer *bufs, u8 num,
+                u32 flags __maybe_unused)
+{
+       struct bman_portal *p;
+       struct bm_rcr_entry *r;
+       u32 i = num - 1;
+       u8 avail;
+
+#ifdef RTE_LIBRTE_DPAA_CHECKING
+       if (!num || (num > 8))
+               return -EINVAL;
+       if (pool->params.flags & BMAN_POOL_FLAG_NO_RELEASE)
+               return -EINVAL;
+#endif
+
+       p = get_affine_portal();
+       avail = bm_rcr_get_avail(&p->p);
+       if (avail < 2)
+               update_rcr_ci(p, avail);
+       r = bm_rcr_start(&p->p);
+       if (unlikely(!r))
+               return -EBUSY;
+
+       /*
+        * we can copy all but the first entry, as this can trigger badness
+        * with the valid-bit
+        */
+       r->bufs[0].opaque =
+               cpu_to_be64(((u64)pool->params.bpid << 48) |
+                           (bufs[0].opaque & BMAN_BUF_MASK));
+       if (i) {
+               for (i = 1; i < num; i++)
+                       r->bufs[i].opaque =
+                               cpu_to_be64(bufs[i].opaque & BMAN_BUF_MASK);
+       }
+
+       bm_rcr_pvb_commit(&p->p, BM_RCR_VERB_CMD_BPID_SINGLE |
+                         (num & BM_RCR_VERB_BUFCOUNT_MASK));
+
+       return 0;
+}
+
+int bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, u8 num,
+                u32 flags __maybe_unused)
+{
+       struct bman_portal *p = get_affine_portal();
+       struct bm_mc_command *mcc;
+       struct bm_mc_result *mcr;
+       int ret, i;
+
+#ifdef RTE_LIBRTE_DPAA_CHECKING
+       if (!num || (num > 8))
+               return -EINVAL;
+       if (pool->params.flags & BMAN_POOL_FLAG_ONLY_RELEASE)
+               return -EINVAL;
+#endif
+
+       mcc = bm_mc_start(&p->p);
+       mcc->acquire.bpid = pool->params.bpid;
+       bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE |
+                       (num & BM_MCC_VERB_ACQUIRE_BUFCOUNT));
+       while (!(mcr = bm_mc_result(&p->p)))
+               cpu_relax();
+       ret = mcr->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT;
+       if (bufs) {
+               for (i = 0; i < num; i++)
+                       bufs[i].opaque =
+                               be64_to_cpu(mcr->acquire.bufs[i].opaque);
+       }
+       if (ret != num)
+               ret = -ENOMEM;
+       return ret;
+}
+
+int bman_query_pools(struct bm_pool_state *state)
+{
+       struct bman_portal *p = get_affine_portal();
+       struct bm_mc_result *mcr;
+
+       bm_mc_start(&p->p);
+       bm_mc_commit(&p->p, BM_MCC_VERB_CMD_QUERY);
+       while (!(mcr = bm_mc_result(&p->p)))
+               cpu_relax();
+       DPAA_ASSERT((mcr->verb & BM_MCR_VERB_CMD_MASK) ==
+                   BM_MCR_VERB_CMD_QUERY);
+       *state = mcr->query;
+       state->as.state.state[0] = be32_to_cpu(state->as.state.state[0]);
+       state->as.state.state[1] = be32_to_cpu(state->as.state.state[1]);
+       state->ds.state.state[0] = be32_to_cpu(state->ds.state.state[0]);
+       state->ds.state.state[1] = be32_to_cpu(state->ds.state.state[1]);
+       return 0;
+}
+
+u32 bman_query_free_buffers(struct bman_pool *pool)
+{
+       return bm_pool_free_buffers(pool->params.bpid);
+}
+
+int bman_update_pool_thresholds(struct bman_pool *pool, const u32 *thresholds)
+{
+       u32 bpid;
+
+       bpid = bman_get_params(pool)->bpid;
+
+       return bm_pool_set(bpid, thresholds);
+}
+
+int bman_shutdown_pool(u32 bpid)
+{
+       struct bman_portal *p = get_affine_portal();
+       return bm_shutdown_pool(&p->p, bpid);
+}
diff --git a/drivers/bus/dpaa/base/qbman/bman.h 
b/drivers/bus/dpaa/base/qbman/bman.h
new file mode 100644
index 0000000..2af30e9
--- /dev/null
+++ b/drivers/bus/dpaa/base/qbman/bman.h
@@ -0,0 +1,550 @@
+/*-
+ * This file is provided under a dual BSD/GPLv2 license. When using or
+ * redistributing this file, you may do so under either license.
+ *
+ *   BSD LICENSE
+ *
+ * Copyright 2010-2016 Freescale Semiconductor Inc.
+ * Copyright 2017 NXP.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of the above-listed copyright holders nor the
+ * names of any contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ *   GPL LICENSE SUMMARY
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __BMAN_H
+#define __BMAN_H
+
+#include "bman_priv.h"
+
+/* Cache-inhibited register offsets */
+#define BM_REG_RCR_PI_CINH     0x3000
+#define BM_REG_RCR_CI_CINH     0x3100
+#define BM_REG_RCR_ITR         0x3200
+#define BM_REG_CFG             0x3300
+#define BM_REG_SCN(n)          (0x3400 + ((n) << 6))
+#define BM_REG_ISR             0x3e00
+#define BM_REG_IIR              0x3ec0
+
+/* Cache-enabled register offsets */
+#define BM_CL_CR               0x0000
+#define BM_CL_RR0              0x0100
+#define BM_CL_RR1              0x0140
+#define BM_CL_RCR              0x1000
+#define BM_CL_RCR_PI_CENA      0x3000
+#define BM_CL_RCR_CI_CENA      0x3100
+
+/* BTW, the drivers (and h/w programming model) already obtain the required
+ * synchronisation for portal accesses via lwsync(), hwsync(), and
+ * data-dependencies. Use of barrier()s or other order-preserving primitives
+ * simply degrade performance. Hence the use of the __raw_*() interfaces, which
+ * simply ensure that the compiler treats the portal registers as volatile (ie.
+ * non-coherent).
+ */
+
+/* Cache-inhibited register access. */
+#define __bm_in(bm, o)         be32_to_cpu(__raw_readl((bm)->ci + (o)))
+#define __bm_out(bm, o, val)    __raw_writel(cpu_to_be32(val), \
+                                            (bm)->ci + (o))
+#define bm_in(reg)             __bm_in(&portal->addr, BM_REG_##reg)
+#define bm_out(reg, val)       __bm_out(&portal->addr, BM_REG_##reg, val)
+
+/* Cache-enabled (index) register access */
+#define __bm_cl_touch_ro(bm, o) dcbt_ro((bm)->ce + (o))
+#define __bm_cl_touch_rw(bm, o) dcbt_rw((bm)->ce + (o))
+#define __bm_cl_in(bm, o)      be32_to_cpu(__raw_readl((bm)->ce + (o)))
+#define __bm_cl_out(bm, o, val) \
+       do { \
+               u32 *__tmpclout = (bm)->ce + (o); \
+               __raw_writel(cpu_to_be32(val), __tmpclout); \
+               dcbf(__tmpclout); \
+       } while (0)
+#define __bm_cl_invalidate(bm, o) dccivac((bm)->ce + (o))
+#define bm_cl_touch_ro(reg) __bm_cl_touch_ro(&portal->addr, BM_CL_##reg##_CENA)
+#define bm_cl_touch_rw(reg) __bm_cl_touch_rw(&portal->addr, BM_CL_##reg##_CENA)
+#define bm_cl_in(reg)      __bm_cl_in(&portal->addr, BM_CL_##reg##_CENA)
+#define bm_cl_out(reg, val) __bm_cl_out(&portal->addr, BM_CL_##reg##_CENA, val)
+#define bm_cl_invalidate(reg)\
+       __bm_cl_invalidate(&portal->addr, BM_CL_##reg##_CENA)
+
+/* Cyclic helper for rings. FIXME: once we are able to do fine-grain perf
+ * analysis, look at using the "extra" bit in the ring index registers to avoid
+ * cyclic issues.
+ */
+static inline u8 bm_cyc_diff(u8 ringsize, u8 first, u8 last)
+{
+       /* 'first' is included, 'last' is excluded */
+       if (first <= last)
+               return last - first;
+       return ringsize + last - first;
+}
+
+/* Portal modes.
+ *   Enum types;
+ *     pmode == production mode
+ *     cmode == consumption mode,
+ *   Enum values use 3 letter codes. First letter matches the portal mode,
+ *   remaining two letters indicate;
+ *     ci == cache-inhibited portal register
+ *     ce == cache-enabled portal register
+ *     vb == in-band valid-bit (cache-enabled)
+ */
+enum bm_rcr_pmode {            /* matches BCSP_CFG::RPM */
+       bm_rcr_pci = 0,         /* PI index, cache-inhibited */
+       bm_rcr_pce = 1,         /* PI index, cache-enabled */
+       bm_rcr_pvb = 2          /* valid-bit */
+};
+
+enum bm_rcr_cmode {            /* s/w-only */
+       bm_rcr_cci,             /* CI index, cache-inhibited */
+       bm_rcr_cce              /* CI index, cache-enabled */
+};
+
+/* --- Portal structures --- */
+
+#define BM_RCR_SIZE            8
+
+struct bm_rcr {
+       struct bm_rcr_entry *ring, *cursor;
+       u8 ci, available, ithresh, vbit;
+#ifdef RTE_LIBRTE_DPAA_CHECKING
+       u32 busy;
+       enum bm_rcr_pmode pmode;
+       enum bm_rcr_cmode cmode;
+#endif
+};
+
+struct bm_mc {
+       struct bm_mc_command *cr;
+       struct bm_mc_result *rr;
+       u8 rridx, vbit;
+#ifdef RTE_LIBRTE_DPAA_CHECKING
+       enum {
+               /* Can only be _mc_start()ed */
+               mc_idle,
+               /* Can only be _mc_commit()ed or _mc_abort()ed */
+               mc_user,
+               /* Can only be _mc_retry()ed */
+               mc_hw
+       } state;
+#endif
+};
+
+struct bm_addr {
+       void __iomem *ce;       /* cache-enabled */
+       void __iomem *ci;       /* cache-inhibited */
+};
+
+struct bm_portal {
+       struct bm_addr addr;
+       struct bm_rcr rcr;
+       struct bm_mc mc;
+       struct bm_portal_config config;
+} ____cacheline_aligned;
+
+/* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
+#define RCR_CARRYCLEAR(p) \
+       (void *)((unsigned long)(p) & (~(unsigned long)(BM_RCR_SIZE << 6)))
+
+/* Bit-wise logic to convert a ring pointer to a ring index */
+static inline u8 RCR_PTR2IDX(struct bm_rcr_entry *e)
+{
+       return ((uintptr_t)e >> 6) & (BM_RCR_SIZE - 1);
+}
+
+/* Increment the 'cursor' ring pointer, taking 'vbit' into account */
+static inline void RCR_INC(struct bm_rcr *rcr)
+{
+       /* NB: this is odd-looking, but experiments show that it generates
+        * fast code with essentially no branching overheads. We increment to
+        * the next RCR pointer and handle overflow and 'vbit'.
+        */
+       struct bm_rcr_entry *partial = rcr->cursor + 1;
+
+       rcr->cursor = RCR_CARRYCLEAR(partial);
+       if (partial != rcr->cursor)
+               rcr->vbit ^= BM_RCR_VERB_VBIT;
+}
+
+static inline int bm_rcr_init(struct bm_portal *portal, enum bm_rcr_pmode 
pmode,
+                             __maybe_unused enum bm_rcr_cmode cmode)
+{
+       /* This use of 'register', as well as all other occurrences, is because
+        * it has been observed to generate much faster code with gcc than is
+        * otherwise the case.
+        */
+       register struct bm_rcr *rcr = &portal->rcr;
+       u32 cfg;
+       u8 pi;
+
+       rcr->ring = portal->addr.ce + BM_CL_RCR;
+       rcr->ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1);
+
+       pi = bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1);
+       rcr->cursor = rcr->ring + pi;
+       rcr->vbit = (bm_in(RCR_PI_CINH) & BM_RCR_SIZE) ?  BM_RCR_VERB_VBIT : 0;
+       rcr->available = BM_RCR_SIZE - 1
+               - bm_cyc_diff(BM_RCR_SIZE, rcr->ci, pi);
+       rcr->ithresh = bm_in(RCR_ITR);
+#ifdef RTE_LIBRTE_DPAA_CHECKING
+       rcr->busy = 0;
+       rcr->pmode = pmode;
+       rcr->cmode = cmode;
+#endif
+       cfg = (bm_in(CFG) & 0xffffffe0) | (pmode & 0x3); /* BCSP_CFG::RPM */
+       bm_out(CFG, cfg);
+       return 0;
+}
+
+static inline void bm_rcr_finish(struct bm_portal *portal)
+{
+       register struct bm_rcr *rcr = &portal->rcr;
+       u8 pi = bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1);
+       u8 ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1);
+
+       DPAA_ASSERT(!rcr->busy);
+       if (pi != RCR_PTR2IDX(rcr->cursor))
+               pr_crit("loosing uncommitted RCR entries\n");
+       if (ci != rcr->ci)
+               pr_crit("missing existing RCR completions\n");
+       if (rcr->ci != RCR_PTR2IDX(rcr->cursor))
+               pr_crit("RCR destroyed unquiesced\n");
+}
+
+static inline struct bm_rcr_entry *bm_rcr_start(struct bm_portal *portal)
+{
+       register struct bm_rcr *rcr = &portal->rcr;
+
+       DPAA_ASSERT(!rcr->busy);
+       if (!rcr->available)
+               return NULL;
+#ifdef RTE_LIBRTE_DPAA_CHECKING
+       rcr->busy = 1;
+#endif
+       dcbz_64(rcr->cursor);
+       return rcr->cursor;
+}
+
+static inline void bm_rcr_abort(struct bm_portal *portal)
+{
+       __maybe_unused register struct bm_rcr *rcr = &portal->rcr;
+
+       DPAA_ASSERT(rcr->busy);
+#ifdef RTE_LIBRTE_DPAA_CHECKING
+       rcr->busy = 0;
+#endif
+}
+
+static inline struct bm_rcr_entry *bm_rcr_pend_and_next(
+                                       struct bm_portal *portal, u8 myverb)
+{
+       register struct bm_rcr *rcr = &portal->rcr;
+
+       DPAA_ASSERT(rcr->busy);
+       DPAA_ASSERT(rcr->pmode != bm_rcr_pvb);
+       if (rcr->available == 1)
+               return NULL;
+       rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit;
+       dcbf_64(rcr->cursor);
+       RCR_INC(rcr);
+       rcr->available--;
+       dcbz_64(rcr->cursor);
+       return rcr->cursor;
+}
+
+static inline void bm_rcr_pci_commit(struct bm_portal *portal, u8 myverb)
+{
+       register struct bm_rcr *rcr = &portal->rcr;
+
+       DPAA_ASSERT(rcr->busy);
+       DPAA_ASSERT(rcr->pmode == bm_rcr_pci);
+       rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit;
+       RCR_INC(rcr);
+       rcr->available--;
+       hwsync();
+       bm_out(RCR_PI_CINH, RCR_PTR2IDX(rcr->cursor));
+#ifdef RTE_LIBRTE_DPAA_CHECKING
+       rcr->busy = 0;
+#endif
+}
+
+static inline void bm_rcr_pce_prefetch(struct bm_portal *portal)
+{
+       __maybe_unused register struct bm_rcr *rcr = &portal->rcr;
+
+       DPAA_ASSERT(rcr->pmode == bm_rcr_pce);
+       bm_cl_invalidate(RCR_PI);
+       bm_cl_touch_rw(RCR_PI);
+}
+
+static inline void bm_rcr_pce_commit(struct bm_portal *portal, u8 myverb)
+{
+       register struct bm_rcr *rcr = &portal->rcr;
+
+       DPAA_ASSERT(rcr->busy);
+       DPAA_ASSERT(rcr->pmode == bm_rcr_pce);
+       rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit;
+       RCR_INC(rcr);
+       rcr->available--;
+       lwsync();
+       bm_cl_out(RCR_PI, RCR_PTR2IDX(rcr->cursor));
+#ifdef RTE_LIBRTE_DPAA_CHECKING
+       rcr->busy = 0;
+#endif
+}
+
+static inline void bm_rcr_pvb_commit(struct bm_portal *portal, u8 myverb)
+{
+       register struct bm_rcr *rcr = &portal->rcr;
+       struct bm_rcr_entry *rcursor;
+
+       DPAA_ASSERT(rcr->busy);
+       DPAA_ASSERT(rcr->pmode == bm_rcr_pvb);
+       lwsync();
+       rcursor = rcr->cursor;
+       rcursor->__dont_write_directly__verb = myverb | rcr->vbit;
+       dcbf_64(rcursor);
+       RCR_INC(rcr);
+       rcr->available--;
+#ifdef RTE_LIBRTE_DPAA_CHECKING
+       rcr->busy = 0;
+#endif
+}
+
+static inline u8 bm_rcr_cci_update(struct bm_portal *portal)
+{
+       register struct bm_rcr *rcr = &portal->rcr;
+       u8 diff, old_ci = rcr->ci;
+
+       DPAA_ASSERT(rcr->cmode == bm_rcr_cci);
+       rcr->ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1);
+       diff = bm_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
+       rcr->available += diff;
+       return diff;
+}
+
+static inline void bm_rcr_cce_prefetch(struct bm_portal *portal)
+{
+       __maybe_unused register struct bm_rcr *rcr = &portal->rcr;
+
+       DPAA_ASSERT(rcr->cmode == bm_rcr_cce);
+       bm_cl_touch_ro(RCR_CI);
+}
+
+static inline u8 bm_rcr_cce_update(struct bm_portal *portal)
+{
+       register struct bm_rcr *rcr = &portal->rcr;
+       u8 diff, old_ci = rcr->ci;
+
+       DPAA_ASSERT(rcr->cmode == bm_rcr_cce);
+       rcr->ci = bm_cl_in(RCR_CI) & (BM_RCR_SIZE - 1);
+       bm_cl_invalidate(RCR_CI);
+       diff = bm_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
+       rcr->available += diff;
+       return diff;
+}
+
+static inline u8 bm_rcr_get_ithresh(struct bm_portal *portal)
+{
+       register struct bm_rcr *rcr = &portal->rcr;
+
+       return rcr->ithresh;
+}
+
+static inline void bm_rcr_set_ithresh(struct bm_portal *portal, u8 ithresh)
+{
+       register struct bm_rcr *rcr = &portal->rcr;
+
+       rcr->ithresh = ithresh;
+       bm_out(RCR_ITR, ithresh);
+}
+
+static inline u8 bm_rcr_get_avail(struct bm_portal *portal)
+{
+       register struct bm_rcr *rcr = &portal->rcr;
+
+       return rcr->available;
+}
+
+static inline u8 bm_rcr_get_fill(struct bm_portal *portal)
+{
+       register struct bm_rcr *rcr = &portal->rcr;
+
+       return BM_RCR_SIZE - 1 - rcr->available;
+}
+
+/* --- Management command API --- */
+
+static inline int bm_mc_init(struct bm_portal *portal)
+{
+       register struct bm_mc *mc = &portal->mc;
+
+       mc->cr = portal->addr.ce + BM_CL_CR;
+       mc->rr = portal->addr.ce + BM_CL_RR0;
+       mc->rridx = (__raw_readb(&mc->cr->__dont_write_directly__verb) &
+                       BM_MCC_VERB_VBIT) ?  0 : 1;
+       mc->vbit = mc->rridx ? BM_MCC_VERB_VBIT : 0;
+#ifdef RTE_LIBRTE_DPAA_CHECKING
+       mc->state = mc_idle;
+#endif
+       return 0;
+}
+
+static inline void bm_mc_finish(struct bm_portal *portal)
+{
+       __maybe_unused register struct bm_mc *mc = &portal->mc;
+
+       DPAA_ASSERT(mc->state == mc_idle);
+#ifdef RTE_LIBRTE_DPAA_CHECKING
+       if (mc->state != mc_idle)
+               pr_crit("Losing incomplete MC command\n");
+#endif
+}
+
+static inline struct bm_mc_command *bm_mc_start(struct bm_portal *portal)
+{
+       register struct bm_mc *mc = &portal->mc;
+
+       DPAA_ASSERT(mc->state == mc_idle);
+#ifdef RTE_LIBRTE_DPAA_CHECKING
+       mc->state = mc_user;
+#endif
+       dcbz_64(mc->cr);
+       return mc->cr;
+}
+
+static inline void bm_mc_abort(struct bm_portal *portal)
+{
+       __maybe_unused register struct bm_mc *mc = &portal->mc;
+
+       DPAA_ASSERT(mc->state == mc_user);
+#ifdef RTE_LIBRTE_DPAA_CHECKING
+       mc->state = mc_idle;
+#endif
+}
+
+static inline void bm_mc_commit(struct bm_portal *portal, u8 myverb)
+{
+       register struct bm_mc *mc = &portal->mc;
+       struct bm_mc_result *rr = mc->rr + mc->rridx;
+
+       DPAA_ASSERT(mc->state == mc_user);
+       lwsync();
+       mc->cr->__dont_write_directly__verb = myverb | mc->vbit;
+       dcbf(mc->cr);
+       dcbit_ro(rr);
+#ifdef RTE_LIBRTE_DPAA_CHECKING
+       mc->state = mc_hw;
+#endif
+}
+
+static inline struct bm_mc_result *bm_mc_result(struct bm_portal *portal)
+{
+       register struct bm_mc *mc = &portal->mc;
+       struct bm_mc_result *rr = mc->rr + mc->rridx;
+
+       DPAA_ASSERT(mc->state == mc_hw);
+       /* The inactive response register's verb byte always returns zero until
+        * its command is submitted and completed. This includes the valid-bit,
+        * in case you were wondering.
+        */
+       if (!__raw_readb(&rr->verb)) {
+               dcbit_ro(rr);
+               return NULL;
+       }
+       mc->rridx ^= 1;
+       mc->vbit ^= BM_MCC_VERB_VBIT;
+#ifdef RTE_LIBRTE_DPAA_CHECKING
+       mc->state = mc_idle;
+#endif
+       return rr;
+}
+
+#define SCN_REG(bpid) BM_REG_SCN((bpid) / 32)
+#define SCN_BIT(bpid) (0x80000000 >> (bpid & 31))
+static inline void bm_isr_bscn_mask(struct bm_portal *portal, u8 bpid,
+                                   int enable)
+{
+       u32 val;
+
+       DPAA_ASSERT(bpid < bman_pool_max);
+       /* REG_SCN for bpid=0..31, REG_SCN+4 for bpid=32..63 */
+       val = __bm_in(&portal->addr, SCN_REG(bpid));
+       if (enable)
+               val |= SCN_BIT(bpid);
+       else
+               val &= ~SCN_BIT(bpid);
+       __bm_out(&portal->addr, SCN_REG(bpid), val);
+}
+
+static inline u32 __bm_isr_read(struct bm_portal *portal, enum bm_isr_reg n)
+{
+#if defined(RTE_ARCH_ARM64)
+       return __bm_in(&portal->addr, BM_REG_ISR + (n << 6));
+#else
+       return __bm_in(&portal->addr, BM_REG_ISR + (n << 2));
+#endif
+}
+
+static inline void __bm_isr_write(struct bm_portal *portal, enum bm_isr_reg n,
+                                 u32 val)
+{
+#if defined(RTE_ARCH_ARM64)
+       __bm_out(&portal->addr, BM_REG_ISR + (n << 6), val);
+#else
+       __bm_out(&portal->addr, BM_REG_ISR + (n << 2), val);
+#endif
+}
+
+/* Buffer Pool Cleanup */
+static inline int bm_shutdown_pool(struct bm_portal *p, u32 bpid)
+{
+       struct bm_mc_command *bm_cmd;
+       struct bm_mc_result *bm_res;
+
+       int aq_count = 0;
+       bool stop = false;
+
+       while (!stop) {
+               /* Acquire buffers until empty */
+               bm_cmd = bm_mc_start(p);
+               bm_cmd->acquire.bpid = bpid;
+               bm_mc_commit(p, BM_MCC_VERB_CMD_ACQUIRE |  1);
+               while (!(bm_res = bm_mc_result(p)))
+                       cpu_relax();
+               if (!(bm_res->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT)) {
+                       /* Pool is empty */
+                       stop = true;
+               } else
+                       ++aq_count;
+       };
+       return 0;
+}
+
+#endif /* __BMAN_H */
diff --git a/drivers/bus/dpaa/base/qbman/bman_driver.c 
b/drivers/bus/dpaa/base/qbman/bman_driver.c
index fb3c50e..28f2cf2 100644
--- a/drivers/bus/dpaa/base/qbman/bman_driver.c
+++ b/drivers/bus/dpaa/base/qbman/bman_driver.c
@@ -65,6 +65,7 @@ static __thread struct dpaa_ioctl_portal_map map = {
 static int fsl_bman_portal_init(uint32_t idx, int is_shared)
 {
        cpu_set_t cpuset;
+       struct bman_portal *portal;
        int loop, ret;
        struct dpaa_ioctl_irq_map irq_map;
 
@@ -111,6 +112,14 @@ static int fsl_bman_portal_init(uint32_t idx, int 
is_shared)
        /* Use the IRQ FD as a unique IRQ number */
        pcfg.irq = fd;
 
+       portal = bman_create_affine_portal(&pcfg);
+       if (!portal) {
+               pr_err("Bman portal initialisation failed (%d)",
+                      pcfg.cpu);
+               process_portal_unmap(&map.addr);
+               return -EBUSY;
+       }
+
        /* Set the IRQ number */
        irq_map.type = dpaa_portal_bman;
        irq_map.portal_cinh = map.addr.cinh;
@@ -120,10 +129,13 @@ static int fsl_bman_portal_init(uint32_t idx, int 
is_shared)
 
 static int fsl_bman_portal_finish(void)
 {
+       __maybe_unused const struct bm_portal_config *cfg;
        int ret;
 
        process_portal_irq_unmap(fd);
 
+       cfg = bman_destroy_affine_portal();
+       BUG_ON(cfg != &pcfg);
        ret = process_portal_unmap(&map.addr);
        if (ret)
                error(0, ret, "process_portal_unmap()");
diff --git a/drivers/bus/dpaa/base/qbman/dpaa_alloc.c 
b/drivers/bus/dpaa/base/qbman/dpaa_alloc.c
index 690576a..35dba7f 100644
--- a/drivers/bus/dpaa/base/qbman/dpaa_alloc.c
+++ b/drivers/bus/dpaa/base/qbman/dpaa_alloc.c
@@ -41,6 +41,22 @@
 #include "dpaa_sys.h"
 #include <process.h>
 #include <fsl_qman.h>
+#include <fsl_bman.h>
+
+int bman_alloc_bpid_range(u32 *result, u32 count, u32 align, int partial)
+{
+       return process_alloc(dpaa_id_bpid, result, count, align, partial);
+}
+
+void bman_release_bpid_range(u32 bpid, u32 count)
+{
+       process_release(dpaa_id_bpid, bpid, count);
+}
+
+int bman_reserve_bpid_range(u32 bpid, u32 count)
+{
+       return process_reserve(dpaa_id_bpid, bpid, count);
+}
 
 int qman_alloc_fqid_range(u32 *result, u32 count, u32 align, int partial)
 {
-- 
2.7.4

Reply via email to