source: rtems-libbsd/linux/drivers/soc/fsl/qbman/bman_api.c @ 28ee86a

#include <machine/rtems-bsd-kernel-space.h>

#include <rtems/bsd/local/opt_dpaa.h>

/* Copyright 2008 - 2015 Freescale Semiconductor, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of Freescale Semiconductor nor the
 *       names of its contributors may be used to endorse or promote products
 *       derived from this software without specific prior written permission.
 *
 * ALTERNATIVELY, this software may be distributed under the terms of the
 * GNU General Public License ("GPL") as published by the Free Software
 * Foundation, either version 2 of that License or (at your option) any
 * later version.
 *
 * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "bman.h"

/* Compilation constants */
#define RCR_THRESH      2       /* reread h/w CI when running out of space */
#define IRQNAME         "BMan portal %d"
#define MAX_IRQNAME     16      /* big enough for "BMan portal %d" */
#define FSL_DPA_PORTAL_SHARE 1  /* Allow portals to be shared */

struct bman_portal {
        struct bm_portal p;
        /* 2-element array. pools[0] is mask, pools[1] is snapshot. */
        struct bman_depletion *pools;
        int thresh_set;
        unsigned long irq_sources;
        u32 slowpoll;   /* only used when interrupts are off */
#ifdef FSL_DPA_CAN_WAIT_SYNC
        struct bman_pool *rcri_owned; /* only 1 release WAIT_SYNC at a time */
#endif
#ifdef FSL_DPA_PORTAL_SHARE
        raw_spinlock_t sharing_lock; /* only used if is_shared */
#ifndef __rtems__
        int is_shared;
        struct bman_portal *sharing_redirect;
#endif /* __rtems__ */
#endif
        /* When the cpu-affine portal is activated, this is non-NULL */
        const struct bm_portal_config *config;
        /* 64-entry hash-table of pool objects that are tracking depletion
         * entry/exit (ie. BMAN_POOL_FLAG_DEPLETION). This isn't fast-path, so
         * we're not fussy about cache-misses and so forth - whereas the above
         * members should all fit in one cacheline.
         * BTW, with 64 entries in the hash table and 64 buffer pools to track,
         * you'll never guess the hash-function ... */
        struct bman_pool *cb[64];
        char irqname[MAX_IRQNAME];
        /* Track if the portal was alloced by the driver */
        u8 alloced;
};


#ifdef FSL_DPA_PORTAL_SHARE
/* For an explanation of the locking, redirection, or affine-portal logic,
 * please consult the QMan driver for details. This is the same, only simpler
 * (no fiddly QMan-specific bits.) */
#ifndef __rtems__
#define PORTAL_IRQ_LOCK(p, irqflags) \
        do { \
                if ((p)->is_shared) \
                        raw_spin_lock_irqsave(&(p)->sharing_lock, irqflags); \
                else \
                        local_irq_save(irqflags); \
        } while (0)
#define PORTAL_IRQ_UNLOCK(p, irqflags) \
        do { \
                if ((p)->is_shared) \
                        raw_spin_unlock_irqrestore(&(p)->sharing_lock, \
                                                   irqflags); \
                else \
                        local_irq_restore(irqflags); \
        } while (0)
#else /* __rtems__ */
#define PORTAL_IRQ_LOCK(p, irqflags) \
    raw_spin_lock_irqsave(&(p)->sharing_lock, irqflags)
#define PORTAL_IRQ_UNLOCK(p, irqflags) \
    raw_spin_unlock_irqrestore(&(p)->sharing_lock, irqflags)
#endif /* __rtems__ */
#else
#define PORTAL_IRQ_LOCK(p, irqflags) local_irq_save(irqflags)
#define PORTAL_IRQ_UNLOCK(p, irqflags) local_irq_restore(irqflags)
#endif

#ifndef __rtems__
static cpumask_t affine_mask;
static DEFINE_SPINLOCK(affine_mask_lock);
#endif /* __rtems__ */
static DEFINE_PER_CPU(struct bman_portal, bman_affine_portal);
static inline struct bman_portal *get_raw_affine_portal(void)
{
        return &get_cpu_var(bman_affine_portal);
}
#ifdef FSL_DPA_PORTAL_SHARE
static inline struct bman_portal *get_affine_portal(void)
{
        struct bman_portal *p = get_raw_affine_portal();

#ifndef __rtems__
        if (p->sharing_redirect)
                return p->sharing_redirect;
#endif /* __rtems__ */
        return p;
}
#else
#define get_affine_portal() get_raw_affine_portal()
#endif
static inline void put_affine_portal(void)
{
        put_cpu_var(bman_affine_portal);
}
static inline struct bman_portal *get_poll_portal(void)
{
        return this_cpu_ptr(&bman_affine_portal);
}
#define put_poll_portal()

/* GOTCHA: this object type refers to a pool, it isn't *the* pool. There may be
 * more than one such object per BMan buffer pool, eg. if different users of the
 * pool are operating via different portals. */
struct bman_pool {
        struct bman_pool_params params;
        /* Used for hash-table admin when using depletion notifications. */
        struct bman_portal *portal;
        struct bman_pool *next;
        /* stockpile state - NULL unless BMAN_POOL_FLAG_STOCKPILE is set */
        struct bm_buffer *sp;
        unsigned int sp_fill;
#ifdef CONFIG_FSL_DPA_CHECKING
        atomic_t in_use;
#endif
};

/* (De)Registration of depletion notification callbacks */
static void depletion_link(struct bman_portal *portal, struct bman_pool *pool)
{
        __maybe_unused unsigned long irqflags;

        pool->portal = portal;
        PORTAL_IRQ_LOCK(portal, irqflags);
        pool->next = portal->cb[pool->params.bpid];
        portal->cb[pool->params.bpid] = pool;
        if (!pool->next)
                /* First object for that bpid on this portal, enable the BSCN
                 * mask bit. */
                bm_isr_bscn_mask(&portal->p, pool->params.bpid, 1);
        PORTAL_IRQ_UNLOCK(portal, irqflags);
}
static void depletion_unlink(struct bman_pool *pool)
{
        struct bman_pool *it, *last = NULL;
        struct bman_pool **base = &pool->portal->cb[pool->params.bpid];
        __maybe_unused unsigned long irqflags;

        PORTAL_IRQ_LOCK(pool->portal, irqflags);
        it = *base;     /* <-- gotcha, don't do this prior to the irq_save */
        while (it != pool) {
                last = it;
                it = it->next;
        }
        if (!last)
                *base = pool->next;
        else
                last->next = pool->next;
        if (!last && !pool->next) {
                /* Last object for that bpid on this portal, disable the BSCN
                 * mask bit. */
                bm_isr_bscn_mask(&pool->portal->p, pool->params.bpid, 0);
                /* And "forget" that we last saw this pool as depleted */
                bman_depletion_unset(&pool->portal->pools[1],
                                        pool->params.bpid);
        }
        PORTAL_IRQ_UNLOCK(pool->portal, irqflags);
}

/* In the case that the application's core loop calls qman_poll() and
 * bman_poll(), we ought to balance how often we incur the overheads of the
 * slow-path poll. We'll use two decrementer sources. The idle decrementer
 * constant is used when the last slow-poll detected no work to do, and the busy
 * decrementer constant when the last slow-poll had work to do. */
#define SLOW_POLL_IDLE 1000
#define SLOW_POLL_BUSY 10
static u32 __poll_portal_slow(struct bman_portal *p, u32 is);

/* Portal interrupt handler */
static irqreturn_t portal_isr(__always_unused int irq, void *ptr)
{
        struct bman_portal *p = ptr;
        u32 clear = p->irq_sources;
        u32 is = bm_isr_status_read(&p->p) & p->irq_sources;

        clear |= __poll_portal_slow(p, is);
        bm_isr_status_clear(&p->p, clear);
        return IRQ_HANDLED;
}


struct bman_portal *bman_create_portal(
                                       struct bman_portal *portal,
                                       const struct bm_portal_config *config)
{
        struct bm_portal *__p;
        const struct bman_depletion *pools = &config->public_cfg.mask;
        int ret;
        u8 bpid = 0;

        if (!portal) {
                portal = kmalloc(sizeof(*portal), GFP_KERNEL);
                if (!portal)
                        return portal;
                portal->alloced = 1;
        } else
                portal->alloced = 0;

        __p = &portal->p;

        /* prep the low-level portal struct with the mapped addresses from the
         * config, everything that follows depends on it and "config" is more
         * for (de)reference... */
        __p->addr.addr_ce = config->addr_virt[DPA_PORTAL_CE];
        __p->addr.addr_ci = config->addr_virt[DPA_PORTAL_CI];
        if (bm_rcr_init(__p, bm_rcr_pvb, bm_rcr_cce)) {
                pr_err("RCR initialisation failed\n");
                goto fail_rcr;
        }
        if (bm_mc_init(__p)) {
                pr_err("MC initialisation failed\n");
                goto fail_mc;
        }
        if (bm_isr_init(__p)) {
                pr_err("ISR initialisation failed\n");
                goto fail_isr;
        }
        portal->pools = kmalloc(2 * sizeof(*pools), GFP_KERNEL);
        if (!portal->pools)
                goto fail_pools;
        portal->pools[0] = *pools;
        bman_depletion_init(portal->pools + 1);
        while (bpid < bman_pool_max) {
                /* Default to all BPIDs disabled, we enable as required at
                 * run-time. */
                bm_isr_bscn_mask(__p, bpid, 0);
                bpid++;
        }
        portal->slowpoll = 0;
#ifdef FSL_DPA_CAN_WAIT_SYNC
        portal->rcri_owned = NULL;
#endif
#ifdef FSL_DPA_PORTAL_SHARE
        raw_spin_lock_init(&portal->sharing_lock);
#ifndef __rtems__
        portal->is_shared = config->public_cfg.is_shared;
        portal->sharing_redirect = NULL;
#endif /* __rtems__ */
#endif
        memset(&portal->cb, 0, sizeof(portal->cb));
        /* Write-to-clear any stale interrupt status bits */
        bm_isr_disable_write(__p, 0xffffffff);
        portal->irq_sources = 0;
        bm_isr_enable_write(__p, portal->irq_sources);
        bm_isr_status_clear(__p, 0xffffffff);
        snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, config->public_cfg.cpu);
        if (request_irq(config->public_cfg.irq, portal_isr, 0, portal->irqname,
                        portal)) {
                pr_err("request_irq() failed\n");
                goto fail_irq;
        }
#ifndef __rtems__
        if ((config->public_cfg.cpu != -1) &&
                        irq_can_set_affinity(config->public_cfg.irq) &&
                        irq_set_affinity(config->public_cfg.irq,
                                cpumask_of(config->public_cfg.cpu))) {
                pr_err("irq_set_affinity() failed\n");
                goto fail_affinity;
        }
#endif /* __rtems__ */

        /* Need RCR to be empty before continuing */
        ret = bm_rcr_get_fill(__p);
        if (ret) {
                pr_err("RCR unclean\n");
                goto fail_rcr_empty;
        }
        /* Success */
        portal->config = config;

        bm_isr_disable_write(__p, 0);
        bm_isr_uninhibit(__p);
        return portal;
fail_rcr_empty:
#ifndef __rtems__
fail_affinity:
#endif /* __rtems__ */
        free_irq(config->public_cfg.irq, portal);
fail_irq:
        kfree(portal->pools);
fail_pools:
        bm_isr_finish(__p);
fail_isr:
        bm_mc_finish(__p);
fail_mc:
        bm_rcr_finish(__p);
fail_rcr:
        if (portal->alloced)
                kfree(portal);
        return NULL;
}

struct bman_portal *bman_create_affine_portal(
                        const struct bm_portal_config *config)
{
        struct bman_portal *portal;

        portal = &per_cpu(bman_affine_portal, config->public_cfg.cpu);
        portal = bman_create_portal(portal, config);
#ifndef __rtems__
        if (portal) {
                spin_lock(&affine_mask_lock);
                cpumask_set_cpu(config->public_cfg.cpu, &affine_mask);
                spin_unlock(&affine_mask_lock);
        }
#endif /* __rtems__ */
        return portal;
}


#ifndef __rtems__
struct bman_portal *bman_create_affine_slave(struct bman_portal *redirect,
                                                                int cpu)
{
#ifdef FSL_DPA_PORTAL_SHARE
        struct bman_portal *p = &per_cpu(bman_affine_portal, cpu);

        BUG_ON(p->config);
        BUG_ON(p->is_shared);
        BUG_ON(!redirect->config->public_cfg.is_shared);
        p->irq_sources = 0;
        p->sharing_redirect = redirect;
        put_affine_portal();
        return p;
#else
        BUG();
        return NULL;
#endif
}
#endif /* __rtems__ */

void bman_destroy_portal(struct bman_portal *bm)
{
        const struct bm_portal_config *pcfg = bm->config;

        bm_rcr_cce_update(&bm->p);
        bm_rcr_cce_update(&bm->p);

        free_irq(pcfg->public_cfg.irq, bm);

        kfree(bm->pools);
        bm_isr_finish(&bm->p);
        bm_mc_finish(&bm->p);
        bm_rcr_finish(&bm->p);
        bm->config = NULL;
        if (bm->alloced)
                kfree(bm);
}

const struct bm_portal_config *bman_destroy_affine_portal(void)
{
        struct bman_portal *bm = get_raw_affine_portal();
        const struct bm_portal_config *pcfg;

#ifdef FSL_DPA_PORTAL_SHARE
#ifndef __rtems__
        if (bm->sharing_redirect) {
                bm->sharing_redirect = NULL;
                put_affine_portal();
                return NULL;
        }
        bm->is_shared = 0;
#endif /* __rtems__ */
#endif
        pcfg = bm->config;
        bman_destroy_portal(bm);
#ifndef __rtems__
        spin_lock(&affine_mask_lock);
        cpumask_clear_cpu(pcfg->public_cfg.cpu, &affine_mask);
        spin_unlock(&affine_mask_lock);
#endif /* __rtems__ */
        put_affine_portal();
        return pcfg;
}

/* When release logic waits on available RCR space, we need a global waitqueue
 * in the case of "affine" use (as the waits wake on different cpus which means
 * different portals - so we can't wait on any per-portal waitqueue). */
static DECLARE_WAIT_QUEUE_HEAD(affine_queue);

static u32 __poll_portal_slow(struct bman_portal *p, u32 is)
{
        struct bman_depletion tmp;
        u32 ret = is;

        /* There is a gotcha to be aware of. If we do the query before clearing
         * the status register, we may miss state changes that occur between the
         * two. If we write to clear the status register before the query, the
         * cache-enabled query command may overtake the status register write
         * unless we use a heavyweight sync (which we don't want). Instead, we
         * write-to-clear the status register then *read it back* before doing
         * the query, hence the odd while loop with the 'is' accumulation. */
        if (is & BM_PIRQ_BSCN) {
                struct bm_mc_result *mcr;
                __maybe_unused unsigned long irqflags;
                unsigned int i, j;
                u32 __is;

                bm_isr_status_clear(&p->p, BM_PIRQ_BSCN);
                while ((__is = bm_isr_status_read(&p->p)) & BM_PIRQ_BSCN) {
                        is |= __is;
                        bm_isr_status_clear(&p->p, BM_PIRQ_BSCN);
                }
                is &= ~BM_PIRQ_BSCN;
                PORTAL_IRQ_LOCK(p, irqflags);
                bm_mc_start(&p->p);
                bm_mc_commit(&p->p, BM_MCC_VERB_CMD_QUERY);
                while (!(mcr = bm_mc_result(&p->p)))
                        cpu_relax();
                tmp = mcr->query.ds.state;
                PORTAL_IRQ_UNLOCK(p, irqflags);
                for (i = 0; i < 2; i++) {
                        int idx = i * 32;
                        /* tmp is a mask of currently-depleted pools.
                         * pools[0] is mask of those we care about.
                         * pools[1] is our previous view (we only want to
                         * be told about changes). */
                        tmp.__state[i] &= p->pools[0].__state[i];
                        if (tmp.__state[i] == p->pools[1].__state[i])
                                /* fast-path, nothing to see, move along */
                                continue;
                        for (j = 0; j <= 31; j++, idx++) {
                                struct bman_pool *pool = p->cb[idx];
                                int b4 = bman_depletion_get(&p->pools[1], idx);
                                int af = bman_depletion_get(&tmp, idx);

                                if (b4 == af)
                                        continue;
                                while (pool) {
                                        pool->params.cb(p, pool,
                                                pool->params.cb_ctx, af);
                                        pool = pool->next;
                                }
                        }
                }
                p->pools[1] = tmp;
        }

        if (is & BM_PIRQ_RCRI) {
                __maybe_unused unsigned long irqflags;

                PORTAL_IRQ_LOCK(p, irqflags);
                bm_rcr_cce_update(&p->p);
#ifdef FSL_DPA_CAN_WAIT_SYNC
                /* If waiting for sync, we only cancel the interrupt threshold
                 * when the ring utilisation hits zero. */
                if (p->rcri_owned) {
                        if (!bm_rcr_get_fill(&p->p)) {
                                p->rcri_owned = NULL;
                                bm_rcr_set_ithresh(&p->p, 0);
                        }
                } else
#endif
                bm_rcr_set_ithresh(&p->p, 0);
                PORTAL_IRQ_UNLOCK(p, irqflags);
                wake_up(&affine_queue);
                bm_isr_status_clear(&p->p, BM_PIRQ_RCRI);
                is &= ~BM_PIRQ_RCRI;
        }

        /* There should be no status register bits left undefined */
        DPA_ASSERT(!is);
        return ret;
}

const struct bman_portal_config *bman_get_portal_config(void)
{
        struct bman_portal *p = get_affine_portal();
        const struct bman_portal_config *ret = &p->config->public_cfg;

        put_affine_portal();
        return ret;
}
EXPORT_SYMBOL(bman_get_portal_config);

u32 bman_irqsource_get(void)
{
        struct bman_portal *p = get_raw_affine_portal();
        u32 ret = p->irq_sources & BM_PIRQ_VISIBLE;

        put_affine_portal();
        return ret;
}
EXPORT_SYMBOL(bman_irqsource_get);

int bman_p_irqsource_add(struct bman_portal *p, __maybe_unused u32 bits)
{
        __maybe_unused unsigned long irqflags;

#ifdef FSL_DPA_PORTAL_SHARE
#ifndef __rtems__
        if (p->sharing_redirect)
                return -EINVAL;
#endif /* __rtems__ */
#endif
        PORTAL_IRQ_LOCK(p, irqflags);
        set_bits(bits & BM_PIRQ_VISIBLE, &p->irq_sources);
        bm_isr_enable_write(&p->p, p->irq_sources);
        PORTAL_IRQ_UNLOCK(p, irqflags);
        return 0;
}
EXPORT_SYMBOL(bman_p_irqsource_add);

int bman_irqsource_add(__maybe_unused u32 bits)
{
        struct bman_portal *p = get_raw_affine_portal();
        int ret = bman_p_irqsource_add(p, bits);

        put_affine_portal();
        return ret;
}
EXPORT_SYMBOL(bman_irqsource_add);

int bman_irqsource_remove(u32 bits)
{
        struct bman_portal *p = get_raw_affine_portal();
        __maybe_unused unsigned long irqflags;
        u32 ier;

#ifdef FSL_DPA_PORTAL_SHARE
#ifndef __rtems__
        if (p->sharing_redirect) {
                put_affine_portal();
                return -EINVAL;
        }
#endif /* __rtems__ */
#endif
        /* Our interrupt handler only processes+clears status register bits that
         * are in p->irq_sources. As we're trimming that mask, if one of them
         * were to assert in the status register just before we remove it from
         * the enable register, there would be an interrupt-storm when we
         * release the IRQ lock. So we wait for the enable register update to
         * take effect in h/w (by reading it back) and then clear all other bits
         * in the status register. Ie. we clear them from ISR once it's certain
         * IER won't allow them to reassert. */
        PORTAL_IRQ_LOCK(p, irqflags);
        bits &= BM_PIRQ_VISIBLE;
        clear_bits(bits, &p->irq_sources);
        bm_isr_enable_write(&p->p, p->irq_sources);
        ier = bm_isr_enable_read(&p->p);
        /* Using "~ier" (rather than "bits" or "~p->irq_sources") creates a
         * data-dependency, ie. to protect against re-ordering. */
        bm_isr_status_clear(&p->p, ~ier);
        PORTAL_IRQ_UNLOCK(p, irqflags);
        put_affine_portal();
        return 0;
}
EXPORT_SYMBOL(bman_irqsource_remove);

#ifndef __rtems__
const cpumask_t *bman_affine_cpus(void)
{
        return &affine_mask;
}
EXPORT_SYMBOL(bman_affine_cpus);
#endif /* __rtems__ */

u32 bman_poll_slow(void)
{
        struct bman_portal *p = get_poll_portal();
        u32 ret;

#ifdef FSL_DPA_PORTAL_SHARE
#ifndef __rtems__
        if (unlikely(p->sharing_redirect))
                ret = (u32)-1;
        else
#endif /* __rtems__ */
#endif
        {
                u32 is = bm_isr_status_read(&p->p) & ~p->irq_sources;

                ret = __poll_portal_slow(p, is);
                bm_isr_status_clear(&p->p, ret);
        }
        put_poll_portal();
        return ret;
}
EXPORT_SYMBOL(bman_poll_slow);

/* Legacy wrapper */
void bman_poll(void)
{
        struct bman_portal *p = get_poll_portal();

#ifdef FSL_DPA_PORTAL_SHARE
#ifndef __rtems__
        if (unlikely(p->sharing_redirect))
                goto done;
#endif /* __rtems__ */
#endif
        if (!(p->slowpoll--)) {
                u32 is = bm_isr_status_read(&p->p) & ~p->irq_sources;
                u32 active = __poll_portal_slow(p, is);

                if (active)
                        p->slowpoll = SLOW_POLL_BUSY;
                else
                        p->slowpoll = SLOW_POLL_IDLE;
        }
#ifdef FSL_DPA_PORTAL_SHARE
#ifndef __rtems__
done:
#endif /* __rtems__ */
#endif
        put_poll_portal();
}
EXPORT_SYMBOL(bman_poll);

static const u32 zero_thresholds[4] = {0, 0, 0, 0};

struct bman_pool *bman_new_pool(const struct bman_pool_params *params)
{
        struct bman_pool *pool = NULL;
        u32 bpid;

        if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID) {
                if (bman_alloc_bpid(&bpid))
                        return NULL;
        } else {
                if (params->bpid >= bman_pool_max)
                        return NULL;
                bpid = params->bpid;
        }
#ifdef CONFIG_FSL_BMAN
        if (params->flags & BMAN_POOL_FLAG_THRESH) {
                if (bm_pool_set(bpid, params->thresholds))
                        goto err;
        }
#else
        if (params->flags & BMAN_POOL_FLAG_THRESH)
                goto err;
#endif
        pool = kmalloc(sizeof(*pool), GFP_KERNEL);
        if (!pool)
                goto err;
        pool->sp = NULL;
        pool->sp_fill = 0;
        pool->params = *params;
#ifdef CONFIG_FSL_DPA_CHECKING
        atomic_set(&pool->in_use, 1);
#endif
        if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID)
                pool->params.bpid = bpid;
        if (params->flags & BMAN_POOL_FLAG_STOCKPILE) {
                pool->sp = kmalloc(sizeof(struct bm_buffer) * BMAN_STOCKPILE_SZ,
                                        GFP_KERNEL);
                if (!pool->sp)
                        goto err;
        }
        if (pool->params.flags & BMAN_POOL_FLAG_DEPLETION) {
                struct bman_portal *p = get_affine_portal();

                if (!p->pools || !bman_depletion_get(&p->pools[0], bpid)) {
                        pr_err("Depletion events disabled for bpid %d\n", bpid);
                        goto err;
                }
                depletion_link(p, pool);
                put_affine_portal();
        }
        return pool;
err:
#ifdef CONFIG_FSL_BMAN
        if (params->flags & BMAN_POOL_FLAG_THRESH)
                bm_pool_set(bpid, zero_thresholds);
#endif
        if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID)
                bman_release_bpid(bpid);
        if (pool) {
                kfree(pool->sp);
                kfree(pool);
        }
        return NULL;
}
EXPORT_SYMBOL(bman_new_pool);

void bman_free_pool(struct bman_pool *pool)
{
#ifdef CONFIG_FSL_BMAN
        if (pool->params.flags & BMAN_POOL_FLAG_THRESH)
                bm_pool_set(pool->params.bpid, zero_thresholds);
#endif
        if (pool->params.flags & BMAN_POOL_FLAG_DEPLETION)
                depletion_unlink(pool);
        if (pool->params.flags & BMAN_POOL_FLAG_STOCKPILE) {
                if (pool->sp_fill)
                        pr_err("Stockpile not flushed, has %u in bpid %u.\n",
                                pool->sp_fill, pool->params.bpid);
                kfree(pool->sp);
                pool->sp = NULL;
                pool->params.flags ^= BMAN_POOL_FLAG_STOCKPILE;
        }
        if (pool->params.flags & BMAN_POOL_FLAG_DYNAMIC_BPID)
                bman_release_bpid(pool->params.bpid);
        kfree(pool);
}
EXPORT_SYMBOL(bman_free_pool);

const struct bman_pool_params *bman_get_params(const struct bman_pool *pool)
{
        return &pool->params;
}
EXPORT_SYMBOL(bman_get_params);

static noinline void update_rcr_ci(struct bman_portal *p, u8 avail)
{
        if (avail)
                bm_rcr_cce_prefetch(&p->p);
        else
                bm_rcr_cce_update(&p->p);
}

int bman_rcr_is_empty(void)
{
        __maybe_unused unsigned long irqflags;
        struct bman_portal *p = get_affine_portal();
        u8 avail;

        PORTAL_IRQ_LOCK(p, irqflags);
        update_rcr_ci(p, 0);
        avail = bm_rcr_get_fill(&p->p);
        PORTAL_IRQ_UNLOCK(p, irqflags);
        put_affine_portal();
        return avail == 0;
}
EXPORT_SYMBOL(bman_rcr_is_empty);

static inline struct bm_rcr_entry *try_rel_start(struct bman_portal **p,
#ifdef FSL_DPA_CAN_WAIT
                                        __maybe_unused struct bman_pool *pool,
#endif
                                        __maybe_unused unsigned long *irqflags,
                                        __maybe_unused u32 flags)
{
        struct bm_rcr_entry *r;
        u8 avail;

        *p = get_affine_portal();
        PORTAL_IRQ_LOCK(*p, (*irqflags));
#ifdef FSL_DPA_CAN_WAIT_SYNC
        if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) &&
                        (flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) {
                if ((*p)->rcri_owned) {
                        PORTAL_IRQ_UNLOCK(*p, (*irqflags));
                        put_affine_portal();
                        return NULL;
                }
                (*p)->rcri_owned = pool;
        }
#endif
        avail = bm_rcr_get_avail(&(*p)->p);
        if (avail < 2)
                update_rcr_ci(*p, avail);
        r = bm_rcr_start(&(*p)->p);
        if (unlikely(!r)) {
#ifdef FSL_DPA_CAN_WAIT_SYNC
                if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) &&
                                (flags & BMAN_RELEASE_FLAG_WAIT_SYNC)))
                        (*p)->rcri_owned = NULL;
#endif
                PORTAL_IRQ_UNLOCK(*p, (*irqflags));
                put_affine_portal();
        }
        return r;
}

#ifdef FSL_DPA_CAN_WAIT
static noinline struct bm_rcr_entry *__wait_rel_start(struct bman_portal **p,
                                        struct bman_pool *pool,
                                        __maybe_unused unsigned long *irqflags,
                                        u32 flags)
{
        struct bm_rcr_entry *rcr = try_rel_start(p, pool, irqflags, flags);

        if (!rcr)
                bm_rcr_set_ithresh(&(*p)->p, 1);
        return rcr;
}

static noinline struct bm_rcr_entry *wait_rel_start(struct bman_portal **p,
                                        struct bman_pool *pool,
                                        __maybe_unused unsigned long *irqflags,
                                        u32 flags)
{
        struct bm_rcr_entry *rcr;
#ifndef FSL_DPA_CAN_WAIT_SYNC
        pool = NULL;
#endif
#ifndef __rtems__
        if (flags & BMAN_RELEASE_FLAG_WAIT_INT)
                wait_event_interruptible(affine_queue,
                        (rcr = __wait_rel_start(p, pool, irqflags, flags)));
        else
#endif /* __rtems__ */
                wait_event(affine_queue,
                        (rcr = __wait_rel_start(p, pool, irqflags, flags)));
        return rcr;
}
#endif

/* to facilitate better copying of bufs into the ring without either (a) copying
 * noise into the first byte (prematurely triggering the command), nor (b) being
 * very inefficient by copying small fields using read-modify-write */
struct overlay_bm_buffer {
        u32 first;
        u32 second;
};

static inline int __bman_release(struct bman_pool *pool,
                        const struct bm_buffer *bufs, u8 num, u32 flags)
{
        struct bman_portal *p;
        struct bm_rcr_entry *r;
        struct overlay_bm_buffer *o_dest;
        struct overlay_bm_buffer *o_src = (struct overlay_bm_buffer *)&bufs[0];
        __maybe_unused unsigned long irqflags;
        u32 i = num - 1;

#ifdef FSL_DPA_CAN_WAIT
        if (flags & BMAN_RELEASE_FLAG_WAIT)
                r = wait_rel_start(&p, pool, &irqflags, flags);
        else
                r = try_rel_start(&p, pool, &irqflags, flags);
#else
        r = try_rel_start(&p, &irqflags, flags);
#endif
        if (!r)
                return -EBUSY;
        /* We can copy all but the first entry, as this can trigger badness
         * with the valid-bit. Use the overlay to mask the verb byte. */
        o_dest = (struct overlay_bm_buffer *)&r->bufs[0];
        o_dest->first = (o_src->first & 0x0000ffff) |
                (((u32)pool->params.bpid << 16) & 0x00ff0000);
        o_dest->second = o_src->second;
        if (i)
                copy_words(&r->bufs[1], &bufs[1], i * sizeof(bufs[0]));
        bm_rcr_pvb_commit(&p->p, BM_RCR_VERB_CMD_BPID_SINGLE |
                        (num & BM_RCR_VERB_BUFCOUNT_MASK));
#ifdef FSL_DPA_CAN_WAIT_SYNC
        /* if we wish to sync we need to set the threshold after h/w sees the
         * new ring entry. As we're mixing cache-enabled and cache-inhibited
         * accesses, this requires a heavy-weight sync. */
        if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) &&
                        (flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) {
                hwsync();
                bm_rcr_set_ithresh(&p->p, 1);
        }
#endif
        PORTAL_IRQ_UNLOCK(p, irqflags);
        put_affine_portal();
#ifdef FSL_DPA_CAN_WAIT_SYNC
        if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) &&
                        (flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) {
#ifndef __rtems__
                if (flags & BMAN_RELEASE_FLAG_WAIT_INT)
                        wait_event_interruptible(affine_queue,
                                        (p->rcri_owned != pool));
                else
#endif /* __rtems__ */
                        wait_event(affine_queue, (p->rcri_owned != pool));
        }
#endif
        return 0;
}

int bman_release(struct bman_pool *pool, const struct bm_buffer *bufs, u8 num,
                        u32 flags)
{
        int ret = 0;

#ifdef CONFIG_FSL_DPA_CHECKING
        if (!num || (num > 8))
                return -EINVAL;
        if (pool->params.flags & BMAN_POOL_FLAG_NO_RELEASE)
                return -EINVAL;
#endif
        /* Without stockpile, this API is a pass-through to the h/w operation */
        if (!(pool->params.flags & BMAN_POOL_FLAG_STOCKPILE))
                return __bman_release(pool, bufs, num, flags);
#ifdef CONFIG_FSL_DPA_CHECKING
        if (!atomic_dec_and_test(&pool->in_use)) {
                pr_crit("Parallel attempts to enter bman_released() detected.");
                panic("only one instance of bman_released/acquired allowed");
        }
#endif
        /* This needs some explanation. Adding the given buffers may take the
         * stockpile over the threshold, but in fact the stockpile may already
         * *be* over the threshold if a previous release-to-hw attempt had
         * failed. So we have 3 cases to cover;
         *   1. we add to the stockpile and don't hit the threshold,
         *   2. we add to the stockpile, hit the threshold and release-to-hw,
         *   3. we have to release-to-hw before adding to the stockpile
         *      (not enough room in the stockpile for case 2).
         * Our constraints on thresholds guarantee that in case 3, there must be
         * at least 8 bufs already in the stockpile, so all release-to-hw ops
         * are for 8 bufs. Despite all this, the API must indicate whether the
         * given buffers were taken off the caller's hands, irrespective of
         * whether a release-to-hw was attempted. */
        while (num) {
                /* Add buffers to stockpile if they fit */
                if ((pool->sp_fill + num) < BMAN_STOCKPILE_SZ) {
                        copy_words(pool->sp + pool->sp_fill, bufs,
                                sizeof(struct bm_buffer) * num);
                        pool->sp_fill += num;
                        num = 0; /* --> will return success no matter what */
                }
                /* Do hw op if hitting the high-water threshold */
                if ((pool->sp_fill + num) >= BMAN_STOCKPILE_HIGH) {
                        ret = __bman_release(pool,
                                pool->sp + (pool->sp_fill - 8), 8, flags);
                        if (ret) {
                                ret = (num ? ret : 0);
                                goto release_done;
                        }
                        pool->sp_fill -= 8;
                }
        }
release_done:
#ifdef CONFIG_FSL_DPA_CHECKING
        atomic_inc(&pool->in_use);
#endif
        return ret;
}
EXPORT_SYMBOL(bman_release);

static inline int __bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs,
                                        u8 num)
{
        struct bman_portal *p = get_affine_portal();
        struct bm_mc_command *mcc;
        struct bm_mc_result *mcr;
        __maybe_unused unsigned long irqflags;
        int ret;

        PORTAL_IRQ_LOCK(p, irqflags);
        mcc = bm_mc_start(&p->p);
        mcc->acquire.bpid = pool->params.bpid;
        bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE |
                        (num & BM_MCC_VERB_ACQUIRE_BUFCOUNT));
        while (!(mcr = bm_mc_result(&p->p)))
                cpu_relax();
        ret = mcr->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT;
        if (bufs)
                copy_words(&bufs[0], &mcr->acquire.bufs[0],
                                num * sizeof(bufs[0]));
        PORTAL_IRQ_UNLOCK(p, irqflags);
        put_affine_portal();
        if (ret != num)
                ret = -ENOMEM;
        return ret;
}

int bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, u8 num,
                        u32 flags)
{
        int ret = 0;

#ifdef CONFIG_FSL_DPA_CHECKING
        if (!num || (num > 8))
                return -EINVAL;
        if (pool->params.flags & BMAN_POOL_FLAG_ONLY_RELEASE)
                return -EINVAL;
#endif
        /* Without stockpile, this API is a pass-through to the h/w operation */
        if (!(pool->params.flags & BMAN_POOL_FLAG_STOCKPILE))
                return __bman_acquire(pool, bufs, num);
#ifdef CONFIG_FSL_DPA_CHECKING
        if (!atomic_dec_and_test(&pool->in_use)) {
                pr_crit("Parallel attempts to enter bman_acquire() detected.");
                panic("only one instance of bman_released/acquired allowed");
        }
#endif
        /* Only need a h/w op if we'll hit the low-water thresh */
        if (!(flags & BMAN_ACQUIRE_FLAG_STOCKPILE) &&
                        (pool->sp_fill <= (BMAN_STOCKPILE_LOW + num))) {
                /* refill stockpile with max amount, but if max amount
                 * isn't available, try amount the user wants */
                int bufcount = 8;

                ret = __bman_acquire(pool, pool->sp + pool->sp_fill, bufcount);
                if (ret < 0 && bufcount != num) {
                        bufcount = num;
                        /* Maybe buffer pool has less than 8 */
                        ret = __bman_acquire(pool, pool->sp + pool->sp_fill,
                                                bufcount);
                }
                if (ret < 0)
                        goto hw_starved;
                DPA_ASSERT(ret == bufcount);
                pool->sp_fill += bufcount;
        } else {
hw_starved:
                if (pool->sp_fill < num) {
                        ret = -ENOMEM;
                        goto acquire_done;
                }
        }
        copy_words(bufs, pool->sp + (pool->sp_fill - num),
                sizeof(struct bm_buffer) * num);
        pool->sp_fill -= num;
        ret = num;
acquire_done:
#ifdef CONFIG_FSL_DPA_CHECKING
        atomic_inc(&pool->in_use);
#endif
        return ret;
}
EXPORT_SYMBOL(bman_acquire);

int bman_flush_stockpile(struct bman_pool *pool, u32 flags)
{
        u8 num;
        int ret;

        while (pool->sp_fill) {
                num = ((pool->sp_fill > 8) ? 8 : pool->sp_fill);
                ret = __bman_release(pool, pool->sp + (pool->sp_fill - num),
                                     num, flags);
                if (ret)
                        return ret;
                pool->sp_fill -= num;
        }
        return 0;
}
EXPORT_SYMBOL(bman_flush_stockpile);

int bman_query_pools(struct bm_pool_state *state)
{
        struct bman_portal *p = get_affine_portal();
        struct bm_mc_result *mcr;
        __maybe_unused unsigned long irqflags;

        PORTAL_IRQ_LOCK(p, irqflags);
        bm_mc_start(&p->p);
        bm_mc_commit(&p->p, BM_MCC_VERB_CMD_QUERY);
        while (!(mcr = bm_mc_result(&p->p)))
                cpu_relax();
        DPA_ASSERT((mcr->verb & BM_MCR_VERB_CMD_MASK) == BM_MCR_VERB_CMD_QUERY);
        *state = mcr->query;
        PORTAL_IRQ_UNLOCK(p, irqflags);
        put_affine_portal();
        return 0;
}
EXPORT_SYMBOL(bman_query_pools);

#ifdef CONFIG_FSL_BMAN
u32 bman_query_free_buffers(struct bman_pool *pool)
{
        return bm_pool_free_buffers(pool->params.bpid);
}
EXPORT_SYMBOL(bman_query_free_buffers);

int bman_update_pool_thresholds(struct bman_pool *pool, const u32 *thresholds)
{
        u32 bpid;

        bpid = bman_get_params(pool)->bpid;

        return bm_pool_set(bpid, thresholds);
}
EXPORT_SYMBOL(bman_update_pool_thresholds);
#endif

int bman_shutdown_pool(u32 bpid)
{
        struct bman_portal *p = get_affine_portal();
        __maybe_unused unsigned long irqflags;
        int ret;

        PORTAL_IRQ_LOCK(p, irqflags);
        ret = bm_shutdown_pool(&p->p, bpid);
        PORTAL_IRQ_UNLOCK(p, irqflags);
        put_affine_portal();
        return ret;
}
EXPORT_SYMBOL(bman_shutdown_pool);

const struct bm_portal_config *
bman_get_bm_portal_config(const struct bman_portal *portal)
{
#ifndef __rtems__
        return portal->sharing_redirect ? NULL : portal->config;
#else /* __rtems__ */
        return portal->config;
#endif /* __rtems__ */
}