Context Navigation

qman.c @ 1908686

55-freebsd-126-freebsd-12

Last change on this file since 1908686 was 1908686, checked in by Sebastian Huber <sebastian.huber@…>, on 05/19/17 at 07:41:16

soc/qman: Do not enable stashing without PAMU

Do not enable stashing in the QMan software portal configuration
(QCSPi_CFG[RE, SE]) in case the PAMU support is not configured.

Signed-off-by: Sebastian Huber <sebastian.huber@…>

Property mode set to 100644

File size: 75.9 KB

Line
1	#include <machine/rtems-bsd-kernel-space.h>
2
3	#include <rtems/bsd/local/opt_dpaa.h>
4
5	/* Copyright 2008 - 2016 Freescale Semiconductor, Inc.
6	*
7	* Redistribution and use in source and binary forms, with or without
8	* modification, are permitted provided that the following conditions are met:
9	* * Redistributions of source code must retain the above copyright
10	* notice, this list of conditions and the following disclaimer.
11	* * Redistributions in binary form must reproduce the above copyright
12	* notice, this list of conditions and the following disclaimer in the
13	* documentation and/or other materials provided with the distribution.
14	* * Neither the name of Freescale Semiconductor nor the
15	* names of its contributors may be used to endorse or promote products
16	* derived from this software without specific prior written permission.
17	*
18	* ALTERNATIVELY, this software may be distributed under the terms of the
19	* GNU General Public License ("GPL") as published by the Free Software
20	* Foundation, either version 2 of that License or (at your option) any
21	* later version.
22	*
23	* THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
24	* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
25	* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
26	* DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
27	* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
28	* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
29	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
30	* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
32	* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33	*/
34
35	#include "qman_priv.h"
36	#ifdef __rtems__
37	#undef dev_crit
38	#undef dev_dbg
39	#undef dev_err
40	#define dev_crit(dev, fmt, ...) printf(fmt, ##__VA_ARGS__)
41	#define dev_dbg dev_crit
42	#define dev_err dev_crit
43	#endif /* __rtems__ */
44
45	#define DQRR_MAXFILL 15
46	#define EQCR_ITHRESH 4 /* if EQCR congests, interrupt threshold */
47	#define IRQNAME "QMan portal %d"
48	#define MAX_IRQNAME 16 /* big enough for "QMan portal %d" */
49	#define QMAN_POLL_LIMIT 32
50	#define QMAN_PIRQ_DQRR_ITHRESH 12
51	#define QMAN_PIRQ_MR_ITHRESH 4
52	#define QMAN_PIRQ_IPERIOD 100
53
54	/* Portal register assists */
55
56	/* Cache-inhibited register offsets */
57	#define QM_REG_EQCR_PI_CINH 0x0000
58	#define QM_REG_EQCR_CI_CINH 0x0004
59	#define QM_REG_EQCR_ITR 0x0008
60	#define QM_REG_DQRR_PI_CINH 0x0040
61	#define QM_REG_DQRR_CI_CINH 0x0044
62	#define QM_REG_DQRR_ITR 0x0048
63	#define QM_REG_DQRR_DCAP 0x0050
64	#define QM_REG_DQRR_SDQCR 0x0054
65	#define QM_REG_DQRR_VDQCR 0x0058
66	#define QM_REG_DQRR_PDQCR 0x005c
67	#define QM_REG_MR_PI_CINH 0x0080
68	#define QM_REG_MR_CI_CINH 0x0084
69	#define QM_REG_MR_ITR 0x0088
70	#define QM_REG_CFG 0x0100
71	#define QM_REG_ISR 0x0e00
72	#define QM_REG_IER 0x0e04
73	#define QM_REG_ISDR 0x0e08
74	#define QM_REG_IIR 0x0e0c
75	#define QM_REG_ITPR 0x0e14
76
77	/* Cache-enabled register offsets */
78	#define QM_CL_EQCR 0x0000
79	#define QM_CL_DQRR 0x1000
80	#define QM_CL_MR 0x2000
81	#define QM_CL_EQCR_PI_CENA 0x3000
82	#define QM_CL_EQCR_CI_CENA 0x3100
83	#define QM_CL_DQRR_PI_CENA 0x3200
84	#define QM_CL_DQRR_CI_CENA 0x3300
85	#define QM_CL_MR_PI_CENA 0x3400
86	#define QM_CL_MR_CI_CENA 0x3500
87	#define QM_CL_CR 0x3800
88	#define QM_CL_RR0 0x3900
89	#define QM_CL_RR1 0x3940
90
91	/*
92	* BTW, the drivers (and h/w programming model) already obtain the required
93	* synchronisation for portal accesses and data-dependencies. Use of barrier()s
94	* or other order-preserving primitives simply degrade performance. Hence the
95	* use of the __raw_*() interfaces, which simply ensure that the compiler treats
96	* the portal registers as volatile
97	*/
98
99	/* Cache-enabled ring access */
100	#define qm_cl(base, idx) ((void *)base + ((idx) << 6))
101
102	/*
103	* Portal modes.
104	* Enum types;
105	* pmode == production mode
106	* cmode == consumption mode,
107	* dmode == h/w dequeue mode.
108	* Enum values use 3 letter codes. First letter matches the portal mode,
109	* remaining two letters indicate;
110	* ci == cache-inhibited portal register
111	* ce == cache-enabled portal register
112	* vb == in-band valid-bit (cache-enabled)
113	* dc == DCA (Discrete Consumption Acknowledgment), DQRR-only
114	* As for "enum qm_dqrr_dmode", it should be self-explanatory.
115	*/
116	enum qm_eqcr_pmode { /* matches QCSP_CFG::EPM */
117	qm_eqcr_pci = 0, /* PI index, cache-inhibited */
118	qm_eqcr_pce = 1, /* PI index, cache-enabled */
119	qm_eqcr_pvb = 2 /* valid-bit */
120	};
121	enum qm_dqrr_dmode { /* matches QCSP_CFG::DP */
122	qm_dqrr_dpush = 0, /* SDQCR + VDQCR */
123	qm_dqrr_dpull = 1 /* PDQCR */
124	};
125	enum qm_dqrr_pmode { /* s/w-only */
126	qm_dqrr_pci, /* reads DQRR_PI_CINH */
127	qm_dqrr_pce, /* reads DQRR_PI_CENA */
128	qm_dqrr_pvb /* reads valid-bit */
129	};
130	enum qm_dqrr_cmode { /* matches QCSP_CFG::DCM */
131	qm_dqrr_cci = 0, /* CI index, cache-inhibited */
132	qm_dqrr_cce = 1, /* CI index, cache-enabled */
133	qm_dqrr_cdc = 2 /* Discrete Consumption Acknowledgment */
134	};
135	enum qm_mr_pmode { /* s/w-only */
136	qm_mr_pci, /* reads MR_PI_CINH */
137	qm_mr_pce, /* reads MR_PI_CENA */
138	qm_mr_pvb /* reads valid-bit */
139	};
140	enum qm_mr_cmode { /* matches QCSP_CFG::MM */
141	qm_mr_cci = 0, /* CI index, cache-inhibited */
142	qm_mr_cce = 1 /* CI index, cache-enabled */
143	};
144
145	/* --- Portal structures --- */
146
147	#define QM_EQCR_SIZE 8
148	#define QM_DQRR_SIZE 16
149	#define QM_MR_SIZE 8
150
151	/* "Enqueue Command" */
152	struct qm_eqcr_entry {
153	u8 _ncw_verb; /* writes to this are non-coherent */
154	u8 dca;
155	__be16 seqnum;
156	u8 __reserved[4];
157	__be32 fqid; /* 24-bit */
158	__be32 tag;
159	struct qm_fd fd;
160	u8 __reserved3[32];
161	} __packed;
162	#define QM_EQCR_VERB_VBIT 0x80
163	#define QM_EQCR_VERB_CMD_MASK 0x61 /* but only one value; */
164	#define QM_EQCR_VERB_CMD_ENQUEUE 0x01
165	#define QM_EQCR_SEQNUM_NESN 0x8000 /* Advance NESN */
166	#define QM_EQCR_SEQNUM_NLIS 0x4000 /* More fragments to come */
167	#define QM_EQCR_SEQNUM_SEQMASK 0x3fff /* sequence number goes here */
168
169	struct qm_eqcr {
170	struct qm_eqcr_entry ring, cursor;
171	u8 ci, available, ithresh, vbit;
172	#ifdef CONFIG_FSL_DPAA_CHECKING
173	u32 busy;
174	enum qm_eqcr_pmode pmode;
175	#endif
176	};
177
178	struct qm_dqrr {
179	const struct qm_dqrr_entry ring, cursor;
180	u8 pi, ci, fill, ithresh, vbit;
181	#ifdef CONFIG_FSL_DPAA_CHECKING
182	enum qm_dqrr_dmode dmode;
183	enum qm_dqrr_pmode pmode;
184	enum qm_dqrr_cmode cmode;
185	#endif
186	};
187
188	struct qm_mr {
189	union qm_mr_entry ring, cursor;
190	u8 pi, ci, fill, ithresh, vbit;
191	#ifdef CONFIG_FSL_DPAA_CHECKING
192	enum qm_mr_pmode pmode;
193	enum qm_mr_cmode cmode;
194	#endif
195	};
196
197	/* MC (Management Command) command */
198	/* "FQ" command layout */
199	struct qm_mcc_fq {
200	u8 _ncw_verb;
201	u8 __reserved1[3];
202	__be32 fqid; /* 24-bit */
203	u8 __reserved2[56];
204	} __packed;
205
206	/* "CGR" command layout */
207	struct qm_mcc_cgr {
208	u8 _ncw_verb;
209	u8 __reserved1[30];
210	u8 cgid;
211	u8 __reserved2[32];
212	};
213
214	#define QM_MCC_VERB_VBIT 0x80
215	#define QM_MCC_VERB_MASK 0x7f /* where the verb contains; */
216	#define QM_MCC_VERB_INITFQ_PARKED 0x40
217	#define QM_MCC_VERB_INITFQ_SCHED 0x41
218	#define QM_MCC_VERB_QUERYFQ 0x44
219	#define QM_MCC_VERB_QUERYFQ_NP 0x45 /* "non-programmable" fields */
220	#define QM_MCC_VERB_QUERYWQ 0x46
221	#define QM_MCC_VERB_QUERYWQ_DEDICATED 0x47
222	#define QM_MCC_VERB_ALTER_SCHED 0x48 /* Schedule FQ */
223	#define QM_MCC_VERB_ALTER_FE 0x49 /* Force Eligible FQ */
224	#define QM_MCC_VERB_ALTER_RETIRE 0x4a /* Retire FQ */
225	#define QM_MCC_VERB_ALTER_OOS 0x4b /* Take FQ out of service */
226	#define QM_MCC_VERB_ALTER_FQXON 0x4d /* FQ XON */
227	#define QM_MCC_VERB_ALTER_FQXOFF 0x4e /* FQ XOFF */
228	#define QM_MCC_VERB_INITCGR 0x50
229	#define QM_MCC_VERB_MODIFYCGR 0x51
230	#define QM_MCC_VERB_CGRTESTWRITE 0x52
231	#define QM_MCC_VERB_QUERYCGR 0x58
232	#define QM_MCC_VERB_QUERYCONGESTION 0x59
233	union qm_mc_command {
234	struct {
235	u8 _ncw_verb; /* writes to this are non-coherent */
236	u8 __reserved[63];
237	};
238	struct qm_mcc_initfq initfq;
239	struct qm_mcc_initcgr initcgr;
240	struct qm_mcc_fq fq;
241	struct qm_mcc_cgr cgr;
242	};
243
244	/* MC (Management Command) result */
245	/* "Query FQ" */
246	struct qm_mcr_queryfq {
247	u8 verb;
248	u8 result;
249	u8 __reserved1[8];
250	struct qm_fqd fqd; /* the FQD fields are here */
251	u8 __reserved2[30];
252	} __packed;
253
254	/* "Alter FQ State Commands" */
255	struct qm_mcr_alterfq {
256	u8 verb;
257	u8 result;
258	u8 fqs; /* Frame Queue Status */
259	u8 __reserved1[61];
260	};
261	#define QM_MCR_VERB_RRID 0x80
262	#define QM_MCR_VERB_MASK QM_MCC_VERB_MASK
263	#define QM_MCR_VERB_INITFQ_PARKED QM_MCC_VERB_INITFQ_PARKED
264	#define QM_MCR_VERB_INITFQ_SCHED QM_MCC_VERB_INITFQ_SCHED
265	#define QM_MCR_VERB_QUERYFQ QM_MCC_VERB_QUERYFQ
266	#define QM_MCR_VERB_QUERYFQ_NP QM_MCC_VERB_QUERYFQ_NP
267	#define QM_MCR_VERB_QUERYWQ QM_MCC_VERB_QUERYWQ
268	#define QM_MCR_VERB_QUERYWQ_DEDICATED QM_MCC_VERB_QUERYWQ_DEDICATED
269	#define QM_MCR_VERB_ALTER_SCHED QM_MCC_VERB_ALTER_SCHED
270	#define QM_MCR_VERB_ALTER_FE QM_MCC_VERB_ALTER_FE
271	#define QM_MCR_VERB_ALTER_RETIRE QM_MCC_VERB_ALTER_RETIRE
272	#define QM_MCR_VERB_ALTER_OOS QM_MCC_VERB_ALTER_OOS
273	#define QM_MCR_RESULT_NULL 0x00
274	#define QM_MCR_RESULT_OK 0xf0
275	#define QM_MCR_RESULT_ERR_FQID 0xf1
276	#define QM_MCR_RESULT_ERR_FQSTATE 0xf2
277	#define QM_MCR_RESULT_ERR_NOTEMPTY 0xf3 /* OOS fails if FQ is !empty */
278	#define QM_MCR_RESULT_ERR_BADCHANNEL 0xf4
279	#define QM_MCR_RESULT_PENDING 0xf8
280	#define QM_MCR_RESULT_ERR_BADCOMMAND 0xff
281	#define QM_MCR_FQS_ORLPRESENT 0x02 /* ORL fragments to come */
282	#define QM_MCR_FQS_NOTEMPTY 0x01 /* FQ has enqueued frames */
283	#define QM_MCR_TIMEOUT 10000 /* us */
284	union qm_mc_result {
285	struct {
286	u8 verb;
287	u8 result;
288	u8 __reserved1[62];
289	};
290	struct qm_mcr_queryfq queryfq;
291	struct qm_mcr_alterfq alterfq;
292	struct qm_mcr_querycgr querycgr;
293	struct qm_mcr_querycongestion querycongestion;
294	struct qm_mcr_querywq querywq;
295	struct qm_mcr_queryfq_np queryfq_np;
296	};
297
298	struct qm_mc {
299	union qm_mc_command *cr;
300	union qm_mc_result *rr;
301	u8 rridx, vbit;
302	#ifdef CONFIG_FSL_DPAA_CHECKING
303	enum {
304	/* Can be _mc_start()ed */
305	qman_mc_idle,
306	/* Can be _mc_commit()ed or _mc_abort()ed */
307	qman_mc_user,
308	/* Can only be _mc_retry()ed */
309	qman_mc_hw
310	} state;
311	#endif
312	};
313
314	struct qm_addr {
315	void __iomem ce; / cache-enabled */
316	void __iomem ci; / cache-inhibited */
317	};
318
319	struct qm_portal {
320	/*
321	* In the non-CONFIG_FSL_DPAA_CHECKING case, the following stuff up to
322	* and including 'mc' fits within a cacheline (yay!). The 'config' part
323	* is setup-only, so isn't a cause for a concern. In other words, don't
324	* rearrange this structure on a whim, there be dragons ...
325	*/
326	struct qm_addr addr;
327	struct qm_eqcr eqcr;
328	struct qm_dqrr dqrr;
329	struct qm_mr mr;
330	struct qm_mc mc;
331	} ____cacheline_aligned;
332
333	/* Cache-inhibited register access. */
334	static inline u32 qm_in(struct qm_portal *p, u32 offset)
335	{
336	return be32_to_cpu(__raw_readl(p->addr.ci + offset));
337	}
338
339	static inline void qm_out(struct qm_portal *p, u32 offset, u32 val)
340	{
341	__raw_writel(cpu_to_be32(val), p->addr.ci + offset);
342	}
343
344	/* Cache Enabled Portal Access */
345	static inline void qm_cl_invalidate(struct qm_portal *p, u32 offset)
346	{
347	dpaa_invalidate(p->addr.ce + offset);
348	}
349
350	static inline void qm_cl_touch_ro(struct qm_portal *p, u32 offset)
351	{
352	dpaa_touch_ro(p->addr.ce + offset);
353	}
354
355	static inline u32 qm_ce_in(struct qm_portal *p, u32 offset)
356	{
357	return be32_to_cpu(__raw_readl(p->addr.ce + offset));
358	}
359
360	/* --- EQCR API --- */
361
362	#define EQCR_SHIFT ilog2(sizeof(struct qm_eqcr_entry))
363	#define EQCR_CARRY (uintptr_t)(QM_EQCR_SIZE << EQCR_SHIFT)
364
365	/* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
366	static struct qm_eqcr_entry eqcr_carryclear(struct qm_eqcr_entry p)
367	{
368	uintptr_t addr = (uintptr_t)p;
369
370	addr &= ~EQCR_CARRY;
371
372	return (struct qm_eqcr_entry *)addr;
373	}
374
375	/* Bit-wise logic to convert a ring pointer to a ring index */
376	static int eqcr_ptr2idx(struct qm_eqcr_entry *e)
377	{
378	return ((uintptr_t)e >> EQCR_SHIFT) & (QM_EQCR_SIZE - 1);
379	}
380
381	/* Increment the 'cursor' ring pointer, taking 'vbit' into account */
382	static inline void eqcr_inc(struct qm_eqcr *eqcr)
383	{
384	/* increment to the next EQCR pointer and handle overflow and 'vbit' */
385	struct qm_eqcr_entry *partial = eqcr->cursor + 1;
386
387	eqcr->cursor = eqcr_carryclear(partial);
388	if (partial != eqcr->cursor)
389	eqcr->vbit ^= QM_EQCR_VERB_VBIT;
390	}
391
392	static inline int qm_eqcr_init(struct qm_portal *portal,
393	enum qm_eqcr_pmode pmode,
394	unsigned int eq_stash_thresh,
395	int eq_stash_prio)
396	{
397	struct qm_eqcr *eqcr = &portal->eqcr;
398	u32 cfg;
399	u8 pi;
400
401	eqcr->ring = portal->addr.ce + QM_CL_EQCR;
402	eqcr->ci = qm_in(portal, QM_REG_EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
403	qm_cl_invalidate(portal, QM_CL_EQCR_CI_CENA);
404	pi = qm_in(portal, QM_REG_EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
405	eqcr->cursor = eqcr->ring + pi;
406	eqcr->vbit = (qm_in(portal, QM_REG_EQCR_PI_CINH) & QM_EQCR_SIZE) ?
407	QM_EQCR_VERB_VBIT : 0;
408	eqcr->available = QM_EQCR_SIZE - 1 -
409	dpaa_cyc_diff(QM_EQCR_SIZE, eqcr->ci, pi);
410	eqcr->ithresh = qm_in(portal, QM_REG_EQCR_ITR);
411	#ifdef CONFIG_FSL_DPAA_CHECKING
412	eqcr->busy = 0;
413	eqcr->pmode = pmode;
414	#endif
415	cfg = (qm_in(portal, QM_REG_CFG) & 0x00ffffff) \|
416	(eq_stash_thresh << 28) \| /* QCSP_CFG: EST */
417	(eq_stash_prio << 26) \| /* QCSP_CFG: EP */
418	((pmode & 0x3) << 24); /* QCSP_CFG::EPM */
419	qm_out(portal, QM_REG_CFG, cfg);
420	return 0;
421	}
422
423	static inline unsigned int qm_eqcr_get_ci_stashing(struct qm_portal *portal)
424	{
425	return (qm_in(portal, QM_REG_CFG) >> 28) & 0x7;
426	}
427
428	static inline void qm_eqcr_finish(struct qm_portal *portal)
429	{
430	struct qm_eqcr *eqcr = &portal->eqcr;
431	u8 pi = qm_in(portal, QM_REG_EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
432	u8 ci = qm_in(portal, QM_REG_EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
433
434	DPAA_ASSERT(!eqcr->busy);
435	if (pi != eqcr_ptr2idx(eqcr->cursor))
436	pr_crit("losing uncommitted EQCR entries\n");
437	if (ci != eqcr->ci)
438	pr_crit("missing existing EQCR completions\n");
439	if (eqcr->ci != eqcr_ptr2idx(eqcr->cursor))
440	pr_crit("EQCR destroyed unquiesced\n");
441	}
442
443	static inline struct qm_eqcr_entry *qm_eqcr_start_no_stash(struct qm_portal
444	*portal)
445	{
446	struct qm_eqcr *eqcr = &portal->eqcr;
447
448	DPAA_ASSERT(!eqcr->busy);
449	if (!eqcr->available)
450	return NULL;
451
452	#ifdef CONFIG_FSL_DPAA_CHECKING
453	eqcr->busy = 1;
454	#endif
455	dpaa_zero(eqcr->cursor);
456	return eqcr->cursor;
457	}
458
459	static inline struct qm_eqcr_entry *qm_eqcr_start_stash(struct qm_portal
460	*portal)
461	{
462	struct qm_eqcr *eqcr = &portal->eqcr;
463	u8 diff, old_ci;
464
465	DPAA_ASSERT(!eqcr->busy);
466	if (!eqcr->available) {
467	old_ci = eqcr->ci;
468	eqcr->ci = qm_ce_in(portal, QM_CL_EQCR_CI_CENA) &
469	(QM_EQCR_SIZE - 1);
470	diff = dpaa_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
471	eqcr->available += diff;
472	if (!diff)
473	return NULL;
474	}
475	#ifdef CONFIG_FSL_DPAA_CHECKING
476	eqcr->busy = 1;
477	#endif
478	dpaa_zero(eqcr->cursor);
479	return eqcr->cursor;
480	}
481
482	static inline void eqcr_commit_checks(struct qm_eqcr *eqcr)
483	{
484	DPAA_ASSERT(eqcr->busy);
485	DPAA_ASSERT(!(be32_to_cpu(eqcr->cursor->fqid) & ~QM_FQID_MASK));
486	DPAA_ASSERT(eqcr->available >= 1);
487	}
488
489	static inline void qm_eqcr_pvb_commit(struct qm_portal *portal, u8 myverb)
490	{
491	struct qm_eqcr *eqcr = &portal->eqcr;
492	struct qm_eqcr_entry *eqcursor;
493
494	eqcr_commit_checks(eqcr);
495	DPAA_ASSERT(eqcr->pmode == qm_eqcr_pvb);
496	dma_wmb();
497	eqcursor = eqcr->cursor;
498	eqcursor->_ncw_verb = myverb \| eqcr->vbit;
499	dpaa_flush(eqcursor);
500	eqcr_inc(eqcr);
501	eqcr->available--;
502	#ifdef CONFIG_FSL_DPAA_CHECKING
503	eqcr->busy = 0;
504	#endif
505	}
506
507	static inline void qm_eqcr_cce_prefetch(struct qm_portal *portal)
508	{
509	qm_cl_touch_ro(portal, QM_CL_EQCR_CI_CENA);
510	}
511
512	static inline u8 qm_eqcr_cce_update(struct qm_portal *portal)
513	{
514	struct qm_eqcr *eqcr = &portal->eqcr;
515	u8 diff, old_ci = eqcr->ci;
516
517	eqcr->ci = qm_ce_in(portal, QM_CL_EQCR_CI_CENA) & (QM_EQCR_SIZE - 1);
518	qm_cl_invalidate(portal, QM_CL_EQCR_CI_CENA);
519	diff = dpaa_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
520	eqcr->available += diff;
521	return diff;
522	}
523
524	static inline void qm_eqcr_set_ithresh(struct qm_portal *portal, u8 ithresh)
525	{
526	struct qm_eqcr *eqcr = &portal->eqcr;
527
528	eqcr->ithresh = ithresh;
529	qm_out(portal, QM_REG_EQCR_ITR, ithresh);
530	}
531
532	static inline u8 qm_eqcr_get_avail(struct qm_portal *portal)
533	{
534	struct qm_eqcr *eqcr = &portal->eqcr;
535
536	return eqcr->available;
537	}
538
539	static inline u8 qm_eqcr_get_fill(struct qm_portal *portal)
540	{
541	struct qm_eqcr *eqcr = &portal->eqcr;
542
543	return QM_EQCR_SIZE - 1 - eqcr->available;
544	}
545
546	/* --- DQRR API --- */
547
548	#define DQRR_SHIFT ilog2(sizeof(struct qm_dqrr_entry))
549	#define DQRR_CARRY (uintptr_t)(QM_DQRR_SIZE << DQRR_SHIFT)
550
551	static const struct qm_dqrr_entry *dqrr_carryclear(
552	const struct qm_dqrr_entry *p)
553	{
554	uintptr_t addr = (uintptr_t)p;
555
556	addr &= ~DQRR_CARRY;
557
558	return (const struct qm_dqrr_entry *)addr;
559	}
560
561	static inline int dqrr_ptr2idx(const struct qm_dqrr_entry *e)
562	{
563	return ((uintptr_t)e >> DQRR_SHIFT) & (QM_DQRR_SIZE - 1);
564	}
565
566	static const struct qm_dqrr_entry dqrr_inc(const struct qm_dqrr_entry e)
567	{
568	return dqrr_carryclear(e + 1);
569	}
570
571	static inline void qm_dqrr_set_maxfill(struct qm_portal *portal, u8 mf)
572	{
573	qm_out(portal, QM_REG_CFG, (qm_in(portal, QM_REG_CFG) & 0xff0fffff) \|
574	((mf & (QM_DQRR_SIZE - 1)) << 20));
575	}
576
577	static inline int qm_dqrr_init(struct qm_portal *portal,
578	const struct qm_portal_config *config,
579	enum qm_dqrr_dmode dmode,
580	enum qm_dqrr_pmode pmode,
581	enum qm_dqrr_cmode cmode, u8 max_fill)
582	{
583	struct qm_dqrr *dqrr = &portal->dqrr;
584	u32 cfg;
585
586	/* Make sure the DQRR will be idle when we enable */
587	qm_out(portal, QM_REG_DQRR_SDQCR, 0);
588	qm_out(portal, QM_REG_DQRR_VDQCR, 0);
589	qm_out(portal, QM_REG_DQRR_PDQCR, 0);
590	dqrr->ring = portal->addr.ce + QM_CL_DQRR;
591	dqrr->pi = qm_in(portal, QM_REG_DQRR_PI_CINH) & (QM_DQRR_SIZE - 1);
592	dqrr->ci = qm_in(portal, QM_REG_DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
593	dqrr->cursor = dqrr->ring + dqrr->ci;
594	dqrr->fill = dpaa_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi);
595	dqrr->vbit = (qm_in(portal, QM_REG_DQRR_PI_CINH) & QM_DQRR_SIZE) ?
596	QM_DQRR_VERB_VBIT : 0;
597	dqrr->ithresh = qm_in(portal, QM_REG_DQRR_ITR);
598	#ifdef CONFIG_FSL_DPAA_CHECKING
599	dqrr->dmode = dmode;
600	dqrr->pmode = pmode;
601	dqrr->cmode = cmode;
602	#endif
603	/* Invalidate every ring entry before beginning */
604	for (cfg = 0; cfg < QM_DQRR_SIZE; cfg++)
605	dpaa_invalidate(qm_cl(dqrr->ring, cfg));
606	cfg = (qm_in(portal, QM_REG_CFG) & 0xff000f00) \|
607	((max_fill & (QM_DQRR_SIZE - 1)) << 20) \| /* DQRR_MF */
608	((dmode & 1) << 18) \| /* DP */
609	((cmode & 3) << 16) \| /* DCM */
610	#ifndef CONFIG_FSL_PAMU
611	0xa0 \| /* RE+SE */
612	#endif
613	(0 ? 0x40 : 0) \| /* Ignore RP */
614	(0 ? 0x10 : 0); /* Ignore SP */
615	qm_out(portal, QM_REG_CFG, cfg);
616	qm_dqrr_set_maxfill(portal, max_fill);
617	return 0;
618	}
619
620	static inline void qm_dqrr_finish(struct qm_portal *portal)
621	{
622	#ifdef CONFIG_FSL_DPAA_CHECKING
623	struct qm_dqrr *dqrr = &portal->dqrr;
624
625	if (dqrr->cmode != qm_dqrr_cdc &&
626	dqrr->ci != dqrr_ptr2idx(dqrr->cursor))
627	pr_crit("Ignoring completed DQRR entries\n");
628	#endif
629	}
630
631	static inline const struct qm_dqrr_entry *qm_dqrr_current(
632	struct qm_portal *portal)
633	{
634	struct qm_dqrr *dqrr = &portal->dqrr;
635
636	if (!dqrr->fill)
637	return NULL;
638	return dqrr->cursor;
639	}
640
641	static inline u8 qm_dqrr_next(struct qm_portal *portal)
642	{
643	struct qm_dqrr *dqrr = &portal->dqrr;
644
645	DPAA_ASSERT(dqrr->fill);
646	dqrr->cursor = dqrr_inc(dqrr->cursor);
647	return --dqrr->fill;
648	}
649
650	static inline void qm_dqrr_pvb_update(struct qm_portal *portal)
651	{
652	struct qm_dqrr *dqrr = &portal->dqrr;
653	struct qm_dqrr_entry *res = qm_cl(dqrr->ring, dqrr->pi);
654
655	DPAA_ASSERT(dqrr->pmode == qm_dqrr_pvb);
656	#ifndef CONFIG_FSL_PAMU
657	/*
658	* If PAMU is not available we need to invalidate the cache.
659	* When PAMU is available the cache is updated by stash
660	*/
661	dpaa_invalidate_touch_ro(res);
662	#endif
663	/*
664	* when accessing 'verb', use __raw_readb() to ensure that compiler
665	* inlining doesn't try to optimise out "excess reads".
666	*/
667	if ((__raw_readb(&res->verb) & QM_DQRR_VERB_VBIT) == dqrr->vbit) {
668	dqrr->pi = (dqrr->pi + 1) & (QM_DQRR_SIZE - 1);
669	if (!dqrr->pi)
670	dqrr->vbit ^= QM_DQRR_VERB_VBIT;
671	dqrr->fill++;
672	}
673	}
674
675	static inline void qm_dqrr_cdc_consume_1ptr(struct qm_portal *portal,
676	const struct qm_dqrr_entry *dq,
677	int park)
678	{
679	__maybe_unused struct qm_dqrr *dqrr = &portal->dqrr;
680	int idx = dqrr_ptr2idx(dq);
681
682	DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
683	DPAA_ASSERT((dqrr->ring + idx) == dq);
684	DPAA_ASSERT(idx < QM_DQRR_SIZE);
685	qm_out(portal, QM_REG_DQRR_DCAP, (0 << 8) \| /* DQRR_DCAP::S */
686	((park ? 1 : 0) << 6) \| /* DQRR_DCAP::PK */
687	idx); /* DQRR_DCAP::DCAP_CI */
688	}
689
690	static inline void qm_dqrr_cdc_consume_n(struct qm_portal *portal, u32 bitmask)
691	{
692	__maybe_unused struct qm_dqrr *dqrr = &portal->dqrr;
693
694	DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
695	qm_out(portal, QM_REG_DQRR_DCAP, (1 << 8) \| /* DQRR_DCAP::S */
696	(bitmask << 16)); /* DQRR_DCAP::DCAP_CI */
697	}
698
699	static inline void qm_dqrr_sdqcr_set(struct qm_portal *portal, u32 sdqcr)
700	{
701	qm_out(portal, QM_REG_DQRR_SDQCR, sdqcr);
702	}
703
704	static inline void qm_dqrr_vdqcr_set(struct qm_portal *portal, u32 vdqcr)
705	{
706	qm_out(portal, QM_REG_DQRR_VDQCR, vdqcr);
707	}
708
709	static inline void qm_dqrr_set_ithresh(struct qm_portal *portal, u8 ithresh)
710	{
711	qm_out(portal, QM_REG_DQRR_ITR, ithresh);
712	}
713
714	/* --- MR API --- */
715
716	#define MR_SHIFT ilog2(sizeof(union qm_mr_entry))
717	#define MR_CARRY (uintptr_t)(QM_MR_SIZE << MR_SHIFT)
718
719	static union qm_mr_entry mr_carryclear(union qm_mr_entry p)
720	{
721	uintptr_t addr = (uintptr_t)p;
722
723	addr &= ~MR_CARRY;
724
725	return (union qm_mr_entry *)addr;
726	}
727
728	static inline int mr_ptr2idx(const union qm_mr_entry *e)
729	{
730	return ((uintptr_t)e >> MR_SHIFT) & (QM_MR_SIZE - 1);
731	}
732
733	static inline union qm_mr_entry mr_inc(union qm_mr_entry e)
734	{
735	return mr_carryclear(e + 1);
736	}
737
738	static inline int qm_mr_init(struct qm_portal *portal, enum qm_mr_pmode pmode,
739	enum qm_mr_cmode cmode)
740	{
741	struct qm_mr *mr = &portal->mr;
742	u32 cfg;
743
744	mr->ring = portal->addr.ce + QM_CL_MR;
745	mr->pi = qm_in(portal, QM_REG_MR_PI_CINH) & (QM_MR_SIZE - 1);
746	mr->ci = qm_in(portal, QM_REG_MR_CI_CINH) & (QM_MR_SIZE - 1);
747	mr->cursor = mr->ring + mr->ci;
748	mr->fill = dpaa_cyc_diff(QM_MR_SIZE, mr->ci, mr->pi);
749	mr->vbit = (qm_in(portal, QM_REG_MR_PI_CINH) & QM_MR_SIZE)
750	? QM_MR_VERB_VBIT : 0;
751	mr->ithresh = qm_in(portal, QM_REG_MR_ITR);
752	#ifdef CONFIG_FSL_DPAA_CHECKING
753	mr->pmode = pmode;
754	mr->cmode = cmode;
755	#endif
756	cfg = (qm_in(portal, QM_REG_CFG) & 0xfffff0ff) \|
757	((cmode & 1) << 8); /* QCSP_CFG:MM */
758	qm_out(portal, QM_REG_CFG, cfg);
759	return 0;
760	}
761
762	static inline void qm_mr_finish(struct qm_portal *portal)
763	{
764	struct qm_mr *mr = &portal->mr;
765
766	if (mr->ci != mr_ptr2idx(mr->cursor))
767	pr_crit("Ignoring completed MR entries\n");
768	}
769
770	static inline const union qm_mr_entry qm_mr_current(struct qm_portal portal)
771	{
772	struct qm_mr *mr = &portal->mr;
773
774	if (!mr->fill)
775	return NULL;
776	return mr->cursor;
777	}
778
779	static inline int qm_mr_next(struct qm_portal *portal)
780	{
781	struct qm_mr *mr = &portal->mr;
782
783	DPAA_ASSERT(mr->fill);
784	mr->cursor = mr_inc(mr->cursor);
785	return --mr->fill;
786	}
787
788	static inline void qm_mr_pvb_update(struct qm_portal *portal)
789	{
790	struct qm_mr *mr = &portal->mr;
791	union qm_mr_entry *res = qm_cl(mr->ring, mr->pi);
792
793	DPAA_ASSERT(mr->pmode == qm_mr_pvb);
794	/*
795	* when accessing 'verb', use __raw_readb() to ensure that compiler
796	* inlining doesn't try to optimise out "excess reads".
797	*/
798	if ((__raw_readb(&res->verb) & QM_MR_VERB_VBIT) == mr->vbit) {
799	mr->pi = (mr->pi + 1) & (QM_MR_SIZE - 1);
800	if (!mr->pi)
801	mr->vbit ^= QM_MR_VERB_VBIT;
802	mr->fill++;
803	res = mr_inc(res);
804	}
805	dpaa_invalidate_touch_ro(res);
806	}
807
808	static inline void qm_mr_cci_consume(struct qm_portal *portal, u8 num)
809	{
810	struct qm_mr *mr = &portal->mr;
811
812	DPAA_ASSERT(mr->cmode == qm_mr_cci);
813	mr->ci = (mr->ci + num) & (QM_MR_SIZE - 1);
814	qm_out(portal, QM_REG_MR_CI_CINH, mr->ci);
815	}
816
817	static inline void qm_mr_cci_consume_to_current(struct qm_portal *portal)
818	{
819	struct qm_mr *mr = &portal->mr;
820
821	DPAA_ASSERT(mr->cmode == qm_mr_cci);
822	mr->ci = mr_ptr2idx(mr->cursor);
823	qm_out(portal, QM_REG_MR_CI_CINH, mr->ci);
824	}
825
826	static inline void qm_mr_set_ithresh(struct qm_portal *portal, u8 ithresh)
827	{
828	qm_out(portal, QM_REG_MR_ITR, ithresh);
829	}
830
831	/* --- Management command API --- */
832
833	static inline int qm_mc_init(struct qm_portal *portal)
834	{
835	struct qm_mc *mc = &portal->mc;
836
837	mc->cr = portal->addr.ce + QM_CL_CR;
838	mc->rr = portal->addr.ce + QM_CL_RR0;
839	mc->rridx = (__raw_readb(&mc->cr->_ncw_verb) & QM_MCC_VERB_VBIT)
840	? 0 : 1;
841	mc->vbit = mc->rridx ? QM_MCC_VERB_VBIT : 0;
842	#ifdef CONFIG_FSL_DPAA_CHECKING
843	mc->state = qman_mc_idle;
844	#endif
845	return 0;
846	}
847
848	static inline void qm_mc_finish(struct qm_portal *portal)
849	{
850	#ifdef CONFIG_FSL_DPAA_CHECKING
851	struct qm_mc *mc = &portal->mc;
852
853	DPAA_ASSERT(mc->state == qman_mc_idle);
854	if (mc->state != qman_mc_idle)
855	pr_crit("Losing incomplete MC command\n");
856	#endif
857	}
858
859	static inline union qm_mc_command qm_mc_start(struct qm_portal portal)
860	{
861	struct qm_mc *mc = &portal->mc;
862
863	DPAA_ASSERT(mc->state == qman_mc_idle);
864	#ifdef CONFIG_FSL_DPAA_CHECKING
865	mc->state = qman_mc_user;
866	#endif
867	dpaa_zero(mc->cr);
868	return mc->cr;
869	}
870
871	static inline void qm_mc_commit(struct qm_portal *portal, u8 myverb)
872	{
873	struct qm_mc *mc = &portal->mc;
874	union qm_mc_result *rr = mc->rr + mc->rridx;
875
876	DPAA_ASSERT(mc->state == qman_mc_user);
877	dma_wmb();
878	mc->cr->_ncw_verb = myverb \| mc->vbit;
879	dpaa_flush(mc->cr);
880	dpaa_invalidate_touch_ro(rr);
881	#ifdef CONFIG_FSL_DPAA_CHECKING
882	mc->state = qman_mc_hw;
883	#endif
884	}
885
886	static inline union qm_mc_result qm_mc_result(struct qm_portal portal)
887	{
888	struct qm_mc *mc = &portal->mc;
889	union qm_mc_result *rr = mc->rr + mc->rridx;
890
891	DPAA_ASSERT(mc->state == qman_mc_hw);
892	/*
893	* The inactive response register's verb byte always returns zero until
894	* its command is submitted and completed. This includes the valid-bit,
895	* in case you were wondering...
896	*/
897	if (!__raw_readb(&rr->verb)) {
898	dpaa_invalidate_touch_ro(rr);
899	return NULL;
900	}
901	mc->rridx ^= 1;
902	mc->vbit ^= QM_MCC_VERB_VBIT;
903	#ifdef CONFIG_FSL_DPAA_CHECKING
904	mc->state = qman_mc_idle;
905	#endif
906	return rr;
907	}
908
909	static inline int qm_mc_result_timeout(struct qm_portal *portal,
910	union qm_mc_result **mcr)
911	{
912	int timeout = QM_MCR_TIMEOUT;
913
914	do {
915	*mcr = qm_mc_result(portal);
916	if (*mcr)
917	break;
918	udelay(1);
919	} while (--timeout);
920
921	return timeout;
922	}
923
924	static inline void fq_set(struct qman_fq *fq, u32 mask)
925	{
926	set_bits(mask, &fq->flags);
927	}
928
929	static inline void fq_clear(struct qman_fq *fq, u32 mask)
930	{
931	clear_bits(mask, &fq->flags);
932	}
933
934	static inline int fq_isset(struct qman_fq *fq, u32 mask)
935	{
936	return fq->flags & mask;
937	}
938
939	static inline int fq_isclear(struct qman_fq *fq, u32 mask)
940	{
941	return !(fq->flags & mask);
942	}
943
944	struct qman_portal {
945	struct qm_portal p;
946	/* PORTAL_BITS_*** - dynamic, strictly internal */
947	unsigned long bits;
948	/* interrupt sources processed by portal_isr(), configurable */
949	unsigned long irq_sources;
950	u32 use_eqcr_ci_stashing;
951	/* only 1 volatile dequeue at a time */
952	struct qman_fq *vdqcr_owned;
953	u32 sdqcr;
954	/* probing time config params for cpu-affine portals */
955	const struct qm_portal_config *config;
956	/* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */
957	struct qman_cgrs *cgrs;
958	/* linked-list of CSCN handlers. */
959	struct list_head cgr_cbs;
960	/* list lock */
961	spinlock_t cgr_lock;
962	#ifndef __rtems__
963	struct work_struct congestion_work;
964	struct work_struct mr_work;
965	#endif /* __rtems__ */
966	char irqname[MAX_IRQNAME];
967	};
968
969	#ifndef __rtems__
970	static cpumask_t affine_mask;
971	static DEFINE_SPINLOCK(affine_mask_lock);
972	static u16 affine_channels[NR_CPUS];
973	#endif /* __rtems__ */
974	static DEFINE_PER_CPU(struct qman_portal, qman_affine_portal);
975	struct qman_portal *affine_portals[NR_CPUS];
976
977	static inline struct qman_portal *get_affine_portal(void)
978	{
979	return &get_cpu_var(qman_affine_portal);
980	}
981
982	static inline void put_affine_portal(void)
983	{
984	put_cpu_var(qman_affine_portal);
985	}
986
987	#ifndef __rtems__
988	static struct workqueue_struct *qm_portal_wq;
989	#endif /* __rtems__ */
990
991	int qman_wq_alloc(void)
992	{
993	#ifndef __rtems__
994	qm_portal_wq = alloc_workqueue("qman_portal_wq", 0, 1);
995	if (!qm_portal_wq)
996	return -ENOMEM;
997	#endif /* __rtems__ */
998	return 0;
999	}
1000
1001	/*
1002	* This is what everything can wait on, even if it migrates to a different cpu
1003	* to the one whose affine portal it is waiting on.
1004	*/
1005	static DECLARE_WAIT_QUEUE_HEAD(affine_queue);
1006
1007	static struct qman_fq **fq_table;
1008	static u32 num_fqids;
1009
1010	int qman_alloc_fq_table(u32 _num_fqids)
1011	{
1012	num_fqids = _num_fqids;
1013
1014	fq_table = vzalloc(num_fqids * 2 * sizeof(struct qman_fq *));
1015	if (!fq_table)
1016	return -ENOMEM;
1017
1018	pr_debug("Allocated fq lookup table at %p, entry count %u\n",
1019	fq_table, num_fqids * 2);
1020	return 0;
1021	}
1022
1023	static struct qman_fq *idx_to_fq(u32 idx)
1024	{
1025	struct qman_fq *fq;
1026
1027	#ifdef CONFIG_FSL_DPAA_CHECKING
1028	if (WARN_ON(idx >= num_fqids * 2))
1029	return NULL;
1030	#endif
1031	fq = fq_table[idx];
1032	DPAA_ASSERT(!fq \|\| idx == fq->idx);
1033
1034	return fq;
1035	}
1036
1037	/*
1038	* Only returns full-service fq objects, not enqueue-only
1039	* references (QMAN_FQ_FLAG_NO_MODIFY).
1040	*/
1041	static struct qman_fq *fqid_to_fq(u32 fqid)
1042	{
1043	return idx_to_fq(fqid * 2);
1044	}
1045
1046	static struct qman_fq *tag_to_fq(u32 tag)
1047	{
1048	#if BITS_PER_LONG == 64
1049	return idx_to_fq(tag);
1050	#else
1051	return (struct qman_fq *)tag;
1052	#endif
1053	}
1054
1055	static u32 fq_to_tag(struct qman_fq *fq)
1056	{
1057	#if BITS_PER_LONG == 64
1058	return fq->idx;
1059	#else
1060	return (u32)fq;
1061	#endif
1062	}
1063
1064	static u32 __poll_portal_slow(struct qman_portal *p, u32 is);
1065	static inline unsigned int __poll_portal_fast(struct qman_portal *p,
1066	unsigned int poll_limit);
1067	#ifndef __rtems__
1068	static void qm_congestion_task(struct work_struct *work);
1069	static void qm_mr_process_task(struct work_struct *work);
1070	#endif /* __rtems__ */
1071
1072	static irqreturn_t portal_isr(int irq, void *ptr)
1073	{
1074	struct qman_portal *p = ptr;
1075
1076	u32 clear = QM_DQAVAIL_MASK;
1077	u32 is = qm_in(&p->p, QM_REG_ISR) & p->irq_sources;
1078
1079	if (unlikely(!is))
1080	return IRQ_NONE;
1081
1082	/* DQRR-handling if it's interrupt-driven */
1083	if (is & QM_PIRQ_DQRI) {
1084	clear \|= QM_PIRQ_DQRI;
1085	__poll_portal_fast(p, QMAN_POLL_LIMIT);
1086	}
1087	/* Handling of anything else that's interrupt-driven */
1088	clear \|= __poll_portal_slow(p, is);
1089	qm_out(&p->p, QM_REG_ISR, clear);
1090	return IRQ_HANDLED;
1091	}
1092
1093	static int drain_mr_fqrni(struct qm_portal *p)
1094	{
1095	const union qm_mr_entry *msg;
1096	loop:
1097	msg = qm_mr_current(p);
1098	if (!msg) {
1099	/*
1100	* if MR was full and h/w had other FQRNI entries to produce, we
1101	* need to allow it time to produce those entries once the
1102	* existing entries are consumed. A worst-case situation
1103	* (fully-loaded system) means h/w sequencers may have to do 3-4
1104	* other things before servicing the portal's MR pump, each of
1105	* which (if slow) may take ~50 qman cycles (which is ~200
1106	* processor cycles). So rounding up and then multiplying this
1107	* worst-case estimate by a factor of 10, just to be
1108	* ultra-paranoid, goes as high as 10,000 cycles. NB, we consume
1109	* one entry at a time, so h/w has an opportunity to produce new
1110	* entries well before the ring has been fully consumed, so
1111	* we're being really paranoid here.
1112	*/
1113	u64 now, then = jiffies;
1114
1115	do {
1116	now = jiffies;
1117	} while ((then + 10000) > now);
1118	msg = qm_mr_current(p);
1119	if (!msg)
1120	return 0;
1121	}
1122	if ((msg->verb & QM_MR_VERB_TYPE_MASK) != QM_MR_VERB_FQRNI) {
1123	/* We aren't draining anything but FQRNIs */
1124	pr_err("Found verb 0x%x in MR\n", msg->verb);
1125	return -1;
1126	}
1127	qm_mr_next(p);
1128	qm_mr_cci_consume(p, 1);
1129	goto loop;
1130	}
1131
1132	static int qman_create_portal(struct qman_portal *portal,
1133	const struct qm_portal_config *c,
1134	const struct qman_cgrs *cgrs)
1135	{
1136	struct qm_portal *p;
1137	int ret;
1138	u32 isdr;
1139
1140	p = &portal->p;
1141
1142	#ifdef CONFIG_FSL_PAMU
1143	/* PAMU is required for stashing */
1144	portal->use_eqcr_ci_stashing = ((qman_ip_rev >= QMAN_REV30) ? 1 : 0);
1145	#else
1146	portal->use_eqcr_ci_stashing = 0;
1147	#endif
1148	/*
1149	* prep the low-level portal struct with the mapped addresses from the
1150	* config, everything that follows depends on it and "config" is more
1151	* for (de)reference
1152	*/
1153	p->addr.ce = c->addr_virt[DPAA_PORTAL_CE];
1154	p->addr.ci = c->addr_virt[DPAA_PORTAL_CI];
1155	/*
1156	* If CI-stashing is used, the current defaults use a threshold of 3,
1157	* and stash with high-than-DQRR priority.
1158	*/
1159	if (qm_eqcr_init(p, qm_eqcr_pvb,
1160	portal->use_eqcr_ci_stashing ? 3 : 0, 1)) {
1161	dev_err(c->dev, "EQCR initialisation failed\n");
1162	goto fail_eqcr;
1163	}
1164	if (qm_dqrr_init(p, c, qm_dqrr_dpush, qm_dqrr_pvb,
1165	qm_dqrr_cdc, DQRR_MAXFILL)) {
1166	dev_err(c->dev, "DQRR initialisation failed\n");
1167	goto fail_dqrr;
1168	}
1169	if (qm_mr_init(p, qm_mr_pvb, qm_mr_cci)) {
1170	dev_err(c->dev, "MR initialisation failed\n");
1171	goto fail_mr;
1172	}
1173	if (qm_mc_init(p)) {
1174	dev_err(c->dev, "MC initialisation failed\n");
1175	goto fail_mc;
1176	}
1177	/* static interrupt-gating controls */
1178	qm_dqrr_set_ithresh(p, QMAN_PIRQ_DQRR_ITHRESH);
1179	qm_mr_set_ithresh(p, QMAN_PIRQ_MR_ITHRESH);
1180	qm_out(p, QM_REG_ITPR, QMAN_PIRQ_IPERIOD);
1181	portal->cgrs = kmalloc(2 * sizeof(*cgrs), GFP_KERNEL);
1182	if (!portal->cgrs)
1183	goto fail_cgrs;
1184	/* initial snapshot is no-depletion */
1185	qman_cgrs_init(&portal->cgrs[1]);
1186	if (cgrs)
1187	portal->cgrs[0] = *cgrs;
1188	else
1189	/* if the given mask is NULL, assume all CGRs can be seen */
1190	qman_cgrs_fill(&portal->cgrs[0]);
1191	INIT_LIST_HEAD(&portal->cgr_cbs);
1192	spin_lock_init(&portal->cgr_lock);
1193	#ifndef __rtems__
1194	INIT_WORK(&portal->congestion_work, qm_congestion_task);
1195	INIT_WORK(&portal->mr_work, qm_mr_process_task);
1196	#endif /* __rtems__ */
1197	portal->bits = 0;
1198	portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS \| QM_SDQCR_COUNT_UPTO3 \|
1199	QM_SDQCR_DEDICATED_PRECEDENCE \| QM_SDQCR_TYPE_PRIO_QOS \|
1200	QM_SDQCR_TOKEN_SET(0xab) \| QM_SDQCR_CHANNELS_DEDICATED;
1201	isdr = 0xffffffff;
1202	qm_out(p, QM_REG_ISDR, isdr);
1203	portal->irq_sources = 0;
1204	qm_out(p, QM_REG_IER, 0);
1205	qm_out(p, QM_REG_ISR, 0xffffffff);
1206	snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu);
1207	if (request_irq(c->irq, portal_isr, 0, portal->irqname, portal)) {
1208	dev_err(c->dev, "request_irq() failed\n");
1209	goto fail_irq;
1210	}
1211	#ifndef __rtems__
1212	if (c->cpu != -1 && irq_can_set_affinity(c->irq) &&
1213	irq_set_affinity(c->irq, cpumask_of(c->cpu))) {
1214	dev_err(c->dev, "irq_set_affinity() failed\n");
1215	goto fail_affinity;
1216	}
1217	#endif /* __rtems__ */
1218
1219	/* Need EQCR to be empty before continuing */
1220	isdr &= ~QM_PIRQ_EQCI;
1221	qm_out(p, QM_REG_ISDR, isdr);
1222	ret = qm_eqcr_get_fill(p);
1223	if (ret) {
1224	dev_err(c->dev, "EQCR unclean\n");
1225	goto fail_eqcr_empty;
1226	}
1227	isdr &= ~(QM_PIRQ_DQRI \| QM_PIRQ_MRI);
1228	qm_out(p, QM_REG_ISDR, isdr);
1229	if (qm_dqrr_current(p)) {
1230	dev_err(c->dev, "DQRR unclean\n");
1231	qm_dqrr_cdc_consume_n(p, 0xffff);
1232	}
1233	if (qm_mr_current(p) && drain_mr_fqrni(p)) {
1234	/* special handling, drain just in case it's a few FQRNIs */
1235	const union qm_mr_entry *e = qm_mr_current(p);
1236
1237	dev_err(c->dev, "MR dirty, VB 0x%x, rc 0x%x, addr 0x%llx\n",
1238	e->verb, e->ern.rc, qm_fd_addr_get64(&e->ern.fd));
1239	goto fail_dqrr_mr_empty;
1240	}
1241	/* Success */
1242	portal->config = c;
1243	qm_out(p, QM_REG_ISDR, 0);
1244	qm_out(p, QM_REG_IIR, 0);
1245	/* Write a sane SDQCR */
1246	qm_dqrr_sdqcr_set(p, portal->sdqcr);
1247	return 0;
1248
1249	fail_dqrr_mr_empty:
1250	fail_eqcr_empty:
1251	#ifndef __rtems__
1252	fail_affinity:
1253	#endif /* __rtems__ */
1254	free_irq(c->irq, portal);
1255	fail_irq:
1256	kfree(portal->cgrs);
1257	fail_cgrs:
1258	qm_mc_finish(p);
1259	fail_mc:
1260	qm_mr_finish(p);
1261	fail_mr:
1262	qm_dqrr_finish(p);
1263	fail_dqrr:
1264	qm_eqcr_finish(p);
1265	fail_eqcr:
1266	return -EIO;
1267	}
1268
1269	struct qman_portal qman_create_affine_portal(const struct qm_portal_config c,
1270	const struct qman_cgrs *cgrs)
1271	{
1272	struct qman_portal *portal;
1273	int err;
1274
1275	portal = &per_cpu(qman_affine_portal, c->cpu);
1276	err = qman_create_portal(portal, c, cgrs);
1277	if (err)
1278	return NULL;
1279
1280	#ifndef __rtems__
1281	spin_lock(&affine_mask_lock);
1282	cpumask_set_cpu(c->cpu, &affine_mask);
1283	affine_channels[c->cpu] = c->channel;
1284	#endif /* __rtems__ */
1285	affine_portals[c->cpu] = portal;
1286	#ifndef __rtems__
1287	spin_unlock(&affine_mask_lock);
1288	#endif /* __rtems__ */
1289
1290	return portal;
1291	}
1292
1293	static void qman_destroy_portal(struct qman_portal *qm)
1294	{
1295	const struct qm_portal_config *pcfg;
1296
1297	/* Stop dequeues on the portal */
1298	qm_dqrr_sdqcr_set(&qm->p, 0);
1299
1300	/*
1301	* NB we do this to "quiesce" EQCR. If we add enqueue-completions or
1302	* something related to QM_PIRQ_EQCI, this may need fixing.
1303	* Also, due to the prefetching model used for CI updates in the enqueue
1304	* path, this update will only invalidate the CI cacheline after
1305	* working on it, so we need to call this twice to ensure a full update
1306	* irrespective of where the enqueue processing was at when the teardown
1307	* began.
1308	*/
1309	qm_eqcr_cce_update(&qm->p);
1310	qm_eqcr_cce_update(&qm->p);
1311	pcfg = qm->config;
1312
1313	free_irq(pcfg->irq, qm);
1314
1315	kfree(qm->cgrs);
1316	qm_mc_finish(&qm->p);
1317	qm_mr_finish(&qm->p);
1318	qm_dqrr_finish(&qm->p);
1319	qm_eqcr_finish(&qm->p);
1320
1321	qm->config = NULL;
1322	}
1323
1324	const struct qm_portal_config *qman_destroy_affine_portal(void)
1325	{
1326	struct qman_portal *qm = get_affine_portal();
1327	const struct qm_portal_config *pcfg;
1328	int cpu;
1329
1330	pcfg = qm->config;
1331	cpu = pcfg->cpu;
1332
1333	qman_destroy_portal(qm);
1334
1335	#ifndef __rtems__
1336	spin_lock(&affine_mask_lock);
1337	cpumask_clear_cpu(cpu, &affine_mask);
1338	spin_unlock(&affine_mask_lock);
1339	#else /* __rtems__ */
1340	(void)cpu;
1341	#endif /* __rtems__ */
1342	put_affine_portal();
1343	return pcfg;
1344	}
1345
1346	/* Inline helper to reduce nesting in __poll_portal_slow() */
1347	static inline void fq_state_change(struct qman_portal p, struct qman_fq fq,
1348	const union qm_mr_entry *msg, u8 verb)
1349	{
1350	switch (verb) {
1351	case QM_MR_VERB_FQRL:
1352	DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_ORL));
1353	fq_clear(fq, QMAN_FQ_STATE_ORL);
1354	break;
1355	case QM_MR_VERB_FQRN:
1356	DPAA_ASSERT(fq->state == qman_fq_state_parked \|\|
1357	fq->state == qman_fq_state_sched);
1358	DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_CHANGING));
1359	fq_clear(fq, QMAN_FQ_STATE_CHANGING);
1360	if (msg->fq.fqs & QM_MR_FQS_NOTEMPTY)
1361	fq_set(fq, QMAN_FQ_STATE_NE);
1362	if (msg->fq.fqs & QM_MR_FQS_ORLPRESENT)
1363	fq_set(fq, QMAN_FQ_STATE_ORL);
1364	fq->state = qman_fq_state_retired;
1365	break;
1366	case QM_MR_VERB_FQPN:
1367	DPAA_ASSERT(fq->state == qman_fq_state_sched);
1368	DPAA_ASSERT(fq_isclear(fq, QMAN_FQ_STATE_CHANGING));
1369	fq->state = qman_fq_state_parked;
1370	}
1371	}
1372
1373	#ifndef __rtems__
1374	static void qm_congestion_task(struct work_struct *work)
1375	{
1376	struct qman_portal *p = container_of(work, struct qman_portal,
1377	congestion_work);
1378	#else /* __rtems__ */
1379	static void qm_congestion_task(struct qman_portal *p)
1380	{
1381	#endif /* __rtems__ */
1382	struct qman_cgrs rr, c;
1383	union qm_mc_result *mcr;
1384	struct qman_cgr *cgr;
1385
1386	spin_lock(&p->cgr_lock);
1387	qm_mc_start(&p->p);
1388	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION);
1389	if (!qm_mc_result_timeout(&p->p, &mcr)) {
1390	spin_unlock(&p->cgr_lock);
1391	dev_crit(p->config->dev, "QUERYCONGESTION timeout\n");
1392	qman_p_irqsource_add(p, QM_PIRQ_CSCI);
1393	return;
1394	}
1395	/* mask out the ones I'm not interested in */
1396	qman_cgrs_and(&rr, (struct qman_cgrs *)&mcr->querycongestion.state,
1397	&p->cgrs[0]);
1398	/* check previous snapshot for delta, enter/exit congestion */
1399	qman_cgrs_xor(&c, &rr, &p->cgrs[1]);
1400	/* update snapshot */
1401	qman_cgrs_cp(&p->cgrs[1], &rr);
1402	/* Invoke callback */
1403	list_for_each_entry(cgr, &p->cgr_cbs, node)
1404	if (cgr->cb && qman_cgrs_get(&c, cgr->cgrid))
1405	cgr->cb(p, cgr, qman_cgrs_get(&rr, cgr->cgrid));
1406	spin_unlock(&p->cgr_lock);
1407	qman_p_irqsource_add(p, QM_PIRQ_CSCI);
1408	}
1409
1410	#ifndef __rtems__
1411	static void qm_mr_process_task(struct work_struct *work)
1412	{
1413	struct qman_portal *p = container_of(work, struct qman_portal,
1414	mr_work);
1415	#else /* __rtems__ */
1416	static void qm_mr_process_task(struct qman_portal *p)
1417	{
1418	#endif /* __rtems__ */
1419	const union qm_mr_entry *msg;
1420	struct qman_fq *fq;
1421	u8 verb, num = 0;
1422
1423	#ifndef __rtems__
1424	preempt_disable();
1425	#endif /* __rtems__ */
1426
1427	while (1) {
1428	qm_mr_pvb_update(&p->p);
1429	msg = qm_mr_current(&p->p);
1430	if (!msg)
1431	break;
1432
1433	verb = msg->verb & QM_MR_VERB_TYPE_MASK;
1434	/* The message is a software ERN iff the 0x20 bit is clear */
1435	if (verb & 0x20) {
1436	switch (verb) {
1437	case QM_MR_VERB_FQRNI:
1438	/* nada, we drop FQRNIs on the floor */
1439	break;
1440	case QM_MR_VERB_FQRN:
1441	case QM_MR_VERB_FQRL:
1442	/* Lookup in the retirement table */
1443	fq = fqid_to_fq(qm_fqid_get(&msg->fq));
1444	if (WARN_ON(!fq))
1445	break;
1446	fq_state_change(p, fq, msg, verb);
1447	if (fq->cb.fqs)
1448	fq->cb.fqs(p, fq, msg);
1449	break;
1450	case QM_MR_VERB_FQPN:
1451	/* Parked */
1452	fq = tag_to_fq(be32_to_cpu(msg->fq.context_b));
1453	fq_state_change(p, fq, msg, verb);
1454	if (fq->cb.fqs)
1455	fq->cb.fqs(p, fq, msg);
1456	break;
1457	case QM_MR_VERB_DC_ERN:
1458	/* DCP ERN */
1459	pr_crit_once("Leaking DCP ERNs!\n");
1460	break;
1461	default:
1462	pr_crit("Invalid MR verb 0x%02x\n", verb);
1463	}
1464	} else {
1465	/* Its a software ERN */
1466	fq = tag_to_fq(be32_to_cpu(msg->ern.tag));
1467	fq->cb.ern(p, fq, msg);
1468	}
1469	num++;
1470	qm_mr_next(&p->p);
1471	}
1472
1473	qm_mr_cci_consume(&p->p, num);
1474	qman_p_irqsource_add(p, QM_PIRQ_MRI);
1475	#ifndef __rtems__
1476	preempt_enable();
1477	#endif /* __rtems__ */
1478	}
1479
1480	static u32 __poll_portal_slow(struct qman_portal *p, u32 is)
1481	{
1482	if (is & QM_PIRQ_CSCI) {
1483	qman_p_irqsource_remove(p, QM_PIRQ_CSCI);
1484	#ifndef __rtems__
1485	queue_work_on(smp_processor_id(), qm_portal_wq,
1486	&p->congestion_work);
1487	#else /* __rtems__ */
1488	qm_congestion_task(p);
1489	#endif /* __rtems__ */
1490	}
1491
1492	if (is & QM_PIRQ_EQRI) {
1493	qm_eqcr_cce_update(&p->p);
1494	qm_eqcr_set_ithresh(&p->p, 0);
1495	wake_up(&affine_queue);
1496	}
1497
1498	if (is & QM_PIRQ_MRI) {
1499	qman_p_irqsource_remove(p, QM_PIRQ_MRI);
1500	#ifndef __rtems__
1501	queue_work_on(smp_processor_id(), qm_portal_wq,
1502	&p->mr_work);
1503	#else /* __rtems__ */
1504	qm_mr_process_task(p);
1505	#endif /* __rtems__ */
1506	}
1507
1508	return is;
1509	}
1510
1511	/*
1512	* remove some slowish-path stuff from the "fast path" and make sure it isn't
1513	* inlined.
1514	*/
1515	static noinline void clear_vdqcr(struct qman_portal p, struct qman_fq fq)
1516	{
1517	p->vdqcr_owned = NULL;
1518	fq_clear(fq, QMAN_FQ_STATE_VDQCR);
1519	wake_up(&affine_queue);
1520	}
1521
1522	/*
1523	* The only states that would conflict with other things if they ran at the
1524	* same time on the same cpu are:
1525	*
1526	* (i) setting/clearing vdqcr_owned, and
1527	* (ii) clearing the NE (Not Empty) flag.
1528	*
1529	* Both are safe. Because;
1530	*
1531	* (i) this clearing can only occur after qman_volatile_dequeue() has set the
1532	* vdqcr_owned field (which it does before setting VDQCR), and
1533	* qman_volatile_dequeue() blocks interrupts and preemption while this is
1534	* done so that we can't interfere.
1535	* (ii) the NE flag is only cleared after qman_retire_fq() has set it, and as
1536	* with (i) that API prevents us from interfering until it's safe.
1537	*
1538	* The good thing is that qman_volatile_dequeue() and qman_retire_fq() run far
1539	* less frequently (ie. per-FQ) than __poll_portal_fast() does, so the nett
1540	* advantage comes from this function not having to "lock" anything at all.
1541	*
1542	* Note also that the callbacks are invoked at points which are safe against the
1543	* above potential conflicts, but that this function itself is not re-entrant
1544	* (this is because the function tracks one end of each FIFO in the portal and
1545	* we do not want to lock that). So the consequence is that it is safe for
1546	* user callbacks to call into any QMan API.
1547	*/
1548	static inline unsigned int __poll_portal_fast(struct qman_portal *p,
1549	unsigned int poll_limit)
1550	{
1551	const struct qm_dqrr_entry *dq;
1552	struct qman_fq *fq;
1553	enum qman_cb_dqrr_result res;
1554	unsigned int limit = 0;
1555
1556	do {
1557	qm_dqrr_pvb_update(&p->p);
1558	dq = qm_dqrr_current(&p->p);
1559	if (!dq)
1560	break;
1561
1562	if (dq->stat & QM_DQRR_STAT_UNSCHEDULED) {
1563	/*
1564	* VDQCR: don't trust context_b as the FQ may have
1565	* been configured for h/w consumption and we're
1566	* draining it post-retirement.
1567	*/
1568	fq = p->vdqcr_owned;
1569	/*
1570	* We only set QMAN_FQ_STATE_NE when retiring, so we
1571	* only need to check for clearing it when doing
1572	* volatile dequeues. It's one less thing to check
1573	* in the critical path (SDQCR).
1574	*/
1575	if (dq->stat & QM_DQRR_STAT_FQ_EMPTY)
1576	fq_clear(fq, QMAN_FQ_STATE_NE);
1577	/*
1578	* This is duplicated from the SDQCR code, but we
1579	* have stuff to do before and after this callback,
1580	* and we don't want multiple if()s in the critical
1581	* path (SDQCR).
1582	*/
1583	res = fq->cb.dqrr(p, fq, dq);
1584	if (res == qman_cb_dqrr_stop)
1585	break;
1586	/* Check for VDQCR completion */
1587	if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED)
1588	clear_vdqcr(p, fq);
1589	} else {
1590	/* SDQCR: context_b points to the FQ */
1591	fq = tag_to_fq(be32_to_cpu(dq->context_b));
1592	/* Now let the callback do its stuff */
1593	res = fq->cb.dqrr(p, fq, dq);
1594	/*
1595	* The callback can request that we exit without
1596	* consuming this entry nor advancing;
1597	*/
1598	if (res == qman_cb_dqrr_stop)
1599	break;
1600	}
1601	/* Interpret 'dq' from a driver perspective. */
1602	/*
1603	* Parking isn't possible unless HELDACTIVE was set. NB,
1604	* FORCEELIGIBLE implies HELDACTIVE, so we only need to
1605	* check for HELDACTIVE to cover both.
1606	*/
1607	DPAA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) \|\|
1608	(res != qman_cb_dqrr_park));
1609	/* just means "skip it, I'll consume it myself later on" */
1610	if (res != qman_cb_dqrr_defer)
1611	qm_dqrr_cdc_consume_1ptr(&p->p, dq,
1612	res == qman_cb_dqrr_park);
1613	/* Move forward */
1614	qm_dqrr_next(&p->p);
1615	/*
1616	* Entry processed and consumed, increment our counter. The
1617	* callback can request that we exit after consuming the
1618	* entry, and we also exit if we reach our processing limit,
1619	* so loop back only if neither of these conditions is met.
1620	*/
1621	} while (++limit < poll_limit && res != qman_cb_dqrr_consume_stop);
1622
1623	return limit;
1624	}
1625
1626	void qman_p_irqsource_add(struct qman_portal *p, u32 bits)
1627	{
1628	unsigned long irqflags;
1629
1630	local_irq_save(irqflags);
1631	set_bits(bits & QM_PIRQ_VISIBLE, &p->irq_sources);
1632	qm_out(&p->p, QM_REG_IER, p->irq_sources);
1633	local_irq_restore(irqflags);
1634	}
1635	EXPORT_SYMBOL(qman_p_irqsource_add);
1636
1637	void qman_p_irqsource_remove(struct qman_portal *p, u32 bits)
1638	{
1639	unsigned long irqflags;
1640	u32 ier;
1641
1642	/*
1643	* Our interrupt handler only processes+clears status register bits that
1644	* are in p->irq_sources. As we're trimming that mask, if one of them
1645	* were to assert in the status register just before we remove it from
1646	* the enable register, there would be an interrupt-storm when we
1647	* release the IRQ lock. So we wait for the enable register update to
1648	* take effect in h/w (by reading it back) and then clear all other bits
1649	* in the status register. Ie. we clear them from ISR once it's certain
1650	* IER won't allow them to reassert.
1651	*/
1652	local_irq_save(irqflags);
1653	bits &= QM_PIRQ_VISIBLE;
1654	clear_bits(bits, &p->irq_sources);
1655	qm_out(&p->p, QM_REG_IER, p->irq_sources);
1656	ier = qm_in(&p->p, QM_REG_IER);
1657	/*
1658	* Using "~ier" (rather than "bits" or "~p->irq_sources") creates a
1659	* data-dependency, ie. to protect against re-ordering.
1660	*/
1661	qm_out(&p->p, QM_REG_ISR, ~ier);
1662	local_irq_restore(irqflags);
1663	}
1664	EXPORT_SYMBOL(qman_p_irqsource_remove);
1665
1666	#ifndef __rtems__
1667	const cpumask_t *qman_affine_cpus(void)
1668	{
1669	return &affine_mask;
1670	}
1671	EXPORT_SYMBOL(qman_affine_cpus);
1672
1673	u16 qman_affine_channel(int cpu)
1674	{
1675	if (cpu < 0) {
1676	struct qman_portal *portal = get_affine_portal();
1677
1678	cpu = portal->config->cpu;
1679	put_affine_portal();
1680	}
1681	WARN_ON(!cpumask_test_cpu(cpu, &affine_mask));
1682	return affine_channels[cpu];
1683	}
1684	EXPORT_SYMBOL(qman_affine_channel);
1685	#endif /* __rtems__ */
1686
1687	struct qman_portal *qman_get_affine_portal(int cpu)
1688	{
1689	return affine_portals[cpu];
1690	}
1691	EXPORT_SYMBOL(qman_get_affine_portal);
1692
1693	int qman_p_poll_dqrr(struct qman_portal *p, unsigned int limit)
1694	{
1695	return __poll_portal_fast(p, limit);
1696	}
1697	EXPORT_SYMBOL(qman_p_poll_dqrr);
1698
1699	void qman_p_static_dequeue_add(struct qman_portal *p, u32 pools)
1700	{
1701	unsigned long irqflags;
1702
1703	local_irq_save(irqflags);
1704	pools &= p->config->pools;
1705	p->sdqcr \|= pools;
1706	qm_dqrr_sdqcr_set(&p->p, p->sdqcr);
1707	local_irq_restore(irqflags);
1708	}
1709	EXPORT_SYMBOL(qman_p_static_dequeue_add);
1710
1711	/* Frame queue API */
1712
1713	static const char *mcr_result_str(u8 result)
1714	{
1715	switch (result) {
1716	case QM_MCR_RESULT_NULL:
1717	return "QM_MCR_RESULT_NULL";
1718	case QM_MCR_RESULT_OK:
1719	return "QM_MCR_RESULT_OK";
1720	case QM_MCR_RESULT_ERR_FQID:
1721	return "QM_MCR_RESULT_ERR_FQID";
1722	case QM_MCR_RESULT_ERR_FQSTATE:
1723	return "QM_MCR_RESULT_ERR_FQSTATE";
1724	case QM_MCR_RESULT_ERR_NOTEMPTY:
1725	return "QM_MCR_RESULT_ERR_NOTEMPTY";
1726	case QM_MCR_RESULT_PENDING:
1727	return "QM_MCR_RESULT_PENDING";
1728	case QM_MCR_RESULT_ERR_BADCOMMAND:
1729	return "QM_MCR_RESULT_ERR_BADCOMMAND";
1730	}
1731	return "<unknown MCR result>";
1732	}
1733
1734	int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq)
1735	{
1736	if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) {
1737	int ret = qman_alloc_fqid(&fqid);
1738
1739	if (ret)
1740	return ret;
1741	}
1742	fq->fqid = fqid;
1743	fq->flags = flags;
1744	fq->state = qman_fq_state_oos;
1745	fq->cgr_groupid = 0;
1746
1747	/* A context_b of 0 is allegedly special, so don't use that fqid */
1748	if (fqid == 0 \|\| fqid >= num_fqids) {
1749	WARN(1, "bad fqid %d\n", fqid);
1750	return -EINVAL;
1751	}
1752
1753	fq->idx = fqid * 2;
1754	if (flags & QMAN_FQ_FLAG_NO_MODIFY)
1755	fq->idx++;
1756
1757	WARN_ON(fq_table[fq->idx]);
1758	fq_table[fq->idx] = fq;
1759
1760	return 0;
1761	}
1762	EXPORT_SYMBOL(qman_create_fq);
1763
1764	void qman_destroy_fq(struct qman_fq *fq)
1765	{
1766	/*
1767	* We don't need to lock the FQ as it is a pre-condition that the FQ be
1768	* quiesced. Instead, run some checks.
1769	*/
1770	switch (fq->state) {
1771	case qman_fq_state_parked:
1772	case qman_fq_state_oos:
1773	if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID))
1774	qman_release_fqid(fq->fqid);
1775
1776	DPAA_ASSERT(fq_table[fq->idx]);
1777	fq_table[fq->idx] = NULL;
1778	return;
1779	default:
1780	break;
1781	}
1782	DPAA_ASSERT(NULL == "qman_free_fq() on unquiesced FQ!");
1783	}
1784	EXPORT_SYMBOL(qman_destroy_fq);
1785
1786	u32 qman_fq_fqid(struct qman_fq *fq)
1787	{
1788	return fq->fqid;
1789	}
1790	EXPORT_SYMBOL(qman_fq_fqid);
1791
1792	int qman_init_fq(struct qman_fq fq, u32 flags, struct qm_mcc_initfq opts)
1793	{
1794	union qm_mc_command *mcc;
1795	union qm_mc_result *mcr;
1796	struct qman_portal *p;
1797	u8 res, myverb;
1798	int ret = 0;
1799
1800	myverb = (flags & QMAN_INITFQ_FLAG_SCHED)
1801	? QM_MCC_VERB_INITFQ_SCHED : QM_MCC_VERB_INITFQ_PARKED;
1802
1803	if (fq->state != qman_fq_state_oos &&
1804	fq->state != qman_fq_state_parked)
1805	return -EINVAL;
1806	#ifdef CONFIG_FSL_DPAA_CHECKING
1807	if (fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))
1808	return -EINVAL;
1809	#endif
1810	if (opts && (be16_to_cpu(opts->we_mask) & QM_INITFQ_WE_OAC)) {
1811	/* And can't be set at the same time as TDTHRESH */
1812	if (be16_to_cpu(opts->we_mask) & QM_INITFQ_WE_TDTHRESH)
1813	return -EINVAL;
1814	}
1815	/* Issue an INITFQ_[PARKED\|SCHED] management command */
1816	p = get_affine_portal();
1817	if (fq_isset(fq, QMAN_FQ_STATE_CHANGING) \|\|
1818	(fq->state != qman_fq_state_oos &&
1819	fq->state != qman_fq_state_parked)) {
1820	ret = -EBUSY;
1821	goto out;
1822	}
1823	mcc = qm_mc_start(&p->p);
1824	if (opts)
1825	mcc->initfq = *opts;
1826	qm_fqid_set(&mcc->fq, fq->fqid);
1827	mcc->initfq.count = 0;
1828	/*
1829	* If the FQ does not have the TO_DCPORTAL flag, context_b is set as a
1830	* demux pointer. Otherwise, the caller-provided value is allowed to
1831	* stand, don't overwrite it.
1832	*/
1833	if (fq_isclear(fq, QMAN_FQ_FLAG_TO_DCPORTAL)) {
1834	dma_addr_t phys_fq;
1835
1836	mcc->initfq.we_mask \|= cpu_to_be16(QM_INITFQ_WE_CONTEXTB);
1837	mcc->initfq.fqd.context_b = cpu_to_be32(fq_to_tag(fq));
1838	/*
1839	* and the physical address - NB, if the user wasn't trying to
1840	* set CONTEXTA, clear the stashing settings.
1841	*/
1842	if (!(be16_to_cpu(mcc->initfq.we_mask) &
1843	QM_INITFQ_WE_CONTEXTA)) {
1844	mcc->initfq.we_mask \|=
1845	cpu_to_be16(QM_INITFQ_WE_CONTEXTA);
1846	memset(&mcc->initfq.fqd.context_a, 0,
1847	sizeof(mcc->initfq.fqd.context_a));
1848	} else {
1849	#ifndef __rtems__
1850	struct qman_portal *p = qman_dma_portal;
1851
1852	phys_fq = dma_map_single(p->config->dev, fq,
1853	sizeof(*fq), DMA_TO_DEVICE);
1854	if (dma_mapping_error(p->config->dev, phys_fq)) {
1855	dev_err(p->config->dev, "dma_mapping failed\n");
1856	ret = -EIO;
1857	goto out;
1858	}
1859	#else /* __rtems__ */
1860	phys_fq = (dma_addr_t)fq;
1861	#endif /* __rtems__ */
1862
1863	qm_fqd_stashing_set64(&mcc->initfq.fqd, phys_fq);
1864	}
1865	}
1866	if (flags & QMAN_INITFQ_FLAG_LOCAL) {
1867	int wq = 0;
1868
1869	if (!(be16_to_cpu(mcc->initfq.we_mask) &
1870	QM_INITFQ_WE_DESTWQ)) {
1871	mcc->initfq.we_mask \|=
1872	cpu_to_be16(QM_INITFQ_WE_DESTWQ);
1873	wq = 4;
1874	}
1875	qm_fqd_set_destwq(&mcc->initfq.fqd, p->config->channel, wq);
1876	}
1877	qm_mc_commit(&p->p, myverb);
1878	if (!qm_mc_result_timeout(&p->p, &mcr)) {
1879	dev_err(p->config->dev, "MCR timeout\n");
1880	ret = -ETIMEDOUT;
1881	goto out;
1882	}
1883
1884	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
1885	res = mcr->result;
1886	if (res != QM_MCR_RESULT_OK) {
1887	ret = -EIO;
1888	goto out;
1889	}
1890	if (opts) {
1891	if (be16_to_cpu(opts->we_mask) & QM_INITFQ_WE_FQCTRL) {
1892	if (be16_to_cpu(opts->fqd.fq_ctrl) & QM_FQCTRL_CGE)
1893	fq_set(fq, QMAN_FQ_STATE_CGR_EN);
1894	else
1895	fq_clear(fq, QMAN_FQ_STATE_CGR_EN);
1896	}
1897	if (be16_to_cpu(opts->we_mask) & QM_INITFQ_WE_CGID)
1898	fq->cgr_groupid = opts->fqd.cgid;
1899	}
1900	fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ?
1901	qman_fq_state_sched : qman_fq_state_parked;
1902
1903	out:
1904	put_affine_portal();
1905	return ret;
1906	}
1907	EXPORT_SYMBOL(qman_init_fq);
1908
1909	int qman_schedule_fq(struct qman_fq *fq)
1910	{
1911	union qm_mc_command *mcc;
1912	union qm_mc_result *mcr;
1913	struct qman_portal *p;
1914	int ret = 0;
1915
1916	if (fq->state != qman_fq_state_parked)
1917	return -EINVAL;
1918	#ifdef CONFIG_FSL_DPAA_CHECKING
1919	if (fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))
1920	return -EINVAL;
1921	#endif
1922	/* Issue a ALTERFQ_SCHED management command */
1923	p = get_affine_portal();
1924	if (fq_isset(fq, QMAN_FQ_STATE_CHANGING) \|\|
1925	fq->state != qman_fq_state_parked) {
1926	ret = -EBUSY;
1927	goto out;
1928	}
1929	mcc = qm_mc_start(&p->p);
1930	qm_fqid_set(&mcc->fq, fq->fqid);
1931	qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_SCHED);
1932	if (!qm_mc_result_timeout(&p->p, &mcr)) {
1933	dev_err(p->config->dev, "ALTER_SCHED timeout\n");
1934	ret = -ETIMEDOUT;
1935	goto out;
1936	}
1937
1938	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_SCHED);
1939	if (mcr->result != QM_MCR_RESULT_OK) {
1940	ret = -EIO;
1941	goto out;
1942	}
1943	fq->state = qman_fq_state_sched;
1944	out:
1945	put_affine_portal();
1946	return ret;
1947	}
1948	EXPORT_SYMBOL(qman_schedule_fq);
1949
1950	int qman_retire_fq(struct qman_fq fq, u32 flags)
1951	{
1952	union qm_mc_command *mcc;
1953	union qm_mc_result *mcr;
1954	struct qman_portal *p;
1955	int ret;
1956	u8 res;
1957
1958	if (fq->state != qman_fq_state_parked &&
1959	fq->state != qman_fq_state_sched)
1960	return -EINVAL;
1961	#ifdef CONFIG_FSL_DPAA_CHECKING
1962	if (fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))
1963	return -EINVAL;
1964	#endif
1965	p = get_affine_portal();
1966	if (fq_isset(fq, QMAN_FQ_STATE_CHANGING) \|\|
1967	fq->state == qman_fq_state_retired \|\|
1968	fq->state == qman_fq_state_oos) {
1969	ret = -EBUSY;
1970	goto out;
1971	}
1972	mcc = qm_mc_start(&p->p);
1973	qm_fqid_set(&mcc->fq, fq->fqid);
1974	qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE);
1975	if (!qm_mc_result_timeout(&p->p, &mcr)) {
1976	dev_crit(p->config->dev, "ALTER_RETIRE timeout\n");
1977	ret = -ETIMEDOUT;
1978	goto out;
1979	}
1980
1981	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_RETIRE);
1982	res = mcr->result;
1983	/*
1984	* "Elegant" would be to treat OK/PENDING the same way; set CHANGING,
1985	* and defer the flags until FQRNI or FQRN (respectively) show up. But
1986	* "Friendly" is to process OK immediately, and not set CHANGING. We do
1987	* friendly, otherwise the caller doesn't necessarily have a fully
1988	* "retired" FQ on return even if the retirement was immediate. However
1989	* this does mean some code duplication between here and
1990	* fq_state_change().
1991	*/
1992	if (res == QM_MCR_RESULT_OK) {
1993	ret = 0;
1994	/* Process 'fq' right away, we'll ignore FQRNI */
1995	if (mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY)
1996	fq_set(fq, QMAN_FQ_STATE_NE);
1997	if (mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)
1998	fq_set(fq, QMAN_FQ_STATE_ORL);
1999	if (flags)
2000	*flags = fq->flags;
2001	fq->state = qman_fq_state_retired;
2002	if (fq->cb.fqs) {
2003	/*
2004	* Another issue with supporting "immediate" retirement
2005	* is that we're forced to drop FQRNIs, because by the
2006	* time they're seen it may already be "too late" (the
2007	* fq may have been OOS'd and free()'d already). But if
2008	* the upper layer wants a callback whether it's
2009	* immediate or not, we have to fake a "MR" entry to
2010	* look like an FQRNI...
2011	*/
2012	union qm_mr_entry msg;
2013
2014	msg.verb = QM_MR_VERB_FQRNI;
2015	msg.fq.fqs = mcr->alterfq.fqs;
2016	qm_fqid_set(&msg.fq, fq->fqid);
2017	msg.fq.context_b = cpu_to_be32(fq_to_tag(fq));
2018	fq->cb.fqs(p, fq, &msg);
2019	}
2020	} else if (res == QM_MCR_RESULT_PENDING) {
2021	ret = 1;
2022	fq_set(fq, QMAN_FQ_STATE_CHANGING);
2023	} else {
2024	ret = -EIO;
2025	}
2026	out:
2027	put_affine_portal();
2028	return ret;
2029	}
2030	EXPORT_SYMBOL(qman_retire_fq);
2031
2032	int qman_oos_fq(struct qman_fq *fq)
2033	{
2034	union qm_mc_command *mcc;
2035	union qm_mc_result *mcr;
2036	struct qman_portal *p;
2037	int ret = 0;
2038
2039	if (fq->state != qman_fq_state_retired)
2040	return -EINVAL;
2041	#ifdef CONFIG_FSL_DPAA_CHECKING
2042	if (fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))
2043	return -EINVAL;
2044	#endif
2045	p = get_affine_portal();
2046	if (fq_isset(fq, QMAN_FQ_STATE_BLOCKOOS) \|\|
2047	fq->state != qman_fq_state_retired) {
2048	ret = -EBUSY;
2049	goto out;
2050	}
2051	mcc = qm_mc_start(&p->p);
2052	qm_fqid_set(&mcc->fq, fq->fqid);
2053	qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS);
2054	if (!qm_mc_result_timeout(&p->p, &mcr)) {
2055	ret = -ETIMEDOUT;
2056	goto out;
2057	}
2058	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS);
2059	if (mcr->result != QM_MCR_RESULT_OK) {
2060	ret = -EIO;
2061	goto out;
2062	}
2063	fq->state = qman_fq_state_oos;
2064	out:
2065	put_affine_portal();
2066	return ret;
2067	}
2068	EXPORT_SYMBOL(qman_oos_fq);
2069
2070	int qman_query_fq(struct qman_fq fq, struct qm_fqd fqd)
2071	{
2072	union qm_mc_command *mcc;
2073	union qm_mc_result *mcr;
2074	struct qman_portal *p = get_affine_portal();
2075	int ret = 0;
2076
2077	mcc = qm_mc_start(&p->p);
2078	qm_fqid_set(&mcc->fq, fq->fqid);
2079	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
2080	if (!qm_mc_result_timeout(&p->p, &mcr)) {
2081	ret = -ETIMEDOUT;
2082	goto out;
2083	}
2084
2085	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ);
2086	if (mcr->result == QM_MCR_RESULT_OK)
2087	*fqd = mcr->queryfq.fqd;
2088	else
2089	ret = -EIO;
2090	out:
2091	put_affine_portal();
2092	return ret;
2093	}
2094
2095	int qman_query_fq_np(struct qman_fq fq, struct qm_mcr_queryfq_np np)
2096	{
2097	union qm_mc_command *mcc;
2098	union qm_mc_result *mcr;
2099	struct qman_portal *p = get_affine_portal();
2100	int ret = 0;
2101
2102	mcc = qm_mc_start(&p->p);
2103	qm_fqid_set(&mcc->fq, fq->fqid);
2104	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
2105	if (!qm_mc_result_timeout(&p->p, &mcr)) {
2106	ret = -ETIMEDOUT;
2107	goto out;
2108	}
2109
2110	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
2111	if (mcr->result == QM_MCR_RESULT_OK)
2112	*np = mcr->queryfq_np;
2113	else if (mcr->result == QM_MCR_RESULT_ERR_FQID)
2114	ret = -ERANGE;
2115	else
2116	ret = -EIO;
2117	out:
2118	put_affine_portal();
2119	return ret;
2120	}
2121	EXPORT_SYMBOL(qman_query_fq_np);
2122
2123	static int qman_query_cgr(struct qman_cgr *cgr,
2124	struct qm_mcr_querycgr *cgrd)
2125	{
2126	union qm_mc_command *mcc;
2127	union qm_mc_result *mcr;
2128	struct qman_portal *p = get_affine_portal();
2129	int ret = 0;
2130
2131	mcc = qm_mc_start(&p->p);
2132	mcc->cgr.cgid = cgr->cgrid;
2133	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCGR);
2134	if (!qm_mc_result_timeout(&p->p, &mcr)) {
2135	ret = -ETIMEDOUT;
2136	goto out;
2137	}
2138	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYCGR);
2139	if (mcr->result == QM_MCR_RESULT_OK)
2140	*cgrd = mcr->querycgr;
2141	else {
2142	dev_err(p->config->dev, "QUERY_CGR failed: %s\n",
2143	mcr_result_str(mcr->result));
2144	ret = -EIO;
2145	}
2146	out:
2147	put_affine_portal();
2148	return ret;
2149	}
2150
2151	int qman_query_cgr_congested(struct qman_cgr cgr, bool result)
2152	{
2153	struct qm_mcr_querycgr query_cgr;
2154	int err;
2155
2156	err = qman_query_cgr(cgr, &query_cgr);
2157	if (err)
2158	return err;
2159
2160	*result = !!query_cgr.cgr.cs;
2161	return 0;
2162	}
2163	EXPORT_SYMBOL(qman_query_cgr_congested);
2164
2165	/* internal function used as a wait_event() expression */
2166	static int set_p_vdqcr(struct qman_portal p, struct qman_fq fq, u32 vdqcr)
2167	{
2168	unsigned long irqflags;
2169	int ret = -EBUSY;
2170
2171	local_irq_save(irqflags);
2172	if (p->vdqcr_owned)
2173	goto out;
2174	if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
2175	goto out;
2176
2177	fq_set(fq, QMAN_FQ_STATE_VDQCR);
2178	p->vdqcr_owned = fq;
2179	qm_dqrr_vdqcr_set(&p->p, vdqcr);
2180	ret = 0;
2181	out:
2182	local_irq_restore(irqflags);
2183	return ret;
2184	}
2185
2186	static int set_vdqcr(struct qman_portal *p, struct qman_fq fq, u32 vdqcr)
2187	{
2188	int ret;
2189
2190	*p = get_affine_portal();
2191	ret = set_p_vdqcr(*p, fq, vdqcr);
2192	put_affine_portal();
2193	return ret;
2194	}
2195
2196	static int wait_vdqcr_start(struct qman_portal *p, struct qman_fq fq,
2197	u32 vdqcr, u32 flags)
2198	{
2199	int ret = 0;
2200
2201	#ifndef __rtems__
2202	if (flags & QMAN_VOLATILE_FLAG_WAIT_INT)
2203	ret = wait_event_interruptible(affine_queue,
2204	!set_vdqcr(p, fq, vdqcr));
2205	else
2206	#endif /* __rtems__ */
2207	wait_event(affine_queue, !set_vdqcr(p, fq, vdqcr));
2208	return ret;
2209	}
2210
2211	int qman_volatile_dequeue(struct qman_fq *fq, u32 flags, u32 vdqcr)
2212	{
2213	struct qman_portal *p;
2214	int ret;
2215
2216	if (fq->state != qman_fq_state_parked &&
2217	fq->state != qman_fq_state_retired)
2218	return -EINVAL;
2219	if (vdqcr & QM_VDQCR_FQID_MASK)
2220	return -EINVAL;
2221	if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
2222	return -EBUSY;
2223	vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) \| fq->fqid;
2224	if (flags & QMAN_VOLATILE_FLAG_WAIT)
2225	ret = wait_vdqcr_start(&p, fq, vdqcr, flags);
2226	else
2227	ret = set_vdqcr(&p, fq, vdqcr);
2228	if (ret)
2229	return ret;
2230	/* VDQCR is set */
2231	if (flags & QMAN_VOLATILE_FLAG_FINISH) {
2232	#ifndef __rtems__
2233	if (flags & QMAN_VOLATILE_FLAG_WAIT_INT)
2234	/*
2235	* NB: don't propagate any error - the caller wouldn't
2236	* know whether the VDQCR was issued or not. A signal
2237	* could arrive after returning anyway, so the caller
2238	* can check signal_pending() if that's an issue.
2239	*/
2240	wait_event_interruptible(affine_queue,
2241	!fq_isset(fq, QMAN_FQ_STATE_VDQCR));
2242	else
2243	#endif /* __rtems__ */
2244	wait_event(affine_queue,
2245	!fq_isset(fq, QMAN_FQ_STATE_VDQCR));
2246	}
2247	return 0;
2248	}
2249	EXPORT_SYMBOL(qman_volatile_dequeue);
2250
2251	static void update_eqcr_ci(struct qman_portal *p, u8 avail)
2252	{
2253	if (avail)
2254	qm_eqcr_cce_prefetch(&p->p);
2255	else
2256	qm_eqcr_cce_update(&p->p);
2257	}
2258
2259	int qman_enqueue(struct qman_fq fq, const struct qm_fd fd)
2260	{
2261	struct qman_portal *p;
2262	struct qm_eqcr_entry *eq;
2263	unsigned long irqflags;
2264	u8 avail;
2265
2266	p = get_affine_portal();
2267	local_irq_save(irqflags);
2268
2269	if (p->use_eqcr_ci_stashing) {
2270	/*
2271	* The stashing case is easy, only update if we need to in
2272	* order to try and liberate ring entries.
2273	*/
2274	eq = qm_eqcr_start_stash(&p->p);
2275	} else {
2276	/*
2277	* The non-stashing case is harder, need to prefetch ahead of
2278	* time.
2279	*/
2280	avail = qm_eqcr_get_avail(&p->p);
2281	if (avail < 2)
2282	update_eqcr_ci(p, avail);
2283	eq = qm_eqcr_start_no_stash(&p->p);
2284	}
2285
2286	if (unlikely(!eq))
2287	goto out;
2288
2289	qm_fqid_set(eq, fq->fqid);
2290	eq->tag = cpu_to_be32(fq_to_tag(fq));
2291	eq->fd = *fd;
2292
2293	qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE);
2294	out:
2295	local_irq_restore(irqflags);
2296	put_affine_portal();
2297	return 0;
2298	}
2299	EXPORT_SYMBOL(qman_enqueue);
2300
2301	static int qm_modify_cgr(struct qman_cgr *cgr, u32 flags,
2302	struct qm_mcc_initcgr *opts)
2303	{
2304	union qm_mc_command *mcc;
2305	union qm_mc_result *mcr;
2306	struct qman_portal *p = get_affine_portal();
2307	u8 verb = QM_MCC_VERB_MODIFYCGR;
2308	int ret = 0;
2309
2310	mcc = qm_mc_start(&p->p);
2311	if (opts)
2312	mcc->initcgr = *opts;
2313	mcc->initcgr.cgid = cgr->cgrid;
2314	if (flags & QMAN_CGR_FLAG_USE_INIT)
2315	verb = QM_MCC_VERB_INITCGR;
2316	qm_mc_commit(&p->p, verb);
2317	if (!qm_mc_result_timeout(&p->p, &mcr)) {
2318	ret = -ETIMEDOUT;
2319	goto out;
2320	}
2321
2322	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == verb);
2323	if (mcr->result != QM_MCR_RESULT_OK)
2324	ret = -EIO;
2325
2326	out:
2327	put_affine_portal();
2328	return ret;
2329	}
2330
2331	#define PORTAL_IDX(n) (n->config->channel - QM_CHANNEL_SWPORTAL0)
2332
2333	/* congestion state change notification target update control */
2334	static void qm_cgr_cscn_targ_set(struct __qm_mc_cgr *cgr, int pi, u32 val)
2335	{
2336	if (qman_ip_rev >= QMAN_REV30)
2337	cgr->cscn_targ_upd_ctrl = cpu_to_be16(pi \|
2338	QM_CGR_TARG_UDP_CTRL_WRITE_BIT);
2339	else
2340	cgr->cscn_targ = cpu_to_be32(val \| QM_CGR_TARG_PORTAL(pi));
2341	}
2342
2343	#ifndef __rtems__
2344	static void qm_cgr_cscn_targ_clear(struct __qm_mc_cgr *cgr, int pi, u32 val)
2345	{
2346	if (qman_ip_rev >= QMAN_REV30)
2347	cgr->cscn_targ_upd_ctrl = cpu_to_be16(pi);
2348	else
2349	cgr->cscn_targ = cpu_to_be32(val & ~QM_CGR_TARG_PORTAL(pi));
2350	}
2351	#endif /* __rtems__ */
2352
2353	static u8 qman_cgr_cpus[CGR_NUM];
2354
2355	void qman_init_cgr_all(void)
2356	{
2357	struct qman_cgr cgr;
2358	int err_cnt = 0;
2359
2360	for (cgr.cgrid = 0; cgr.cgrid < CGR_NUM; cgr.cgrid++) {
2361	if (qm_modify_cgr(&cgr, QMAN_CGR_FLAG_USE_INIT, NULL))
2362	err_cnt++;
2363	}
2364
2365	if (err_cnt)
2366	pr_err("Warning: %d error%s while initialising CGR h/w\n",
2367	err_cnt, (err_cnt > 1) ? "s" : "");
2368	}
2369
2370	int qman_create_cgr(struct qman_cgr *cgr, u32 flags,
2371	struct qm_mcc_initcgr *opts)
2372	{
2373	struct qm_mcr_querycgr cgr_state;
2374	int ret;
2375	struct qman_portal *p;
2376
2377	/*
2378	* We have to check that the provided CGRID is within the limits of the
2379	* data-structures, for obvious reasons. However we'll let h/w take
2380	* care of determining whether it's within the limits of what exists on
2381	* the SoC.
2382	*/
2383	if (cgr->cgrid >= CGR_NUM)
2384	return -EINVAL;
2385
2386	preempt_disable();
2387	p = get_affine_portal();
2388	qman_cgr_cpus[cgr->cgrid] = smp_processor_id();
2389	preempt_enable();
2390
2391	cgr->chan = p->config->channel;
2392	spin_lock(&p->cgr_lock);
2393
2394	if (opts) {
2395	struct qm_mcc_initcgr local_opts = *opts;
2396
2397	ret = qman_query_cgr(cgr, &cgr_state);
2398	if (ret)
2399	goto out;
2400
2401	qm_cgr_cscn_targ_set(&local_opts.cgr, PORTAL_IDX(p),
2402	be32_to_cpu(cgr_state.cgr.cscn_targ));
2403	local_opts.we_mask \|= cpu_to_be16(QM_CGR_WE_CSCN_TARG);
2404
2405	/* send init if flags indicate so */
2406	if (flags & QMAN_CGR_FLAG_USE_INIT)
2407	ret = qm_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT,
2408	&local_opts);
2409	else
2410	ret = qm_modify_cgr(cgr, 0, &local_opts);
2411	if (ret)
2412	goto out;
2413	}
2414
2415	list_add(&cgr->node, &p->cgr_cbs);
2416
2417	/* Determine if newly added object requires its callback to be called */
2418	ret = qman_query_cgr(cgr, &cgr_state);
2419	if (ret) {
2420	/* we can't go back, so proceed and return success */
2421	dev_err(p->config->dev, "CGR HW state partially modified\n");
2422	ret = 0;
2423	goto out;
2424	}
2425	if (cgr->cb && cgr_state.cgr.cscn_en &&
2426	qman_cgrs_get(&p->cgrs[1], cgr->cgrid))
2427	cgr->cb(p, cgr, 1);
2428	out:
2429	spin_unlock(&p->cgr_lock);
2430	put_affine_portal();
2431	return ret;
2432	}
2433	EXPORT_SYMBOL(qman_create_cgr);
2434
2435	#ifndef __rtems__
2436	int qman_delete_cgr(struct qman_cgr *cgr)
2437	{
2438	unsigned long irqflags;
2439	struct qm_mcr_querycgr cgr_state;
2440	struct qm_mcc_initcgr local_opts;
2441	int ret = 0;
2442	struct qman_cgr *i;
2443	struct qman_portal *p = get_affine_portal();
2444
2445	if (cgr->chan != p->config->channel) {
2446	/* attempt to delete from other portal than creator */
2447	dev_err(p->config->dev, "CGR not owned by current portal");
2448	dev_dbg(p->config->dev, " create 0x%x, delete 0x%x\n",
2449	cgr->chan, p->config->channel);
2450
2451	ret = -EINVAL;
2452	goto put_portal;
2453	}
2454	memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
2455	spin_lock_irqsave(&p->cgr_lock, irqflags);
2456	list_del(&cgr->node);
2457	/*
2458	* If there are no other CGR objects for this CGRID in the list,
2459	* update CSCN_TARG accordingly
2460	*/
2461	list_for_each_entry(i, &p->cgr_cbs, node)
2462	if (i->cgrid == cgr->cgrid && i->cb)
2463	goto release_lock;
2464	ret = qman_query_cgr(cgr, &cgr_state);
2465	if (ret) {
2466	/* add back to the list */
2467	list_add(&cgr->node, &p->cgr_cbs);
2468	goto release_lock;
2469	}
2470
2471	local_opts.we_mask = cpu_to_be16(QM_CGR_WE_CSCN_TARG);
2472	qm_cgr_cscn_targ_clear(&local_opts.cgr, PORTAL_IDX(p),
2473	be32_to_cpu(cgr_state.cgr.cscn_targ));
2474
2475	ret = qm_modify_cgr(cgr, 0, &local_opts);
2476	if (ret)
2477	/* add back to the list */
2478	list_add(&cgr->node, &p->cgr_cbs);
2479	release_lock:
2480	spin_unlock_irqrestore(&p->cgr_lock, irqflags);
2481	put_portal:
2482	put_affine_portal();
2483	return ret;
2484	}
2485	EXPORT_SYMBOL(qman_delete_cgr);
2486
2487	struct cgr_comp {
2488	struct qman_cgr *cgr;
2489	struct completion completion;
2490	};
2491
2492	static int qman_delete_cgr_thread(void *p)
2493	{
2494	struct cgr_comp cgr_comp = (struct cgr_comp )p;
2495	int ret;
2496
2497	ret = qman_delete_cgr(cgr_comp->cgr);
2498	complete(&cgr_comp->completion);
2499
2500	return ret;
2501	}
2502
2503	void qman_delete_cgr_safe(struct qman_cgr *cgr)
2504	{
2505	struct task_struct *thread;
2506	struct cgr_comp cgr_comp;
2507
2508	preempt_disable();
2509	if (qman_cgr_cpus[cgr->cgrid] != smp_processor_id()) {
2510	init_completion(&cgr_comp.completion);
2511	cgr_comp.cgr = cgr;
2512	thread = kthread_create(qman_delete_cgr_thread, &cgr_comp,
2513	"cgr_del");
2514
2515	if (IS_ERR(thread))
2516	goto out;
2517
2518	kthread_bind(thread, qman_cgr_cpus[cgr->cgrid]);
2519	wake_up_process(thread);
2520	wait_for_completion(&cgr_comp.completion);
2521	preempt_enable();
2522	return;
2523	}
2524	out:
2525	qman_delete_cgr(cgr);
2526	preempt_enable();
2527	}
2528	EXPORT_SYMBOL(qman_delete_cgr_safe);
2529	#endif /* __rtems__ */
2530
2531	/* Cleanup FQs */
2532
2533	static int _qm_mr_consume_and_match_verb(struct qm_portal *p, int v)
2534	{
2535	const union qm_mr_entry *msg;
2536	int found = 0;
2537
2538	qm_mr_pvb_update(p);
2539	msg = qm_mr_current(p);
2540	while (msg) {
2541	if ((msg->verb & QM_MR_VERB_TYPE_MASK) == v)
2542	found = 1;
2543	qm_mr_next(p);
2544	qm_mr_cci_consume_to_current(p);
2545	qm_mr_pvb_update(p);
2546	msg = qm_mr_current(p);
2547	}
2548	return found;
2549	}
2550
2551	static int _qm_dqrr_consume_and_match(struct qm_portal *p, u32 fqid, int s,
2552	bool wait)
2553	{
2554	const struct qm_dqrr_entry *dqrr;
2555	int found = 0;
2556
2557	do {
2558	qm_dqrr_pvb_update(p);
2559	dqrr = qm_dqrr_current(p);
2560	if (!dqrr)
2561	cpu_relax();
2562	} while (wait && !dqrr);
2563
2564	while (dqrr) {
2565	if (qm_fqid_get(dqrr) == fqid && (dqrr->stat & s))
2566	found = 1;
2567	qm_dqrr_cdc_consume_1ptr(p, dqrr, 0);
2568	qm_dqrr_pvb_update(p);
2569	qm_dqrr_next(p);
2570	dqrr = qm_dqrr_current(p);
2571	}
2572	return found;
2573	}
2574
2575	#define qm_mr_drain(p, V) \
2576	_qm_mr_consume_and_match_verb(p, QM_MR_VERB_##V)
2577
2578	#define qm_dqrr_drain(p, f, S) \
2579	_qm_dqrr_consume_and_match(p, f, QM_DQRR_STAT_##S, false)
2580
2581	#define qm_dqrr_drain_wait(p, f, S) \
2582	_qm_dqrr_consume_and_match(p, f, QM_DQRR_STAT_##S, true)
2583
2584	#define qm_dqrr_drain_nomatch(p) \
2585	_qm_dqrr_consume_and_match(p, 0, 0, false)
2586
2587	static int qman_shutdown_fq(u32 fqid)
2588	{
2589	struct qman_portal *p;
2590	#ifndef __rtems__
2591	struct device *dev;
2592	#endif /* __rtems__ */
2593	union qm_mc_command *mcc;
2594	union qm_mc_result *mcr;
2595	int orl_empty, drain = 0, ret = 0;
2596	u32 channel, wq, res;
2597	u8 state;
2598
2599	p = get_affine_portal();
2600	#ifndef __rtems__
2601	dev = p->config->dev;
2602	#endif /* __rtems__ */
2603	/* Determine the state of the FQID */
2604	mcc = qm_mc_start(&p->p);
2605	qm_fqid_set(&mcc->fq, fqid);
2606	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
2607	if (!qm_mc_result_timeout(&p->p, &mcr)) {
2608	dev_err(dev, "QUERYFQ_NP timeout\n");
2609	ret = -ETIMEDOUT;
2610	goto out;
2611	}
2612
2613	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
2614	state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK;
2615	if (state == QM_MCR_NP_STATE_OOS)
2616	goto out; /* Already OOS, no need to do anymore checks */
2617
2618	/* Query which channel the FQ is using */
2619	mcc = qm_mc_start(&p->p);
2620	qm_fqid_set(&mcc->fq, fqid);
2621	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
2622	if (!qm_mc_result_timeout(&p->p, &mcr)) {
2623	dev_err(dev, "QUERYFQ timeout\n");
2624	ret = -ETIMEDOUT;
2625	goto out;
2626	}
2627
2628	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ);
2629	/* Need to store these since the MCR gets reused */
2630	channel = qm_fqd_get_chan(&mcr->queryfq.fqd);
2631	wq = qm_fqd_get_wq(&mcr->queryfq.fqd);
2632
2633	switch (state) {
2634	case QM_MCR_NP_STATE_TEN_SCHED:
2635	case QM_MCR_NP_STATE_TRU_SCHED:
2636	case QM_MCR_NP_STATE_ACTIVE:
2637	case QM_MCR_NP_STATE_PARKED:
2638	orl_empty = 0;
2639	mcc = qm_mc_start(&p->p);
2640	qm_fqid_set(&mcc->fq, fqid);
2641	qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE);
2642	if (!qm_mc_result_timeout(&p->p, &mcr)) {
2643	dev_err(dev, "QUERYFQ_NP timeout\n");
2644	ret = -ETIMEDOUT;
2645	goto out;
2646	}
2647	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
2648	QM_MCR_VERB_ALTER_RETIRE);
2649	res = mcr->result; /* Make a copy as we reuse MCR below */
2650
2651	if (res == QM_MCR_RESULT_PENDING) {
2652	/*
2653	* Need to wait for the FQRN in the message ring, which
2654	* will only occur once the FQ has been drained. In
2655	* order for the FQ to drain the portal needs to be set
2656	* to dequeue from the channel the FQ is scheduled on
2657	*/
2658	int found_fqrn = 0;
2659	u16 dequeue_wq = 0;
2660
2661	/* Flag that we need to drain FQ */
2662	drain = 1;
2663
2664	if (channel >= qm_channel_pool1 &&
2665	channel < qm_channel_pool1 + 15) {
2666	/* Pool channel, enable the bit in the portal */
2667	dequeue_wq = (channel -
2668	qm_channel_pool1 + 1)<<4 \| wq;
2669	} else if (channel < qm_channel_pool1) {
2670	/* Dedicated channel */
2671	dequeue_wq = wq;
2672	} else {
2673	dev_err(dev, "Can't recover FQ 0x%x, ch: 0x%x",
2674	fqid, channel);
2675	ret = -EBUSY;
2676	goto out;
2677	}
2678	#ifdef __rtems__
2679	(void)dequeue_wq;
2680	#endif /* __rtems__ */
2681	/* Set the sdqcr to drain this channel */
2682	if (channel < qm_channel_pool1)
2683	qm_dqrr_sdqcr_set(&p->p,
2684	QM_SDQCR_TYPE_ACTIVE \|
2685	QM_SDQCR_CHANNELS_DEDICATED);
2686	else
2687	qm_dqrr_sdqcr_set(&p->p,
2688	QM_SDQCR_TYPE_ACTIVE \|
2689	QM_SDQCR_CHANNELS_POOL_CONV
2690	(channel));
2691	do {
2692	/* Keep draining DQRR while checking the MR*/
2693	qm_dqrr_drain_nomatch(&p->p);
2694	/* Process message ring too */
2695	found_fqrn = qm_mr_drain(&p->p, FQRN);
2696	cpu_relax();
2697	} while (!found_fqrn);
2698
2699	}
2700	if (res != QM_MCR_RESULT_OK &&
2701	res != QM_MCR_RESULT_PENDING) {
2702	dev_err(dev, "retire_fq failed: FQ 0x%x, res=0x%x\n",
2703	fqid, res);
2704	ret = -EIO;
2705	goto out;
2706	}
2707	if (!(mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)) {
2708	/*
2709	* ORL had no entries, no need to wait until the
2710	* ERNs come in
2711	*/
2712	orl_empty = 1;
2713	}
2714	/*
2715	* Retirement succeeded, check to see if FQ needs
2716	* to be drained
2717	*/
2718	if (drain \|\| mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) {
2719	/* FQ is Not Empty, drain using volatile DQ commands */
2720	do {
2721	u32 vdqcr = fqid \| QM_VDQCR_NUMFRAMES_SET(3);
2722
2723	qm_dqrr_vdqcr_set(&p->p, vdqcr);
2724	/*
2725	* Wait for a dequeue and process the dequeues,
2726	* making sure to empty the ring completely
2727	*/
2728	} while (qm_dqrr_drain_wait(&p->p, fqid, FQ_EMPTY));
2729	}
2730	qm_dqrr_sdqcr_set(&p->p, 0);
2731
2732	while (!orl_empty) {
2733	/* Wait for the ORL to have been completely drained */
2734	orl_empty = qm_mr_drain(&p->p, FQRL);
2735	cpu_relax();
2736	}
2737	mcc = qm_mc_start(&p->p);
2738	qm_fqid_set(&mcc->fq, fqid);
2739	qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS);
2740	if (!qm_mc_result_timeout(&p->p, &mcr)) {
2741	ret = -ETIMEDOUT;
2742	goto out;
2743	}
2744
2745	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
2746	QM_MCR_VERB_ALTER_OOS);
2747	if (mcr->result != QM_MCR_RESULT_OK) {
2748	dev_err(dev, "OOS after drain fail: FQ 0x%x (0x%x)\n",
2749	fqid, mcr->result);
2750	ret = -EIO;
2751	goto out;
2752	}
2753	break;
2754
2755	case QM_MCR_NP_STATE_RETIRED:
2756	/* Send OOS Command */
2757	mcc = qm_mc_start(&p->p);
2758	qm_fqid_set(&mcc->fq, fqid);
2759	qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS);
2760	if (!qm_mc_result_timeout(&p->p, &mcr)) {
2761	ret = -ETIMEDOUT;
2762	goto out;
2763	}
2764
2765	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
2766	QM_MCR_VERB_ALTER_OOS);
2767	if (mcr->result) {
2768	dev_err(dev, "OOS fail: FQ 0x%x (0x%x)\n",
2769	fqid, mcr->result);
2770	ret = -EIO;
2771	goto out;
2772	}
2773	break;
2774
2775	case QM_MCR_NP_STATE_OOS:
2776	/* Done */
2777	break;
2778
2779	default:
2780	ret = -EIO;
2781	}
2782
2783	out:
2784	put_affine_portal();
2785	return ret;
2786	}
2787
2788	const struct qm_portal_config *qman_get_qm_portal_config(
2789	struct qman_portal *portal)
2790	{
2791	return portal->config;
2792	}
2793	EXPORT_SYMBOL(qman_get_qm_portal_config);
2794
2795	struct gen_pool qm_fqalloc; / FQID allocator */
2796	struct gen_pool qm_qpalloc; / pool-channel allocator */
2797	struct gen_pool qm_cgralloc; / CGR ID allocator */
2798
2799	static int qman_alloc_range(struct gen_pool p, u32 result, u32 cnt)
2800	{
2801	unsigned long addr;
2802
2803	addr = gen_pool_alloc(p, cnt);
2804	if (!addr)
2805	return -ENOMEM;
2806
2807	*result = addr & ~DPAA_GENALLOC_OFF;
2808
2809	return 0;
2810	}
2811
2812	int qman_alloc_fqid_range(u32 *result, u32 count)
2813	{
2814	return qman_alloc_range(qm_fqalloc, result, count);
2815	}
2816	EXPORT_SYMBOL(qman_alloc_fqid_range);
2817
2818	int qman_alloc_pool_range(u32 *result, u32 count)
2819	{
2820	return qman_alloc_range(qm_qpalloc, result, count);
2821	}
2822	EXPORT_SYMBOL(qman_alloc_pool_range);
2823
2824	int qman_alloc_cgrid_range(u32 *result, u32 count)
2825	{
2826	return qman_alloc_range(qm_cgralloc, result, count);
2827	}
2828	EXPORT_SYMBOL(qman_alloc_cgrid_range);
2829
2830	int qman_release_fqid(u32 fqid)
2831	{
2832	int ret = qman_shutdown_fq(fqid);
2833
2834	if (ret) {
2835	pr_debug("FQID %d leaked\n", fqid);
2836	return ret;
2837	}
2838
2839	gen_pool_free(qm_fqalloc, fqid \| DPAA_GENALLOC_OFF, 1);
2840	return 0;
2841	}
2842	EXPORT_SYMBOL(qman_release_fqid);
2843
2844	static int qpool_cleanup(u32 qp)
2845	{
2846	/*
2847	* We query all FQDs starting from
2848	* FQID 1 until we get an "invalid FQID" error, looking for non-OOS FQDs
2849	* whose destination channel is the pool-channel being released.
2850	* When a non-OOS FQD is found we attempt to clean it up
2851	*/
2852	struct qman_fq fq = {
2853	.fqid = QM_FQID_RANGE_START
2854	};
2855	int err;
2856
2857	do {
2858	struct qm_mcr_queryfq_np np;
2859
2860	err = qman_query_fq_np(&fq, &np);
2861	if (err == -ERANGE)
2862	/* FQID range exceeded, found no problems */
2863	return 0;
2864	else if (WARN_ON(err))
2865	return err;
2866
2867	if ((np.state & QM_MCR_NP_STATE_MASK) != QM_MCR_NP_STATE_OOS) {
2868	struct qm_fqd fqd;
2869
2870	err = qman_query_fq(&fq, &fqd);
2871	if (WARN_ON(err))
2872	return err;
2873	if (qm_fqd_get_chan(&fqd) == qp) {
2874	/* The channel is the FQ's target, clean it */
2875	err = qman_shutdown_fq(fq.fqid);
2876	if (err)
2877	/*
2878	* Couldn't shut down the FQ
2879	* so the pool must be leaked
2880	*/
2881	return err;
2882	}
2883	}
2884	/* Move to the next FQID */
2885	fq.fqid++;
2886	} while (1);
2887	}
2888
2889	int qman_release_pool(u32 qp)
2890	{
2891	int ret;
2892
2893	ret = qpool_cleanup(qp);
2894	if (ret) {
2895	pr_debug("CHID %d leaked\n", qp);
2896	return ret;
2897	}
2898
2899	gen_pool_free(qm_qpalloc, qp \| DPAA_GENALLOC_OFF, 1);
2900	return 0;
2901	}
2902	EXPORT_SYMBOL(qman_release_pool);
2903
2904	static int cgr_cleanup(u32 cgrid)
2905	{
2906	/*
2907	* query all FQDs starting from FQID 1 until we get an "invalid FQID"
2908	* error, looking for non-OOS FQDs whose CGR is the CGR being released
2909	*/
2910	struct qman_fq fq = {
2911	.fqid = QM_FQID_RANGE_START
2912	};
2913	int err;
2914
2915	do {
2916	struct qm_mcr_queryfq_np np;
2917
2918	err = qman_query_fq_np(&fq, &np);
2919	if (err == -ERANGE)
2920	/* FQID range exceeded, found no problems */
2921	return 0;
2922	else if (WARN_ON(err))
2923	return err;
2924
2925	if ((np.state & QM_MCR_NP_STATE_MASK) != QM_MCR_NP_STATE_OOS) {
2926	struct qm_fqd fqd;
2927
2928	err = qman_query_fq(&fq, &fqd);
2929	if (WARN_ON(err))
2930	return err;
2931	if (be16_to_cpu(fqd.fq_ctrl) & QM_FQCTRL_CGE &&
2932	fqd.cgid == cgrid) {
2933	pr_err("CRGID 0x%x is being used by FQID 0x%x, CGR will be leaked\n",
2934	cgrid, fq.fqid);
2935	return -EIO;
2936	}
2937	}
2938	/* Move to the next FQID */
2939	fq.fqid++;
2940	} while (1);
2941	}
2942
2943	int qman_release_cgrid(u32 cgrid)
2944	{
2945	int ret;
2946
2947	ret = cgr_cleanup(cgrid);
2948	if (ret) {
2949	pr_debug("CGRID %d leaked\n", cgrid);
2950	return ret;
2951	}
2952
2953	gen_pool_free(qm_cgralloc, cgrid \| DPAA_GENALLOC_OFF, 1);
2954	return 0;
2955	}
2956	EXPORT_SYMBOL(qman_release_cgrid);

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: rtems-libbsd/linux/drivers/soc/fsl/qbman/qman.c @ 1908686

Download in other formats: