Context Navigation

dwc_otg.c @ c04e7c2

55-freebsd-126-freebsd-12freebsd-9.3

Last change on this file since c04e7c2 was c04e7c2, checked in by hselasky <hselasky@…>, on 01/05/16 at 09:18:43

Fix for directly connected FULL or LOW speed USB devices.

Found by: Sebastian Huber <sebastian.huber@…>
MFC after: 1 week

Property mode set to 100644

File size: 114.3 KB

Line
1	#include <machine/rtems-bsd-kernel-space.h>
2
3	/* $FreeBSD$ */
4	/*-
5	* Copyright (c) 2015 Daisuke Aoyama. All rights reserved.
6	* Copyright (c) 2012-2015 Hans Petter Selasky. All rights reserved.
7	* Copyright (c) 2010-2011 Aleksandr Rybalko. All rights reserved.
8	*
9	* Redistribution and use in source and binary forms, with or without
10	* modification, are permitted provided that the following conditions
11	* are met:
12	* 1. Redistributions of source code must retain the above copyright
13	* notice, this list of conditions and the following disclaimer.
14	* 2. Redistributions in binary form must reproduce the above copyright
15	* notice, this list of conditions and the following disclaimer in the
16	* documentation and/or other materials provided with the distribution.
17	*
18	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28	* SUCH DAMAGE.
29	*/
30
31	/*
32	* This file contains the driver for the DesignWare series USB 2.0 OTG
33	* Controller.
34	*/
35
36	/*
37	* LIMITATION: Drivers must be bound to all OUT endpoints in the
38	* active configuration for this driver to work properly. Blocking any
39	* OUT endpoint will block all OUT endpoints including the control
40	* endpoint. Usually this is not a problem.
41	*/
42
43	/*
44	* NOTE: Writing to non-existing registers appears to cause an
45	* internal reset.
46	*/
47
48	#ifdef USB_GLOBAL_INCLUDE_FILE
49	#include USB_GLOBAL_INCLUDE_FILE
50	#else
51	#include <sys/stdint.h>
52	#include <sys/stddef.h>
53	#include <rtems/bsd/sys/param.h>
54	#include <sys/queue.h>
55	#include <rtems/bsd/sys/types.h>
56	#include <sys/systm.h>
57	#include <sys/kernel.h>
58	#include <sys/bus.h>
59	#include <sys/module.h>
60	#include <rtems/bsd/sys/lock.h>
61	#include <sys/mutex.h>
62	#include <sys/condvar.h>
63	#include <sys/sysctl.h>
64	#include <sys/sx.h>
65	#include <rtems/bsd/sys/unistd.h>
66	#include <sys/callout.h>
67	#include <sys/malloc.h>
68	#include <sys/priv.h>
69
70	#include <dev/usb/usb.h>
71	#include <dev/usb/usbdi.h>
72
73	#define USB_DEBUG_VAR dwc_otg_debug
74
75	#include <dev/usb/usb_core.h>
76	#include <dev/usb/usb_debug.h>
77	#include <dev/usb/usb_busdma.h>
78	#include <dev/usb/usb_process.h>
79	#include <dev/usb/usb_transfer.h>
80	#include <dev/usb/usb_device.h>
81	#include <dev/usb/usb_hub.h>
82	#include <dev/usb/usb_util.h>
83
84	#include <dev/usb/usb_controller.h>
85	#include <dev/usb/usb_bus.h>
86	#endif /* USB_GLOBAL_INCLUDE_FILE */
87
88	#include <dev/usb/controller/dwc_otg.h>
89	#include <dev/usb/controller/dwc_otgreg.h>
90
91	#define DWC_OTG_BUS2SC(bus) \
92	((struct dwc_otg_softc )(((uint8_t )(bus)) - \
93	((uint8_t )&(((struct dwc_otg_softc )0)->sc_bus))))
94
95	#define DWC_OTG_PC2UDEV(pc) \
96	(USB_DMATAG_TO_XROOT((pc)->tag_parent)->udev)
97
98	#define DWC_OTG_MSK_GINT_ENABLED \
99	(GINTMSK_ENUMDONEMSK \| \
100	GINTMSK_USBRSTMSK \| \
101	GINTMSK_USBSUSPMSK \| \
102	GINTMSK_IEPINTMSK \| \
103	GINTMSK_SESSREQINTMSK \| \
104	GINTMSK_RXFLVLMSK \| \
105	GINTMSK_HCHINTMSK \| \
106	GINTMSK_OTGINTMSK \| \
107	GINTMSK_PRTINTMSK)
108
109	#define DWC_OTG_MSK_GINT_THREAD_IRQ \
110	(GINTSTS_USBRST \| GINTSTS_ENUMDONE \| GINTSTS_PRTINT \| \
111	GINTSTS_WKUPINT \| GINTSTS_USBSUSP \| GINTMSK_OTGINTMSK \| \
112	GINTSTS_SESSREQINT)
113
114	#define DWC_OTG_PHY_ULPI 0
115	#define DWC_OTG_PHY_HSIC 1
116	#define DWC_OTG_PHY_INTERNAL 2
117
118	#ifndef DWC_OTG_PHY_DEFAULT
119	#define DWC_OTG_PHY_DEFAULT DWC_OTG_PHY_ULPI
120	#endif
121
122	static int dwc_otg_phy_type = DWC_OTG_PHY_DEFAULT;
123
124	static SYSCTL_NODE(_hw_usb, OID_AUTO, dwc_otg, CTLFLAG_RW, 0, "USB DWC OTG");
125	SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, phy_type, CTLFLAG_RDTUN,
126	&dwc_otg_phy_type, 0, "DWC OTG PHY TYPE - 0/1/2 - ULPI/HSIC/INTERNAL");
127
128	#ifdef USB_DEBUG
129	static int dwc_otg_debug;
130
131	SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, debug, CTLFLAG_RWTUN,
132	&dwc_otg_debug, 0, "DWC OTG debug level");
133	#endif
134
135	#define DWC_OTG_INTR_ENDPT 1
136
137	/* prototypes */
138
139	static const struct usb_bus_methods dwc_otg_bus_methods;
140	static const struct usb_pipe_methods dwc_otg_device_non_isoc_methods;
141	static const struct usb_pipe_methods dwc_otg_device_isoc_methods;
142
143	static dwc_otg_cmd_t dwc_otg_setup_rx;
144	static dwc_otg_cmd_t dwc_otg_data_rx;
145	static dwc_otg_cmd_t dwc_otg_data_tx;
146	static dwc_otg_cmd_t dwc_otg_data_tx_sync;
147
148	static dwc_otg_cmd_t dwc_otg_host_setup_tx;
149	static dwc_otg_cmd_t dwc_otg_host_data_tx;
150	static dwc_otg_cmd_t dwc_otg_host_data_rx;
151
152	static void dwc_otg_device_done(struct usb_xfer *, usb_error_t);
153	static void dwc_otg_do_poll(struct usb_bus *);
154	static void dwc_otg_standard_done(struct usb_xfer *);
155	static void dwc_otg_root_intr(struct dwc_otg_softc *);
156	static void dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *);
157
158	/*
159	* Here is a configuration that the chip supports.
160	*/
161	static const struct usb_hw_ep_profile dwc_otg_ep_profile[1] = {
162
163	[0] = {
164	.max_in_frame_size = 64,/* fixed */
165	.max_out_frame_size = 64, /* fixed */
166	.is_simplex = 1,
167	.support_control = 1,
168	}
169	};
170
171	static void
172	dwc_otg_get_hw_ep_profile(struct usb_device *udev,
173	const struct usb_hw_ep_profile **ppf, uint8_t ep_addr)
174	{
175	struct dwc_otg_softc *sc;
176
177	sc = DWC_OTG_BUS2SC(udev->bus);
178
179	if (ep_addr < sc->sc_dev_ep_max)
180	*ppf = &sc->sc_hw_ep_profile[ep_addr].usb;
181	else
182	*ppf = NULL;
183	}
184
185	static void
186	dwc_otg_write_fifo(struct dwc_otg_softc sc, struct usb_page_cache pc,
187	uint32_t offset, uint32_t fifo, uint32_t count)
188	{
189	uint32_t temp;
190
191	/* round down length to nearest 4-bytes */
192	temp = count & ~3;
193
194	/* check if we can write the data directly */
195	if (temp != 0 && usb_pc_buffer_is_aligned(pc, offset, temp, 3)) {
196	struct usb_page_search buf_res;
197
198	/* pre-subtract length */
199	count -= temp;
200
201	/* iterate buffer list */
202	do {
203	/* get current buffer pointer */
204	usbd_get_page(pc, offset, &buf_res);
205
206	if (buf_res.length > temp)
207	buf_res.length = temp;
208
209	/* transfer data into FIFO */
210	bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
211	fifo, buf_res.buffer, buf_res.length / 4);
212
213	offset += buf_res.length;
214	fifo += buf_res.length;
215	temp -= buf_res.length;
216	} while (temp != 0);
217	}
218
219	/* check for remainder */
220	if (count != 0) {
221	/* clear topmost word before copy */
222	sc->sc_bounce_buffer[(count - 1) / 4] = 0;
223
224	/* copy out data */
225	usbd_copy_out(pc, offset,
226	sc->sc_bounce_buffer, count);
227
228	/* transfer data into FIFO */
229	bus_space_write_region_4(sc->sc_io_tag,
230	sc->sc_io_hdl, fifo, sc->sc_bounce_buffer,
231	(count + 3) / 4);
232	}
233	}
234
235	static void
236	dwc_otg_read_fifo(struct dwc_otg_softc sc, struct usb_page_cache pc,
237	uint32_t offset, uint32_t count)
238	{
239	uint32_t temp;
240
241	/* round down length to nearest 4-bytes */
242	temp = count & ~3;
243
244	/* check if we can read the data directly */
245	if (temp != 0 && usb_pc_buffer_is_aligned(pc, offset, temp, 3)) {
246	struct usb_page_search buf_res;
247
248	/* pre-subtract length */
249	count -= temp;
250
251	/* iterate buffer list */
252	do {
253	/* get current buffer pointer */
254	usbd_get_page(pc, offset, &buf_res);
255
256	if (buf_res.length > temp)
257	buf_res.length = temp;
258
259	/* transfer data from FIFO */
260	bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
261	sc->sc_current_rx_fifo, buf_res.buffer, buf_res.length / 4);
262
263	offset += buf_res.length;
264	sc->sc_current_rx_fifo += buf_res.length;
265	sc->sc_current_rx_bytes -= buf_res.length;
266	temp -= buf_res.length;
267	} while (temp != 0);
268	}
269
270	/* check for remainder */
271	if (count != 0) {
272	/* read data into bounce buffer */
273	bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
274	sc->sc_current_rx_fifo,
275	sc->sc_bounce_buffer, (count + 3) / 4);
276
277	/* store data into proper buffer */
278	usbd_copy_in(pc, offset, sc->sc_bounce_buffer, count);
279
280	/* round length up to nearest 4 bytes */
281	count = (count + 3) & ~3;
282
283	/* update counters */
284	sc->sc_current_rx_bytes -= count;
285	sc->sc_current_rx_fifo += count;
286	}
287	}
288
289	static void
290	dwc_otg_tx_fifo_reset(struct dwc_otg_softc *sc, uint32_t value)
291	{
292	uint32_t temp;
293
294	/* reset FIFO */
295	DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL, value);
296
297	/* wait for reset to complete */
298	for (temp = 0; temp != 16; temp++) {
299	value = DWC_OTG_READ_4(sc, DOTG_GRSTCTL);
300	if (!(value & (GRSTCTL_TXFFLSH \| GRSTCTL_RXFFLSH)))
301	break;
302	}
303	}
304
305	static int
306	dwc_otg_init_fifo(struct dwc_otg_softc *sc, uint8_t mode)
307	{
308	struct dwc_otg_profile *pf;
309	uint32_t fifo_size;
310	uint32_t fifo_regs;
311	uint32_t tx_start;
312	uint8_t x;
313
314	fifo_size = sc->sc_fifo_size;
315
316	/*
317	* NOTE: Reserved fixed size area at end of RAM, which must
318	* not be allocated to the FIFOs:
319	*/
320	fifo_regs = 4 * 16;
321
322	if (fifo_size < fifo_regs) {
323	DPRINTF("Too little FIFO\n");
324	return (EINVAL);
325	}
326
327	/* subtract FIFO regs from total once */
328	fifo_size -= fifo_regs;
329
330	/* split equally for IN and OUT */
331	fifo_size /= 2;
332
333	/* Align to 4 bytes boundary (refer to PGM) */
334	fifo_size &= ~3;
335
336	/* set global receive FIFO size */
337	DWC_OTG_WRITE_4(sc, DOTG_GRXFSIZ, fifo_size / 4);
338
339	tx_start = fifo_size;
340
341	if (fifo_size < 64) {
342	DPRINTFN(-1, "Not enough data space for EP0 FIFO.\n");
343	return (EINVAL);
344	}
345
346	if (mode == DWC_MODE_HOST) {
347
348	/* reset active endpoints */
349	sc->sc_active_rx_ep = 0;
350
351	/* split equally for periodic and non-periodic */
352	fifo_size /= 2;
353
354	DPRINTF("PTX/NPTX FIFO=%u\n", fifo_size);
355
356	/* align to 4 bytes boundary */
357	fifo_size &= ~3;
358
359	DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
360	((fifo_size / 4) << 16) \|
361	(tx_start / 4));
362
363	tx_start += fifo_size;
364
365	for (x = 0; x != sc->sc_host_ch_max; x++) {
366	/* enable all host interrupts */
367	DWC_OTG_WRITE_4(sc, DOTG_HCINTMSK(x),
368	HCINT_DEFAULT_MASK);
369	}
370
371	DWC_OTG_WRITE_4(sc, DOTG_HPTXFSIZ,
372	((fifo_size / 4) << 16) \|
373	(tx_start / 4));
374
375	/* reset host channel state */
376	memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
377
378	/* enable all host channel interrupts */
379	DWC_OTG_WRITE_4(sc, DOTG_HAINTMSK,
380	(1U << sc->sc_host_ch_max) - 1U);
381	}
382
383	if (mode == DWC_MODE_DEVICE) {
384
385	DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
386	(0x10 << 16) \| (tx_start / 4));
387	fifo_size -= 0x40;
388	tx_start += 0x40;
389
390	/* setup control endpoint profile */
391	sc->sc_hw_ep_profile[0].usb = dwc_otg_ep_profile[0];
392
393	/* reset active endpoints */
394	sc->sc_active_rx_ep = 1;
395
396	for (x = 1; x != sc->sc_dev_ep_max; x++) {
397
398	pf = sc->sc_hw_ep_profile + x;
399
400	pf->usb.max_out_frame_size = 1024 * 3;
401	pf->usb.is_simplex = 0; /* assume duplex */
402	pf->usb.support_bulk = 1;
403	pf->usb.support_interrupt = 1;
404	pf->usb.support_isochronous = 1;
405	pf->usb.support_out = 1;
406
407	if (x < sc->sc_dev_in_ep_max) {
408	uint32_t limit;
409
410	limit = (x == 1) ? MIN(DWC_OTG_TX_MAX_FIFO_SIZE,
411	DWC_OTG_MAX_TXN) : MIN(DWC_OTG_MAX_TXN / 2,
412	DWC_OTG_TX_MAX_FIFO_SIZE);
413
414	/* see if there is enough FIFO space */
415	if (limit <= fifo_size) {
416	pf->max_buffer = limit;
417	pf->usb.support_in = 1;
418	} else {
419	limit = MIN(DWC_OTG_TX_MAX_FIFO_SIZE, 0x40);
420	if (limit <= fifo_size) {
421	pf->usb.support_in = 1;
422	} else {
423	pf->usb.is_simplex = 1;
424	limit = 0;
425	}
426	}
427	/* set FIFO size */
428	DWC_OTG_WRITE_4(sc, DOTG_DIEPTXF(x),
429	((limit / 4) << 16) \| (tx_start / 4));
430	tx_start += limit;
431	fifo_size -= limit;
432	pf->usb.max_in_frame_size = limit;
433	} else {
434	pf->usb.is_simplex = 1;
435	}
436
437	DPRINTF("FIFO%d = IN:%d / OUT:%d\n", x,
438	pf->usb.max_in_frame_size,
439	pf->usb.max_out_frame_size);
440	}
441	}
442
443	/* reset RX FIFO */
444	dwc_otg_tx_fifo_reset(sc, GRSTCTL_RXFFLSH);
445
446	if (mode != DWC_MODE_OTG) {
447	/* reset all TX FIFOs */
448	dwc_otg_tx_fifo_reset(sc,
449	GRSTCTL_TXFIFO(0x10) \|
450	GRSTCTL_TXFFLSH);
451	} else {
452	/* reset active endpoints */
453	sc->sc_active_rx_ep = 0;
454
455	/* reset host channel state */
456	memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
457	}
458	return (0);
459	}
460
461	static uint8_t
462	dwc_otg_uses_split(struct usb_device *udev)
463	{
464	/*
465	* When a LOW or FULL speed device is connected directly to
466	* the USB port we don't use split transactions:
467	*/
468	return (udev->speed != USB_SPEED_HIGH &&
469	udev->parent_hs_hub != NULL &&
470	udev->parent_hs_hub->parent_hub != NULL);
471	}
472
473	static void
474	dwc_otg_update_host_frame_interval(struct dwc_otg_softc *sc)
475	{
476
477	/*
478	* Disabled until further. Assuming that the register is already
479	* programmed correctly by the boot loader.
480	*/
481	#if 0
482	uint32_t temp;
483
484	/* setup HOST frame interval register, based on existing value */
485	temp = DWC_OTG_READ_4(sc, DOTG_HFIR) & HFIR_FRINT_MASK;
486	if (temp >= 10000)
487	temp /= 1000;
488	else
489	temp /= 125;
490
491	/* figure out nearest X-tal value */
492	if (temp >= 54)
493	temp = 60; /* MHz */
494	else if (temp >= 39)
495	temp = 48; /* MHz */
496	else
497	temp = 30; /* MHz */
498
499	if (sc->sc_flags.status_high_speed)
500	temp *= 125;
501	else
502	temp *= 1000;
503
504	DPRINTF("HFIR=0x%08x\n", temp);
505
506	DWC_OTG_WRITE_4(sc, DOTG_HFIR, temp);
507	#endif
508	}
509
510	static void
511	dwc_otg_clocks_on(struct dwc_otg_softc *sc)
512	{
513	if (sc->sc_flags.clocks_off &&
514	sc->sc_flags.port_powered) {
515
516	DPRINTFN(5, "\n");
517
518	/* TODO - platform specific */
519
520	sc->sc_flags.clocks_off = 0;
521	}
522	}
523
524	static void
525	dwc_otg_clocks_off(struct dwc_otg_softc *sc)
526	{
527	if (!sc->sc_flags.clocks_off) {
528
529	DPRINTFN(5, "\n");
530
531	/* TODO - platform specific */
532
533	sc->sc_flags.clocks_off = 1;
534	}
535	}
536
537	static void
538	dwc_otg_pull_up(struct dwc_otg_softc *sc)
539	{
540	uint32_t temp;
541
542	/* pullup D+, if possible */
543
544	if (!sc->sc_flags.d_pulled_up &&
545	sc->sc_flags.port_powered) {
546	sc->sc_flags.d_pulled_up = 1;
547
548	temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
549	temp &= ~DCTL_SFTDISCON;
550	DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
551	}
552	}
553
554	static void
555	dwc_otg_pull_down(struct dwc_otg_softc *sc)
556	{
557	uint32_t temp;
558
559	/* pulldown D+, if possible */
560
561	if (sc->sc_flags.d_pulled_up) {
562	sc->sc_flags.d_pulled_up = 0;
563
564	temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
565	temp \|= DCTL_SFTDISCON;
566	DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
567	}
568	}
569
570	static void
571	dwc_otg_enable_sof_irq(struct dwc_otg_softc *sc)
572	{
573	/* In device mode we don't use the SOF interrupt */
574	if (sc->sc_flags.status_device_mode != 0)
575	return;
576	/* Ensure the SOF interrupt is not disabled */
577	sc->sc_needsof = 1;
578	/* Check if the SOF interrupt is already enabled */
579	if ((sc->sc_irq_mask & GINTMSK_SOFMSK) != 0)
580	return;
581	sc->sc_irq_mask \|= GINTMSK_SOFMSK;
582	DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
583	}
584
585	static void
586	dwc_otg_resume_irq(struct dwc_otg_softc *sc)
587	{
588	if (sc->sc_flags.status_suspend) {
589	/* update status bits */
590	sc->sc_flags.status_suspend = 0;
591	sc->sc_flags.change_suspend = 1;
592
593	if (sc->sc_flags.status_device_mode) {
594	/*
595	* Disable resume interrupt and enable suspend
596	* interrupt:
597	*/
598	sc->sc_irq_mask &= ~GINTMSK_WKUPINTMSK;
599	sc->sc_irq_mask \|= GINTMSK_USBSUSPMSK;
600	DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
601	}
602
603	/* complete root HUB interrupt endpoint */
604	dwc_otg_root_intr(sc);
605	}
606	}
607
608	static void
609	dwc_otg_suspend_irq(struct dwc_otg_softc *sc)
610	{
611	if (!sc->sc_flags.status_suspend) {
612	/* update status bits */
613	sc->sc_flags.status_suspend = 1;
614	sc->sc_flags.change_suspend = 1;
615
616	if (sc->sc_flags.status_device_mode) {
617	/*
618	* Disable suspend interrupt and enable resume
619	* interrupt:
620	*/
621	sc->sc_irq_mask &= ~GINTMSK_USBSUSPMSK;
622	sc->sc_irq_mask \|= GINTMSK_WKUPINTMSK;
623	DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
624	}
625
626	/* complete root HUB interrupt endpoint */
627	dwc_otg_root_intr(sc);
628	}
629	}
630
631	static void
632	dwc_otg_wakeup_peer(struct dwc_otg_softc *sc)
633	{
634	if (!sc->sc_flags.status_suspend)
635	return;
636
637	DPRINTFN(5, "Remote wakeup\n");
638
639	if (sc->sc_flags.status_device_mode) {
640	uint32_t temp;
641
642	/* enable remote wakeup signalling */
643	temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
644	temp \|= DCTL_RMTWKUPSIG;
645	DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
646
647	/* Wait 8ms for remote wakeup to complete. */
648	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 125);
649
650	temp &= ~DCTL_RMTWKUPSIG;
651	DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
652	} else {
653	/* enable USB port */
654	DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
655
656	/* wait 10ms */
657	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
658
659	/* resume port */
660	sc->sc_hprt_val \|= HPRT_PRTRES;
661	DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
662
663	/* Wait 100ms for resume signalling to complete. */
664	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 10);
665
666	/* clear suspend and resume */
667	sc->sc_hprt_val &= ~(HPRT_PRTSUSP \| HPRT_PRTRES);
668	DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
669
670	/* Wait 4ms */
671	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
672	}
673
674	/* need to fake resume IRQ */
675	dwc_otg_resume_irq(sc);
676	}
677
678	static void
679	dwc_otg_set_address(struct dwc_otg_softc *sc, uint8_t addr)
680	{
681	uint32_t temp;
682
683	DPRINTFN(5, "addr=%d\n", addr);
684
685	temp = DWC_OTG_READ_4(sc, DOTG_DCFG);
686	temp &= ~DCFG_DEVADDR_SET(0x7F);
687	temp \|= DCFG_DEVADDR_SET(addr);
688	DWC_OTG_WRITE_4(sc, DOTG_DCFG, temp);
689	}
690
691	static void
692	dwc_otg_common_rx_ack(struct dwc_otg_softc *sc)
693	{
694	DPRINTFN(5, "RX status clear\n");
695
696	/* enable RX FIFO level interrupt */
697	sc->sc_irq_mask \|= GINTMSK_RXFLVLMSK;
698	DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
699
700	if (sc->sc_current_rx_bytes != 0) {
701	/* need to dump remaining data */
702	bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
703	sc->sc_current_rx_fifo, sc->sc_bounce_buffer,
704	sc->sc_current_rx_bytes / 4);
705	/* clear number of active bytes to receive */
706	sc->sc_current_rx_bytes = 0;
707	}
708	/* clear cached status */
709	sc->sc_last_rx_status = 0;
710	}
711
712	static void
713	dwc_otg_clear_hcint(struct dwc_otg_softc *sc, uint8_t x)
714	{
715	uint32_t hcint;
716
717	/* clear all pending interrupts */
718	hcint = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
719	DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), hcint);
720
721	/* clear buffered interrupts */
722	sc->sc_chan_state[x].hcint = 0;
723	}
724
725	static uint8_t
726	dwc_otg_host_check_tx_fifo_empty(struct dwc_otg_softc sc, struct dwc_otg_td td)
727	{
728	uint32_t temp;
729
730	temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
731
732	if (td->ep_type == UE_ISOCHRONOUS) {
733	/*
734	* NOTE: USB INTERRUPT transactions are executed like
735	* USB CONTROL transactions! See the setup standard
736	* chain function for more information.
737	*/
738	if (!(temp & GINTSTS_PTXFEMP)) {
739	DPRINTF("Periodic TX FIFO is not empty\n");
740	if (!(sc->sc_irq_mask & GINTMSK_PTXFEMPMSK)) {
741	sc->sc_irq_mask \|= GINTMSK_PTXFEMPMSK;
742	DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
743	}
744	return (1); /* busy */
745	}
746	} else {
747	if (!(temp & GINTSTS_NPTXFEMP)) {
748	DPRINTF("Non-periodic TX FIFO is not empty\n");
749	if (!(sc->sc_irq_mask & GINTMSK_NPTXFEMPMSK)) {
750	sc->sc_irq_mask \|= GINTMSK_NPTXFEMPMSK;
751	DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
752	}
753	return (1); /* busy */
754	}
755	}
756	return (0); /* ready for transmit */
757	}
758
759	static uint8_t
760	dwc_otg_host_channel_alloc(struct dwc_otg_softc *sc,
761	struct dwc_otg_td *td, uint8_t is_out)
762	{
763	uint8_t x;
764	uint8_t y;
765	uint8_t z;
766
767	if (td->channel[0] < DWC_OTG_MAX_CHANNELS)
768	return (0); /* already allocated */
769
770	/* check if device is suspended */
771	if (DWC_OTG_PC2UDEV(td->pc)->flags.self_suspended != 0)
772	return (1); /* busy - cannot transfer data */
773
774	/* compute needed TX FIFO size */
775	if (is_out != 0) {
776	if (dwc_otg_host_check_tx_fifo_empty(sc, td) != 0)
777	return (1); /* busy - cannot transfer data */
778	}
779	z = td->max_packet_count;
780	for (x = y = 0; x != sc->sc_host_ch_max; x++) {
781	/* check if channel is allocated */
782	if (sc->sc_chan_state[x].allocated != 0)
783	continue;
784	/* check if channel is still enabled */
785	if (sc->sc_chan_state[x].wait_halted != 0)
786	continue;
787	/* store channel number */
788	td->channel[y++] = x;
789	/* check if we got all channels */
790	if (y == z)
791	break;
792	}
793	if (y != z) {
794	/* reset channel variable */
795	td->channel[0] = DWC_OTG_MAX_CHANNELS;
796	td->channel[1] = DWC_OTG_MAX_CHANNELS;
797	td->channel[2] = DWC_OTG_MAX_CHANNELS;
798	/* wait a bit */
799	dwc_otg_enable_sof_irq(sc);
800	return (1); /* busy - not enough channels */
801	}
802
803	for (y = 0; y != z; y++) {
804	x = td->channel[y];
805
806	/* set allocated */
807	sc->sc_chan_state[x].allocated = 1;
808
809	/* set wait halted */
810	sc->sc_chan_state[x].wait_halted = 1;
811
812	/* clear interrupts */
813	dwc_otg_clear_hcint(sc, x);
814
815	DPRINTF("CH=%d HCCHAR=0x%08x "
816	"HCSPLT=0x%08x\n", x, td->hcchar, td->hcsplt);
817
818	/* set active channel */
819	sc->sc_active_rx_ep \|= (1 << x);
820	}
821	return (0); /* allocated */
822	}
823
824	static void
825	dwc_otg_host_channel_free_sub(struct dwc_otg_softc sc, struct dwc_otg_td td, uint8_t index)
826	{
827	uint32_t hcchar;
828	uint8_t x;
829
830	if (td->channel[index] >= DWC_OTG_MAX_CHANNELS)
831	return; /* already freed */
832
833	/* free channel */
834	x = td->channel[index];
835	td->channel[index] = DWC_OTG_MAX_CHANNELS;
836
837	DPRINTF("CH=%d\n", x);
838
839	/*
840	* We need to let programmed host channels run till complete
841	* else the host channel will stop functioning.
842	*/
843	sc->sc_chan_state[x].allocated = 0;
844
845	/* ack any pending messages */
846	if (sc->sc_last_rx_status != 0 &&
847	GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) == x) {
848	dwc_otg_common_rx_ack(sc);
849	}
850
851	/* clear active channel */
852	sc->sc_active_rx_ep &= ~(1 << x);
853
854	/* check if already halted */
855	if (sc->sc_chan_state[x].wait_halted == 0)
856	return;
857
858	/* disable host channel */
859	hcchar = DWC_OTG_READ_4(sc, DOTG_HCCHAR(x));
860	if (hcchar & HCCHAR_CHENA) {
861	DPRINTF("Halting channel %d\n", x);
862	DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(x),
863	hcchar \| HCCHAR_CHDIS);
864	/* don't write HCCHAR until the channel is halted */
865	} else {
866	sc->sc_chan_state[x].wait_halted = 0;
867	}
868	}
869
870	static void
871	dwc_otg_host_channel_free(struct dwc_otg_softc sc, struct dwc_otg_td td)
872	{
873	uint8_t x;
874	for (x = 0; x != td->max_packet_count; x++)
875	dwc_otg_host_channel_free_sub(sc, td, x);
876	}
877
878	static void
879	dwc_otg_host_dump_rx(struct dwc_otg_softc sc, struct dwc_otg_td td)
880	{
881	uint8_t x;
882	/* dump any pending messages */
883	if (sc->sc_last_rx_status == 0)
884	return;
885	for (x = 0; x != td->max_packet_count; x++) {
886	if (td->channel[x] >= DWC_OTG_MAX_CHANNELS \|\|
887	td->channel[x] != GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status))
888	continue;
889	dwc_otg_common_rx_ack(sc);
890	break;
891	}
892	}
893
894	static uint8_t
895	dwc_otg_host_setup_tx(struct dwc_otg_softc sc, struct dwc_otg_td td)
896	{
897	struct usb_device_request req __aligned(4);
898	uint32_t hcint;
899	uint32_t hcchar;
900	uint8_t delta;
901
902	dwc_otg_host_dump_rx(sc, td);
903
904	if (td->channel[0] < DWC_OTG_MAX_CHANNELS) {
905	hcint = sc->sc_chan_state[td->channel[0]].hcint;
906
907	DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
908	td->channel[0], td->state, hcint,
909	DWC_OTG_READ_4(sc, DOTG_HCCHAR(td->channel[0])),
910	DWC_OTG_READ_4(sc, DOTG_HCTSIZ(td->channel[0])));
911	} else {
912	hcint = 0;
913	goto check_state;
914	}
915
916	if (hcint & (HCINT_RETRY \|
917	HCINT_ACK \| HCINT_NYET)) {
918	/* give success bits priority over failure bits */
919	} else if (hcint & HCINT_STALL) {
920	DPRINTF("CH=%d STALL\n", td->channel[0]);
921	td->error_stall = 1;
922	td->error_any = 1;
923	goto complete;
924	} else if (hcint & HCINT_ERRORS) {
925	DPRINTF("CH=%d ERROR\n", td->channel[0]);
926	td->errcnt++;
927	if (td->hcsplt != 0 \|\| td->errcnt >= 3) {
928	td->error_any = 1;
929	goto complete;
930	}
931	}
932
933	if (hcint & (HCINT_ERRORS \| HCINT_RETRY \|
934	HCINT_ACK \| HCINT_NYET)) {
935	if (!(hcint & HCINT_ERRORS))
936	td->errcnt = 0;
937	}
938
939	check_state:
940	switch (td->state) {
941	case DWC_CHAN_ST_START:
942	goto send_pkt;
943
944	case DWC_CHAN_ST_WAIT_ANE:
945	if (hcint & (HCINT_RETRY \| HCINT_ERRORS)) {
946	td->did_nak = 1;
947	td->tt_scheduled = 0;
948	goto send_pkt;
949	} else if (hcint & (HCINT_ACK \| HCINT_NYET)) {
950	td->offset += td->tx_bytes;
951	td->remainder -= td->tx_bytes;
952	td->toggle = 1;
953	td->tt_scheduled = 0;
954	goto complete;
955	}
956	break;
957
958	case DWC_CHAN_ST_WAIT_S_ANE:
959	if (hcint & (HCINT_RETRY \| HCINT_ERRORS)) {
960	td->did_nak = 1;
961	td->tt_scheduled = 0;
962	goto send_pkt;
963	} else if (hcint & (HCINT_ACK \| HCINT_NYET)) {
964	goto send_cpkt;
965	}
966	break;
967
968	case DWC_CHAN_ST_WAIT_C_ANE:
969	if (hcint & HCINT_NYET) {
970	goto send_cpkt;
971	} else if (hcint & (HCINT_RETRY \| HCINT_ERRORS)) {
972	td->did_nak = 1;
973	td->tt_scheduled = 0;
974	goto send_pkt;
975	} else if (hcint & HCINT_ACK) {
976	td->offset += td->tx_bytes;
977	td->remainder -= td->tx_bytes;
978	td->toggle = 1;
979	goto complete;
980	}
981	break;
982
983	case DWC_CHAN_ST_WAIT_C_PKT:
984	goto send_cpkt;
985
986	default:
987	break;
988	}
989	goto busy;
990
991	send_pkt:
992	/* free existing channel, if any */
993	dwc_otg_host_channel_free(sc, td);
994
995	if (sizeof(req) != td->remainder) {
996	td->error_any = 1;
997	goto complete;
998	}
999
1000	if (td->hcsplt != 0) {
1001	delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1002	if (td->tt_scheduled == 0 \|\| delta < DWC_OTG_TT_SLOT_MAX) {
1003	td->state = DWC_CHAN_ST_START;
1004	goto busy;
1005	}
1006	delta = sc->sc_last_frame_num - td->tt_start_slot;
1007	if (delta > 5) {
1008	/* missed it */
1009	td->tt_scheduled = 0;
1010	td->state = DWC_CHAN_ST_START;
1011	goto busy;
1012	}
1013	}
1014
1015	/* allocate a new channel */
1016	if (dwc_otg_host_channel_alloc(sc, td, 1)) {
1017	td->state = DWC_CHAN_ST_START;
1018	goto busy;
1019	}
1020
1021	if (td->hcsplt != 0) {
1022	td->hcsplt &= ~HCSPLT_COMPSPLT;
1023	td->state = DWC_CHAN_ST_WAIT_S_ANE;
1024	} else {
1025	td->state = DWC_CHAN_ST_WAIT_ANE;
1026	}
1027
1028	/* copy out control request */
1029	usbd_copy_out(td->pc, 0, &req, sizeof(req));
1030
1031	DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel[0]),
1032	(sizeof(req) << HCTSIZ_XFERSIZE_SHIFT) \|
1033	(1 << HCTSIZ_PKTCNT_SHIFT) \|
1034	(HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
1035
1036	DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel[0]), td->hcsplt);
1037
1038	hcchar = td->hcchar;
1039	hcchar &= ~(HCCHAR_EPDIR_IN \| HCCHAR_EPTYPE_MASK);
1040	hcchar \|= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
1041
1042	/* must enable channel before writing data to FIFO */
1043	DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel[0]), hcchar);
1044
1045	/* transfer data into FIFO */
1046	bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
1047	DOTG_DFIFO(td->channel[0]), (uint32_t *)&req, sizeof(req) / 4);
1048
1049	/* wait until next slot before trying complete split */
1050	td->tt_complete_slot = sc->sc_last_frame_num + 1;
1051
1052	/* store number of bytes transmitted */
1053	td->tx_bytes = sizeof(req);
1054	goto busy;
1055
1056	send_cpkt:
1057	/* free existing channel, if any */
1058	dwc_otg_host_channel_free(sc, td);
1059
1060	delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1061	if (td->tt_scheduled == 0 \|\| delta < DWC_OTG_TT_SLOT_MAX) {
1062	td->state = DWC_CHAN_ST_WAIT_C_PKT;
1063	goto busy;
1064	}
1065	delta = sc->sc_last_frame_num - td->tt_start_slot;
1066	if (delta > DWC_OTG_TT_SLOT_MAX) {
1067	/* we missed the service interval */
1068	if (td->ep_type != UE_ISOCHRONOUS)
1069	td->error_any = 1;
1070	goto complete;
1071	}
1072	/* allocate a new channel */
1073	if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1074	td->state = DWC_CHAN_ST_WAIT_C_PKT;
1075	goto busy;
1076	}
1077
1078	/* wait until next slot before trying complete split */
1079	td->tt_complete_slot = sc->sc_last_frame_num + 1;
1080
1081	td->hcsplt \|= HCSPLT_COMPSPLT;
1082	td->state = DWC_CHAN_ST_WAIT_C_ANE;
1083
1084	DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel[0]),
1085	(HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
1086
1087	DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel[0]), td->hcsplt);
1088
1089	hcchar = td->hcchar;
1090	hcchar &= ~(HCCHAR_EPDIR_IN \| HCCHAR_EPTYPE_MASK);
1091	hcchar \|= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
1092
1093	/* must enable channel before writing data to FIFO */
1094	DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel[0]), hcchar);
1095
1096	busy:
1097	return (1); /* busy */
1098
1099	complete:
1100	dwc_otg_host_channel_free(sc, td);
1101	return (0); /* complete */
1102	}
1103
1104	static uint8_t
1105	dwc_otg_setup_rx(struct dwc_otg_softc sc, struct dwc_otg_td td)
1106	{
1107	struct usb_device_request req __aligned(4);
1108	uint32_t temp;
1109	uint16_t count;
1110
1111	/* check endpoint status */
1112
1113	if (sc->sc_last_rx_status == 0)
1114	goto not_complete;
1115
1116	if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != 0)
1117	goto not_complete;
1118
1119	if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1120	GRXSTSRD_STP_DATA) {
1121	if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1122	GRXSTSRD_STP_COMPLETE \|\| td->remainder != 0) {
1123	/* release FIFO */
1124	dwc_otg_common_rx_ack(sc);
1125	goto not_complete;
1126	}
1127	/* release FIFO */
1128	dwc_otg_common_rx_ack(sc);
1129	return (0); /* complete */
1130	}
1131
1132	if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
1133	GRXSTSRD_DPID_DATA0) {
1134	/* release FIFO */
1135	dwc_otg_common_rx_ack(sc);
1136	goto not_complete;
1137	}
1138
1139	DPRINTFN(5, "GRXSTSR=0x%08x\n", sc->sc_last_rx_status);
1140
1141	/* clear did stall */
1142	td->did_stall = 0;
1143
1144	/* get the packet byte count */
1145	count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1146
1147	if (count != sizeof(req)) {
1148	DPRINTFN(0, "Unsupported SETUP packet "
1149	"length, %d bytes\n", count);
1150	/* release FIFO */
1151	dwc_otg_common_rx_ack(sc);
1152	goto not_complete;
1153	}
1154
1155	/* read FIFO */
1156	dwc_otg_read_fifo(sc, td->pc, 0, sizeof(req));
1157
1158	/* copy out control request */
1159	usbd_copy_out(td->pc, 0, &req, sizeof(req));
1160
1161	td->offset = sizeof(req);
1162	td->remainder = 0;
1163
1164	/* sneak peek the set address */
1165	if ((req.bmRequestType == UT_WRITE_DEVICE) &&
1166	(req.bRequest == UR_SET_ADDRESS)) {
1167	/* must write address before ZLP */
1168	dwc_otg_set_address(sc, req.wValue[0] & 0x7F);
1169	}
1170
1171	/* don't send any data by default */
1172	DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(0), DIEPCTL_EPDIS);
1173	DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0), DOEPCTL_EPDIS);
1174
1175	/* reset IN endpoint buffer */
1176	dwc_otg_tx_fifo_reset(sc,
1177	GRSTCTL_TXFIFO(0) \|
1178	GRSTCTL_TXFFLSH);
1179
1180	/* acknowledge RX status */
1181	dwc_otg_common_rx_ack(sc);
1182	td->did_stall = 1;
1183
1184	not_complete:
1185	/* abort any ongoing transfer, before enabling again */
1186	if (!td->did_stall) {
1187	td->did_stall = 1;
1188
1189	DPRINTFN(5, "stalling IN and OUT direction\n");
1190
1191	temp = sc->sc_out_ctl[0];
1192
1193	/* set stall after enabling endpoint */
1194	DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(0),
1195	temp \| DOEPCTL_STALL);
1196
1197	temp = sc->sc_in_ctl[0];
1198
1199	/* set stall assuming endpoint is enabled */
1200	DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(0),
1201	temp \| DIEPCTL_STALL);
1202	}
1203	return (1); /* not complete */
1204	}
1205
1206	static uint8_t
1207	dwc_otg_host_rate_check_interrupt(struct dwc_otg_softc sc, struct dwc_otg_td td)
1208	{
1209	uint8_t delta;
1210
1211	delta = sc->sc_tmr_val - td->tmr_val;
1212	if (delta >= 128)
1213	return (1); /* busy */
1214
1215	td->tmr_val = sc->sc_tmr_val + td->tmr_res;
1216
1217	/* set toggle, if any */
1218	if (td->set_toggle) {
1219	td->set_toggle = 0;
1220	td->toggle = 1;
1221	}
1222	return (0);
1223	}
1224
1225	static uint8_t
1226	dwc_otg_host_rate_check(struct dwc_otg_softc sc, struct dwc_otg_td td)
1227	{
1228	uint8_t frame_num = (uint8_t)sc->sc_last_frame_num;
1229
1230	if (td->ep_type == UE_ISOCHRONOUS) {
1231	/* non TT isochronous traffic */
1232	if (frame_num & (td->tmr_res - 1))
1233	goto busy;
1234	if ((frame_num ^ td->tmr_val) & td->tmr_res)
1235	goto busy;
1236	td->tmr_val = td->tmr_res + sc->sc_last_frame_num;
1237	td->toggle = 0;
1238	return (0);
1239	} else if (td->ep_type == UE_INTERRUPT) {
1240	if (!td->tt_scheduled)
1241	goto busy;
1242	td->tt_scheduled = 0;
1243	return (0);
1244	} else if (td->did_nak != 0) {
1245	/* check if we should pause sending queries for 125us */
1246	if (td->tmr_res == frame_num) {
1247	/* wait a bit */
1248	dwc_otg_enable_sof_irq(sc);
1249	goto busy;
1250	}
1251	} else if (td->set_toggle) {
1252	td->set_toggle = 0;
1253	td->toggle = 1;
1254	}
1255	/* query for data one more time */
1256	td->tmr_res = frame_num;
1257	td->did_nak = 0;
1258	return (0);
1259	busy:
1260	return (1);
1261	}
1262
1263	static uint8_t
1264	dwc_otg_host_data_rx_sub(struct dwc_otg_softc sc, struct dwc_otg_td td,
1265	uint8_t channel)
1266	{
1267	uint32_t count;
1268
1269	/* check endpoint status */
1270	if (sc->sc_last_rx_status == 0)
1271	goto busy;
1272
1273	if (channel >= DWC_OTG_MAX_CHANNELS)
1274	goto busy;
1275
1276	if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != channel)
1277	goto busy;
1278
1279	switch (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) {
1280	case GRXSTSRH_IN_DATA:
1281
1282	DPRINTF("DATA ST=%d STATUS=0x%08x\n",
1283	(int)td->state, (int)sc->sc_last_rx_status);
1284
1285	if (sc->sc_chan_state[channel].hcint & HCINT_SOFTWARE_ONLY) {
1286	/*
1287	* When using SPLIT transactions on interrupt
1288	* endpoints, sometimes data occurs twice.
1289	*/
1290	DPRINTF("Data already received\n");
1291	break;
1292	}
1293
1294	/* get the packet byte count */
1295	count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1296
1297	/* check for ISOCHRONOUS endpoint */
1298	if (td->ep_type == UE_ISOCHRONOUS) {
1299	if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
1300	GRXSTSRD_DPID_DATA0) {
1301	/* more data to be received */
1302	td->tt_xactpos = HCSPLT_XACTPOS_MIDDLE;
1303	} else {
1304	/* all data received */
1305	td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
1306	/* verify the packet byte count */
1307	if (count != td->remainder) {
1308	/* we have a short packet */
1309	td->short_pkt = 1;
1310	td->got_short = 1;
1311	}
1312	}
1313	} else {
1314	/* verify the packet byte count */
1315	if (count != td->max_packet_size) {
1316	if (count < td->max_packet_size) {
1317	/* we have a short packet */
1318	td->short_pkt = 1;
1319	td->got_short = 1;
1320	} else {
1321	/* invalid USB packet */
1322	td->error_any = 1;
1323
1324	/* release FIFO */
1325	dwc_otg_common_rx_ack(sc);
1326	goto complete;
1327	}
1328	}
1329	td->toggle ^= 1;
1330	td->tt_scheduled = 0;
1331	}
1332
1333	/* verify the packet byte count */
1334	if (count > td->remainder) {
1335	/* invalid USB packet */
1336	td->error_any = 1;
1337
1338	/* release FIFO */
1339	dwc_otg_common_rx_ack(sc);
1340	goto complete;
1341	}
1342
1343	/* read data from FIFO */
1344	dwc_otg_read_fifo(sc, td->pc, td->offset, count);
1345
1346	td->remainder -= count;
1347	td->offset += count;
1348	sc->sc_chan_state[channel].hcint \|= HCINT_SOFTWARE_ONLY;
1349	break;
1350	default:
1351	break;
1352	}
1353	/* release FIFO */
1354	dwc_otg_common_rx_ack(sc);
1355	busy:
1356	return (0);
1357	complete:
1358	return (1);
1359	}
1360
1361	static uint8_t
1362	dwc_otg_host_data_rx(struct dwc_otg_softc sc, struct dwc_otg_td td)
1363	{
1364	uint32_t hcint = 0;
1365	uint32_t hcchar;
1366	uint8_t delta;
1367	uint8_t channel;
1368	uint8_t x;
1369
1370	for (x = 0; x != td->max_packet_count; x++) {
1371	channel = td->channel[x];
1372	if (channel >= DWC_OTG_MAX_CHANNELS)
1373	continue;
1374	hcint \|= sc->sc_chan_state[channel].hcint;
1375
1376	DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1377	channel, td->state, hcint,
1378	DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1379	DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1380
1381	/* check interrupt bits */
1382	if (hcint & (HCINT_RETRY \|
1383	HCINT_ACK \| HCINT_NYET)) {
1384	/* give success bits priority over failure bits */
1385	} else if (hcint & HCINT_STALL) {
1386	DPRINTF("CH=%d STALL\n", channel);
1387	td->error_stall = 1;
1388	td->error_any = 1;
1389	goto complete;
1390	} else if (hcint & HCINT_ERRORS) {
1391	DPRINTF("CH=%d ERROR\n", channel);
1392	td->errcnt++;
1393	if (td->hcsplt != 0 \|\| td->errcnt >= 3) {
1394	if (td->ep_type != UE_ISOCHRONOUS) {
1395	td->error_any = 1;
1396	goto complete;
1397	}
1398	}
1399	}
1400
1401	/* check channels for data, if any */
1402	if (dwc_otg_host_data_rx_sub(sc, td, channel))
1403	goto complete;
1404
1405	/* refresh interrupt status */
1406	hcint \|= sc->sc_chan_state[channel].hcint;
1407
1408	if (hcint & (HCINT_ERRORS \| HCINT_RETRY \|
1409	HCINT_ACK \| HCINT_NYET)) {
1410	if (!(hcint & HCINT_ERRORS))
1411	td->errcnt = 0;
1412	}
1413	}
1414
1415	switch (td->state) {
1416	case DWC_CHAN_ST_START:
1417	if (td->hcsplt != 0)
1418	goto receive_spkt;
1419	else
1420	goto receive_pkt;
1421
1422	case DWC_CHAN_ST_WAIT_ANE:
1423	if (hcint & (HCINT_RETRY \| HCINT_ERRORS)) {
1424	if (td->ep_type == UE_INTERRUPT) {
1425	/*
1426	* The USB specification does not
1427	* mandate a particular data toggle
1428	* value for USB INTERRUPT
1429	* transfers. Switch the data toggle
1430	* value to receive the packet
1431	* correctly:
1432	*/
1433	if (hcint & HCINT_DATATGLERR) {
1434	DPRINTF("Retrying packet due to "
1435	"data toggle error\n");
1436	td->toggle ^= 1;
1437	goto receive_pkt;
1438	}
1439	} else if (td->ep_type == UE_ISOCHRONOUS) {
1440	goto complete;
1441	}
1442	td->did_nak = 1;
1443	td->tt_scheduled = 0;
1444	if (td->hcsplt != 0)
1445	goto receive_spkt;
1446	else
1447	goto receive_pkt;
1448	} else if (hcint & HCINT_NYET) {
1449	if (td->hcsplt != 0) {
1450	/* try again */
1451	goto receive_pkt;
1452	} else {
1453	/* not a valid token for IN endpoints */
1454	td->error_any = 1;
1455	goto complete;
1456	}
1457	} else if (hcint & HCINT_ACK) {
1458	/* wait for data - ACK arrived first */
1459	if (!(hcint & HCINT_SOFTWARE_ONLY))
1460	goto busy;
1461
1462	if (td->ep_type == UE_ISOCHRONOUS) {
1463	/* check if we are complete */
1464	if (td->tt_xactpos == HCSPLT_XACTPOS_BEGIN) {
1465	goto complete;
1466	} else {
1467	/* get more packets */
1468	goto busy;
1469	}
1470	} else {
1471	/* check if we are complete */
1472	if ((td->remainder == 0) \|\| (td->got_short != 0)) {
1473	if (td->short_pkt)
1474	goto complete;
1475
1476	/*
1477	* Else need to receive a zero length
1478	* packet.
1479	*/
1480	}
1481	td->tt_scheduled = 0;
1482	td->did_nak = 0;
1483	if (td->hcsplt != 0)
1484	goto receive_spkt;
1485	else
1486	goto receive_pkt;
1487	}
1488	}
1489	break;
1490
1491	case DWC_CHAN_ST_WAIT_S_ANE:
1492	/*
1493	* NOTE: The DWC OTG hardware provides a fake ACK in
1494	* case of interrupt and isochronous transfers:
1495	*/
1496	if (hcint & (HCINT_RETRY \| HCINT_ERRORS)) {
1497	td->did_nak = 1;
1498	td->tt_scheduled = 0;
1499	goto receive_spkt;
1500	} else if (hcint & HCINT_NYET) {
1501	td->tt_scheduled = 0;
1502	goto receive_spkt;
1503	} else if (hcint & HCINT_ACK) {
1504	td->did_nak = 0;
1505	goto receive_pkt;
1506	}
1507	break;
1508
1509	case DWC_CHAN_ST_WAIT_C_PKT:
1510	goto receive_pkt;
1511
1512	default:
1513	break;
1514	}
1515	goto busy;
1516
1517	receive_pkt:
1518	/* free existing channel, if any */
1519	dwc_otg_host_channel_free(sc, td);
1520
1521	if (td->hcsplt != 0) {
1522	delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1523	if (td->tt_scheduled == 0 \|\| delta < DWC_OTG_TT_SLOT_MAX) {
1524	td->state = DWC_CHAN_ST_WAIT_C_PKT;
1525	goto busy;
1526	}
1527	delta = sc->sc_last_frame_num - td->tt_start_slot;
1528	if (delta > DWC_OTG_TT_SLOT_MAX) {
1529	if (td->ep_type != UE_ISOCHRONOUS) {
1530	/* we missed the service interval */
1531	td->error_any = 1;
1532	}
1533	goto complete;
1534	}
1535	/* complete split */
1536	td->hcsplt \|= HCSPLT_COMPSPLT;
1537	} else if (dwc_otg_host_rate_check(sc, td)) {
1538	td->state = DWC_CHAN_ST_WAIT_C_PKT;
1539	goto busy;
1540	}
1541
1542	/* allocate a new channel */
1543	if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1544	td->state = DWC_CHAN_ST_WAIT_C_PKT;
1545	goto busy;
1546	}
1547
1548	/* set toggle, if any */
1549	if (td->set_toggle) {
1550	td->set_toggle = 0;
1551	td->toggle = 1;
1552	}
1553
1554	td->state = DWC_CHAN_ST_WAIT_ANE;
1555
1556	for (x = 0; x != td->max_packet_count; x++) {
1557	channel = td->channel[x];
1558
1559	/* receive one packet */
1560	DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1561	(td->max_packet_size << HCTSIZ_XFERSIZE_SHIFT) \|
1562	(1 << HCTSIZ_PKTCNT_SHIFT) \|
1563	(td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1564	(HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1565
1566	DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1567
1568	hcchar = td->hcchar;
1569	hcchar \|= HCCHAR_EPDIR_IN;
1570
1571	/* receive complete split ASAP */
1572	if ((sc->sc_last_frame_num & 1) != 0 &&
1573	td->ep_type == UE_ISOCHRONOUS)
1574	hcchar \|= HCCHAR_ODDFRM;
1575	else
1576	hcchar &= ~HCCHAR_ODDFRM;
1577
1578	/* must enable channel before data can be received */
1579	DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1580	}
1581	/* wait until next slot before trying complete split */
1582	td->tt_complete_slot = sc->sc_last_frame_num + 1;
1583
1584	goto busy;
1585
1586	receive_spkt:
1587	/* free existing channel(s), if any */
1588	dwc_otg_host_channel_free(sc, td);
1589
1590	delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1591	if (td->tt_scheduled == 0 \|\| delta < DWC_OTG_TT_SLOT_MAX) {
1592	td->state = DWC_CHAN_ST_START;
1593	goto busy;
1594	}
1595	delta = sc->sc_last_frame_num - td->tt_start_slot;
1596	if (delta > 5) {
1597	/* missed it */
1598	td->tt_scheduled = 0;
1599	td->state = DWC_CHAN_ST_START;
1600	goto busy;
1601	}
1602
1603	/* allocate a new channel */
1604	if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1605	td->state = DWC_CHAN_ST_START;
1606	goto busy;
1607	}
1608
1609	channel = td->channel[0];
1610
1611	td->hcsplt &= ~HCSPLT_COMPSPLT;
1612	td->state = DWC_CHAN_ST_WAIT_S_ANE;
1613
1614	/* receive one packet */
1615	DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1616	(HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
1617
1618	DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1619
1620	/* send after next SOF event */
1621	if ((sc->sc_last_frame_num & 1) == 0 &&
1622	td->ep_type == UE_ISOCHRONOUS)
1623	td->hcchar \|= HCCHAR_ODDFRM;
1624	else
1625	td->hcchar &= ~HCCHAR_ODDFRM;
1626
1627	hcchar = td->hcchar;
1628	hcchar \|= HCCHAR_EPDIR_IN;
1629
1630	/* wait until next slot before trying complete split */
1631	td->tt_complete_slot = sc->sc_last_frame_num + 1;
1632
1633	/* must enable channel before data can be received */
1634	DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1635	busy:
1636	return (1); /* busy */
1637
1638	complete:
1639	dwc_otg_host_channel_free(sc, td);
1640	return (0); /* complete */
1641	}
1642
1643	static uint8_t
1644	dwc_otg_data_rx(struct dwc_otg_softc sc, struct dwc_otg_td td)
1645	{
1646	uint32_t temp;
1647	uint16_t count;
1648	uint8_t got_short;
1649
1650	got_short = 0;
1651
1652	/* check endpoint status */
1653	if (sc->sc_last_rx_status == 0)
1654	goto not_complete;
1655
1656	if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != td->ep_no)
1657	goto not_complete;
1658
1659	/* check for SETUP packet */
1660	if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1661	GRXSTSRD_STP_DATA \|\|
1662	(sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1663	GRXSTSRD_STP_COMPLETE) {
1664	if (td->remainder == 0) {
1665	/*
1666	* We are actually complete and have
1667	* received the next SETUP
1668	*/
1669	DPRINTFN(5, "faking complete\n");
1670	return (0); /* complete */
1671	}
1672	/*
1673	* USB Host Aborted the transfer.
1674	*/
1675	td->error_any = 1;
1676	return (0); /* complete */
1677	}
1678
1679	if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1680	GRXSTSRD_OUT_DATA) {
1681	/* release FIFO */
1682	dwc_otg_common_rx_ack(sc);
1683	goto not_complete;
1684	}
1685
1686	/* get the packet byte count */
1687	count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1688
1689	/* verify the packet byte count */
1690	if (count != td->max_packet_size) {
1691	if (count < td->max_packet_size) {
1692	/* we have a short packet */
1693	td->short_pkt = 1;
1694	got_short = 1;
1695	} else {
1696	/* invalid USB packet */
1697	td->error_any = 1;
1698
1699	/* release FIFO */
1700	dwc_otg_common_rx_ack(sc);
1701	return (0); /* we are complete */
1702	}
1703	}
1704	/* verify the packet byte count */
1705	if (count > td->remainder) {
1706	/* invalid USB packet */
1707	td->error_any = 1;
1708
1709	/* release FIFO */
1710	dwc_otg_common_rx_ack(sc);
1711	return (0); /* we are complete */
1712	}
1713
1714	/* read data from FIFO */
1715	dwc_otg_read_fifo(sc, td->pc, td->offset, count);
1716
1717	td->remainder -= count;
1718	td->offset += count;
1719
1720	/* release FIFO */
1721	dwc_otg_common_rx_ack(sc);
1722
1723	temp = sc->sc_out_ctl[td->ep_no];
1724
1725	/* check for isochronous mode */
1726	if ((temp & DIEPCTL_EPTYPE_MASK) ==
1727	(DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
1728	/* toggle odd or even frame bit */
1729	if (temp & DIEPCTL_SETD1PID) {
1730	temp &= ~DIEPCTL_SETD1PID;
1731	temp \|= DIEPCTL_SETD0PID;
1732	} else {
1733	temp &= ~DIEPCTL_SETD0PID;
1734	temp \|= DIEPCTL_SETD1PID;
1735	}
1736	sc->sc_out_ctl[td->ep_no] = temp;
1737	}
1738
1739	/* check if we are complete */
1740	if ((td->remainder == 0) \|\| got_short) {
1741	if (td->short_pkt) {
1742	/* we are complete */
1743	return (0);
1744	}
1745	/* else need to receive a zero length packet */
1746	}
1747
1748	not_complete:
1749
1750	/* enable SETUP and transfer complete interrupt */
1751	if (td->ep_no == 0) {
1752	DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0),
1753	DXEPTSIZ_SET_MULTI(3) \|
1754	DXEPTSIZ_SET_NPKT(1) \|
1755	DXEPTSIZ_SET_NBYTES(td->max_packet_size));
1756	} else {
1757	/* allow reception of multiple packets */
1758	DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(td->ep_no),
1759	DXEPTSIZ_SET_MULTI(1) \|
1760	DXEPTSIZ_SET_NPKT(4) \|
1761	DXEPTSIZ_SET_NBYTES(4 *
1762	((td->max_packet_size + 3) & ~3)));
1763	}
1764	temp = sc->sc_out_ctl[td->ep_no];
1765	DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(td->ep_no), temp \|
1766	DOEPCTL_EPENA \| DOEPCTL_CNAK);
1767
1768	return (1); /* not complete */
1769	}
1770
1771	static uint8_t
1772	dwc_otg_host_data_tx(struct dwc_otg_softc sc, struct dwc_otg_td td)
1773	{
1774	uint32_t count;
1775	uint32_t hcint;
1776	uint32_t hcchar;
1777	uint8_t delta;
1778	uint8_t channel;
1779	uint8_t x;
1780
1781	dwc_otg_host_dump_rx(sc, td);
1782
1783	/* check that last channel is complete */
1784	channel = td->channel[td->npkt];
1785
1786	if (channel < DWC_OTG_MAX_CHANNELS) {
1787	hcint = sc->sc_chan_state[channel].hcint;
1788
1789	DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1790	channel, td->state, hcint,
1791	DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1792	DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1793
1794	if (hcint & (HCINT_RETRY \|
1795	HCINT_ACK \| HCINT_NYET)) {
1796	/* give success bits priority over failure bits */
1797	} else if (hcint & HCINT_STALL) {
1798	DPRINTF("CH=%d STALL\n", channel);
1799	td->error_stall = 1;
1800	td->error_any = 1;
1801	goto complete;
1802	} else if (hcint & HCINT_ERRORS) {
1803	DPRINTF("CH=%d ERROR\n", channel);
1804	td->errcnt++;
1805	if (td->hcsplt != 0 \|\| td->errcnt >= 3) {
1806	td->error_any = 1;
1807	goto complete;
1808	}
1809	}
1810
1811	if (hcint & (HCINT_ERRORS \| HCINT_RETRY \|
1812	HCINT_ACK \| HCINT_NYET)) {
1813
1814	if (!(hcint & HCINT_ERRORS))
1815	td->errcnt = 0;
1816	}
1817	} else {
1818	hcint = 0;
1819	}
1820
1821	switch (td->state) {
1822	case DWC_CHAN_ST_START:
1823	goto send_pkt;
1824
1825	case DWC_CHAN_ST_WAIT_ANE:
1826	if (hcint & (HCINT_RETRY \| HCINT_ERRORS)) {
1827	td->did_nak = 1;
1828	td->tt_scheduled = 0;
1829	goto send_pkt;
1830	} else if (hcint & (HCINT_ACK \| HCINT_NYET)) {
1831	td->offset += td->tx_bytes;
1832	td->remainder -= td->tx_bytes;
1833	td->toggle ^= 1;
1834	/* check if next response will be a NAK */
1835	if (hcint & HCINT_NYET)
1836	td->did_nak = 1;
1837	else
1838	td->did_nak = 0;
1839	td->tt_scheduled = 0;
1840
1841	/* check remainder */
1842	if (td->remainder == 0) {
1843	if (td->short_pkt)
1844	goto complete;
1845
1846	/*
1847	* Else we need to transmit a short
1848	* packet:
1849	*/
1850	}
1851	goto send_pkt;
1852	}
1853	break;
1854
1855	case DWC_CHAN_ST_WAIT_S_ANE:
1856	if (hcint & (HCINT_RETRY \| HCINT_ERRORS)) {
1857	td->did_nak = 1;
1858	td->tt_scheduled = 0;
1859	goto send_pkt;
1860	} else if (hcint & (HCINT_ACK \| HCINT_NYET)) {
1861	td->did_nak = 0;
1862	goto send_cpkt;
1863	}
1864	break;
1865
1866	case DWC_CHAN_ST_WAIT_C_ANE:
1867	if (hcint & HCINT_NYET) {
1868	goto send_cpkt;
1869	} else if (hcint & (HCINT_RETRY \| HCINT_ERRORS)) {
1870	td->did_nak = 1;
1871	td->tt_scheduled = 0;
1872	goto send_pkt;
1873	} else if (hcint & HCINT_ACK) {
1874	td->offset += td->tx_bytes;
1875	td->remainder -= td->tx_bytes;
1876	td->toggle ^= 1;
1877	td->did_nak = 0;
1878	td->tt_scheduled = 0;
1879
1880	/* check remainder */
1881	if (td->remainder == 0) {
1882	if (td->short_pkt)
1883	goto complete;
1884
1885	/* else we need to transmit a short packet */
1886	}
1887	goto send_pkt;
1888	}
1889	break;
1890
1891	case DWC_CHAN_ST_WAIT_C_PKT:
1892	goto send_cpkt;
1893
1894	case DWC_CHAN_ST_TX_WAIT_ISOC:
1895	/* Check if ISOCHRONOUS OUT traffic is complete */
1896	if ((hcint & HCINT_HCH_DONE_MASK) == 0)
1897	break;
1898
1899	td->offset += td->tx_bytes;
1900	td->remainder -= td->tx_bytes;
1901	goto complete;
1902	default:
1903	break;
1904	}
1905	goto busy;
1906
1907	send_pkt:
1908	/* free existing channel(s), if any */
1909	dwc_otg_host_channel_free(sc, td);
1910
1911	if (td->hcsplt != 0) {
1912	delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1913	if (td->tt_scheduled == 0 \|\| delta < DWC_OTG_TT_SLOT_MAX) {
1914	td->state = DWC_CHAN_ST_START;
1915	goto busy;
1916	}
1917	delta = sc->sc_last_frame_num - td->tt_start_slot;
1918	if (delta > 5) {
1919	/* missed it */
1920	td->tt_scheduled = 0;
1921	td->state = DWC_CHAN_ST_START;
1922	goto busy;
1923	}
1924	} else if (dwc_otg_host_rate_check(sc, td)) {
1925	td->state = DWC_CHAN_ST_START;
1926	goto busy;
1927	}
1928
1929	/* allocate a new channel */
1930	if (dwc_otg_host_channel_alloc(sc, td, 1)) {
1931	td->state = DWC_CHAN_ST_START;
1932	goto busy;
1933	}
1934
1935	/* set toggle, if any */
1936	if (td->set_toggle) {
1937	td->set_toggle = 0;
1938	td->toggle = 1;
1939	}
1940
1941	if (td->ep_type == UE_ISOCHRONOUS) {
1942	/* ISOCHRONOUS OUT transfers don't have any ACKs */
1943	td->state = DWC_CHAN_ST_TX_WAIT_ISOC;
1944	td->hcsplt &= ~HCSPLT_COMPSPLT;
1945	if (td->hcsplt != 0) {
1946	/* get maximum transfer length */
1947	count = td->remainder;
1948	if (count > HCSPLT_XACTLEN_BURST) {
1949	DPRINTF("TT overflow\n");
1950	td->error_any = 1;
1951	goto complete;
1952	}
1953	/* Update transaction position */
1954	td->hcsplt &= ~HCSPLT_XACTPOS_MASK;
1955	td->hcsplt \|= (HCSPLT_XACTPOS_ALL << HCSPLT_XACTPOS_SHIFT);
1956	}
1957	} else if (td->hcsplt != 0) {
1958	td->hcsplt &= ~HCSPLT_COMPSPLT;
1959	/* Wait for ACK/NAK/ERR from TT */
1960	td->state = DWC_CHAN_ST_WAIT_S_ANE;
1961	} else {
1962	/* Wait for ACK/NAK/STALL from device */
1963	td->state = DWC_CHAN_ST_WAIT_ANE;
1964	}
1965
1966	td->tx_bytes = 0;
1967
1968	for (x = 0; x != td->max_packet_count; x++) {
1969	uint32_t rem_bytes;
1970
1971	channel = td->channel[x];
1972
1973	/* send one packet at a time */
1974	count = td->max_packet_size;
1975	rem_bytes = td->remainder - td->tx_bytes;
1976	if (rem_bytes < count) {
1977	/* we have a short packet */
1978	td->short_pkt = 1;
1979	count = rem_bytes;
1980	}
1981	if (count == rem_bytes) {
1982	/* last packet */
1983	switch (x) {
1984	case 0:
1985	DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1986	(count << HCTSIZ_XFERSIZE_SHIFT) \|
1987	(1 << HCTSIZ_PKTCNT_SHIFT) \|
1988	(td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1989	(HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1990	break;
1991	case 1:
1992	DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1993	(count << HCTSIZ_XFERSIZE_SHIFT) \|
1994	(1 << HCTSIZ_PKTCNT_SHIFT) \|
1995	(HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT));
1996	break;
1997	default:
1998	DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1999	(count << HCTSIZ_XFERSIZE_SHIFT) \|
2000	(1 << HCTSIZ_PKTCNT_SHIFT) \|
2001	(HCTSIZ_PID_DATA2 << HCTSIZ_PID_SHIFT));
2002	break;
2003	}
2004	} else if (td->ep_type == UE_ISOCHRONOUS &&
2005	td->max_packet_count > 1) {
2006	/* ISOCHRONOUS multi packet */
2007	DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2008	(count << HCTSIZ_XFERSIZE_SHIFT) \|
2009	(1 << HCTSIZ_PKTCNT_SHIFT) \|
2010	(HCTSIZ_PID_MDATA << HCTSIZ_PID_SHIFT));
2011	} else {
2012	/* TODO: HCTSIZ_DOPNG */
2013	/* standard BULK/INTERRUPT/CONTROL packet */
2014	DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2015	(count << HCTSIZ_XFERSIZE_SHIFT) \|
2016	(1 << HCTSIZ_PKTCNT_SHIFT) \|
2017	(td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
2018	(HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
2019	}
2020
2021	DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
2022
2023	hcchar = td->hcchar;
2024	hcchar &= ~HCCHAR_EPDIR_IN;
2025
2026	/* send after next SOF event */
2027	if ((sc->sc_last_frame_num & 1) == 0 &&
2028	td->ep_type == UE_ISOCHRONOUS)
2029	hcchar \|= HCCHAR_ODDFRM;
2030	else
2031	hcchar &= ~HCCHAR_ODDFRM;
2032
2033	/* must enable before writing data to FIFO */
2034	DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
2035
2036	if (count != 0) {
2037	/* write data into FIFO */
2038	dwc_otg_write_fifo(sc, td->pc, td->offset +
2039	td->tx_bytes, DOTG_DFIFO(channel), count);
2040	}
2041
2042	/* store number of bytes transmitted */
2043	td->tx_bytes += count;
2044
2045	/* store last packet index */
2046	td->npkt = x;
2047
2048	/* check for last packet */
2049	if (count == rem_bytes)
2050	break;
2051	}
2052	goto busy;
2053
2054	send_cpkt:
2055	/* free existing channel, if any */
2056	dwc_otg_host_channel_free(sc, td);
2057
2058	delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
2059	if (td->tt_scheduled == 0 \|\| delta < DWC_OTG_TT_SLOT_MAX) {
2060	td->state = DWC_CHAN_ST_WAIT_C_PKT;
2061	goto busy;
2062	}
2063	delta = sc->sc_last_frame_num - td->tt_start_slot;
2064	if (delta > DWC_OTG_TT_SLOT_MAX) {
2065	/* we missed the service interval */
2066	if (td->ep_type != UE_ISOCHRONOUS)
2067	td->error_any = 1;
2068	goto complete;
2069	}
2070
2071	/* allocate a new channel */
2072	if (dwc_otg_host_channel_alloc(sc, td, 0)) {
2073	td->state = DWC_CHAN_ST_WAIT_C_PKT;
2074	goto busy;
2075	}
2076
2077	channel = td->channel[0];
2078
2079	td->hcsplt \|= HCSPLT_COMPSPLT;
2080	td->state = DWC_CHAN_ST_WAIT_C_ANE;
2081
2082	DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2083	(HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
2084
2085	DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
2086
2087	hcchar = td->hcchar;
2088	hcchar &= ~HCCHAR_EPDIR_IN;
2089
2090	/* receive complete split ASAP */
2091	if ((sc->sc_last_frame_num & 1) != 0 &&
2092	td->ep_type == UE_ISOCHRONOUS)
2093	hcchar \|= HCCHAR_ODDFRM;
2094	else
2095	hcchar &= ~HCCHAR_ODDFRM;
2096
2097	/* must enable channel before data can be received */
2098	DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
2099
2100	/* wait until next slot before trying complete split */
2101	td->tt_complete_slot = sc->sc_last_frame_num + 1;
2102	busy:
2103	return (1); /* busy */
2104
2105	complete:
2106	dwc_otg_host_channel_free(sc, td);
2107	return (0); /* complete */
2108	}
2109
2110	static uint8_t
2111	dwc_otg_data_tx(struct dwc_otg_softc sc, struct dwc_otg_td td)
2112	{
2113	uint32_t max_buffer;
2114	uint32_t count;
2115	uint32_t fifo_left;
2116	uint32_t mpkt;
2117	uint32_t temp;
2118	uint8_t to;
2119
2120	to = 3; /* don't loop forever! */
2121
2122	max_buffer = sc->sc_hw_ep_profile[td->ep_no].max_buffer;
2123
2124	repeat:
2125	/* check for for endpoint 0 data */
2126
2127	temp = sc->sc_last_rx_status;
2128
2129	if ((td->ep_no == 0) && (temp != 0) &&
2130	(GRXSTSRD_CHNUM_GET(temp) == 0)) {
2131
2132	if ((temp & GRXSTSRD_PKTSTS_MASK) !=
2133	GRXSTSRD_STP_DATA &&
2134	(temp & GRXSTSRD_PKTSTS_MASK) !=
2135	GRXSTSRD_STP_COMPLETE) {
2136
2137	/* dump data - wrong direction */
2138	dwc_otg_common_rx_ack(sc);
2139	} else {
2140	/*
2141	* The current transfer was cancelled
2142	* by the USB Host:
2143	*/
2144	td->error_any = 1;
2145	return (0); /* complete */
2146	}
2147	}
2148
2149	/* fill in more TX data, if possible */
2150	if (td->tx_bytes != 0) {
2151
2152	uint16_t cpkt;
2153
2154	/* check if packets have been transferred */
2155	temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2156
2157	/* get current packet number */
2158	cpkt = DXEPTSIZ_GET_NPKT(temp);
2159
2160	if (cpkt >= td->npkt) {
2161	fifo_left = 0;
2162	} else {
2163	if (max_buffer != 0) {
2164	fifo_left = (td->npkt - cpkt) *
2165	td->max_packet_size;
2166
2167	if (fifo_left > max_buffer)
2168	fifo_left = max_buffer;
2169	} else {
2170	fifo_left = td->max_packet_size;
2171	}
2172	}
2173
2174	count = td->tx_bytes;
2175	if (count > fifo_left)
2176	count = fifo_left;
2177
2178	if (count != 0) {
2179	/* write data into FIFO */
2180	dwc_otg_write_fifo(sc, td->pc, td->offset,
2181	DOTG_DFIFO(td->ep_no), count);
2182
2183	td->tx_bytes -= count;
2184	td->remainder -= count;
2185	td->offset += count;
2186	td->npkt = cpkt;
2187	}
2188	if (td->tx_bytes != 0)
2189	goto not_complete;
2190
2191	/* check remainder */
2192	if (td->remainder == 0) {
2193	if (td->short_pkt)
2194	return (0); /* complete */
2195
2196	/* else we need to transmit a short packet */
2197	}
2198	}
2199
2200	if (!to--)
2201	goto not_complete;
2202
2203	/* check if not all packets have been transferred */
2204	temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2205
2206	if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2207
2208	DPRINTFN(5, "busy ep=%d npkt=%d DIEPTSIZ=0x%08x "
2209	"DIEPCTL=0x%08x\n", td->ep_no,
2210	DXEPTSIZ_GET_NPKT(temp),
2211	temp, DWC_OTG_READ_4(sc, DOTG_DIEPCTL(td->ep_no)));
2212
2213	goto not_complete;
2214	}
2215
2216	DPRINTFN(5, "rem=%u ep=%d\n", td->remainder, td->ep_no);
2217
2218	/* try to optimise by sending more data */
2219	if ((max_buffer != 0) && ((td->max_packet_size & 3) == 0)) {
2220
2221	/* send multiple packets at the same time */
2222	mpkt = max_buffer / td->max_packet_size;
2223
2224	if (mpkt > 0x3FE)
2225	mpkt = 0x3FE;
2226
2227	count = td->remainder;
2228	if (count > 0x7FFFFF)
2229	count = 0x7FFFFF - (0x7FFFFF % td->max_packet_size);
2230
2231	td->npkt = count / td->max_packet_size;
2232
2233	/*
2234	* NOTE: We could use 0x3FE instead of "mpkt" in the
2235	* check below to get more throughput, but then we
2236	* have a dependency towards non-generic chip features
2237	* to disable the TX-FIFO-EMPTY interrupts on a per
2238	* endpoint basis. Increase the maximum buffer size of
2239	* the IN endpoint to increase the performance.
2240	*/
2241	if (td->npkt > mpkt) {
2242	td->npkt = mpkt;
2243	count = td->max_packet_size * mpkt;
2244	} else if ((count == 0) \|\| (count % td->max_packet_size)) {
2245	/* we are transmitting a short packet */
2246	td->npkt++;
2247	td->short_pkt = 1;
2248	}
2249	} else {
2250	/* send one packet at a time */
2251	mpkt = 1;
2252	count = td->max_packet_size;
2253	if (td->remainder < count) {
2254	/* we have a short packet */
2255	td->short_pkt = 1;
2256	count = td->remainder;
2257	}
2258	td->npkt = 1;
2259	}
2260	DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(td->ep_no),
2261	DXEPTSIZ_SET_MULTI(1) \|
2262	DXEPTSIZ_SET_NPKT(td->npkt) \|
2263	DXEPTSIZ_SET_NBYTES(count));
2264
2265	/* make room for buffering */
2266	td->npkt += mpkt;
2267
2268	temp = sc->sc_in_ctl[td->ep_no];
2269
2270	/* check for isochronous mode */
2271	if ((temp & DIEPCTL_EPTYPE_MASK) ==
2272	(DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
2273	/* toggle odd or even frame bit */
2274	if (temp & DIEPCTL_SETD1PID) {
2275	temp &= ~DIEPCTL_SETD1PID;
2276	temp \|= DIEPCTL_SETD0PID;
2277	} else {
2278	temp &= ~DIEPCTL_SETD0PID;
2279	temp \|= DIEPCTL_SETD1PID;
2280	}
2281	sc->sc_in_ctl[td->ep_no] = temp;
2282	}
2283
2284	/* must enable before writing data to FIFO */
2285	DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(td->ep_no), temp \|
2286	DIEPCTL_EPENA \| DIEPCTL_CNAK);
2287
2288	td->tx_bytes = count;
2289
2290	/* check remainder */
2291	if (td->tx_bytes == 0 &&
2292	td->remainder == 0) {
2293	if (td->short_pkt)
2294	return (0); /* complete */
2295
2296	/* else we need to transmit a short packet */
2297	}
2298	goto repeat;
2299
2300	not_complete:
2301	return (1); /* not complete */
2302	}
2303
2304	static uint8_t
2305	dwc_otg_data_tx_sync(struct dwc_otg_softc sc, struct dwc_otg_td td)
2306	{
2307	uint32_t temp;
2308
2309	/*
2310	* If all packets are transferred we are complete:
2311	*/
2312	temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2313
2314	/* check that all packets have been transferred */
2315	if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2316	DPRINTFN(5, "busy ep=%d\n", td->ep_no);
2317	goto not_complete;
2318	}
2319	return (0);
2320
2321	not_complete:
2322
2323	/* we only want to know if there is a SETUP packet or free IN packet */
2324
2325	temp = sc->sc_last_rx_status;
2326
2327	if ((td->ep_no == 0) && (temp != 0) &&
2328	(GRXSTSRD_CHNUM_GET(temp) == 0)) {
2329
2330	if ((temp & GRXSTSRD_PKTSTS_MASK) ==
2331	GRXSTSRD_STP_DATA \|\|
2332	(temp & GRXSTSRD_PKTSTS_MASK) ==
2333	GRXSTSRD_STP_COMPLETE) {
2334	DPRINTFN(5, "faking complete\n");
2335	/*
2336	* Race condition: We are complete!
2337	*/
2338	return (0);
2339	} else {
2340	/* dump data - wrong direction */
2341	dwc_otg_common_rx_ack(sc);
2342	}
2343	}
2344	return (1); /* not complete */
2345	}
2346
2347	static void
2348	dwc_otg_xfer_do_fifo(struct dwc_otg_softc sc, struct usb_xfer xfer)
2349	{
2350	struct dwc_otg_td *td;
2351	uint8_t toggle;
2352	uint8_t tmr_val;
2353	uint8_t tmr_res;
2354
2355	DPRINTFN(9, "\n");
2356
2357	td = xfer->td_transfer_cache;
2358	if (td == NULL)
2359	return;
2360
2361	while (1) {
2362	if ((td->func) (sc, td)) {
2363	/* operation in progress */
2364	break;
2365	}
2366	if (((void *)td) == xfer->td_transfer_last) {
2367	goto done;
2368	}
2369	if (td->error_any) {
2370	goto done;
2371	} else if (td->remainder > 0) {
2372	/*
2373	* We had a short transfer. If there is no alternate
2374	* next, stop processing !
2375	*/
2376	if (!td->alt_next)
2377	goto done;
2378	}
2379
2380	/*
2381	* Fetch the next transfer descriptor and transfer
2382	* some flags to the next transfer descriptor
2383	*/
2384	tmr_res = td->tmr_res;
2385	tmr_val = td->tmr_val;
2386	toggle = td->toggle;
2387	td = td->obj_next;
2388	xfer->td_transfer_cache = td;
2389	td->toggle = toggle; /* transfer toggle */
2390	td->tmr_res = tmr_res;
2391	td->tmr_val = tmr_val;
2392	}
2393	return;
2394
2395	done:
2396	xfer->td_transfer_cache = NULL;
2397	sc->sc_xfer_complete = 1;
2398	}
2399
2400	static uint8_t
2401	dwc_otg_xfer_do_complete_locked(struct dwc_otg_softc sc, struct usb_xfer xfer)
2402	{
2403	struct dwc_otg_td *td;
2404
2405	DPRINTFN(9, "\n");
2406
2407	td = xfer->td_transfer_cache;
2408	if (td == NULL) {
2409	/* compute all actual lengths */
2410	dwc_otg_standard_done(xfer);
2411	return (1);
2412	}
2413	return (0);
2414	}
2415
2416	static void
2417	dwc_otg_timer(void *_sc)
2418	{
2419	struct dwc_otg_softc *sc = _sc;
2420
2421	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2422
2423	DPRINTF("\n");
2424
2425	USB_BUS_SPIN_LOCK(&sc->sc_bus);
2426
2427	/* increment timer value */
2428	sc->sc_tmr_val++;
2429
2430	/* enable SOF interrupt, which will poll jobs */
2431	dwc_otg_enable_sof_irq(sc);
2432
2433	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2434
2435	if (sc->sc_timer_active) {
2436	/* restart timer */
2437	usb_callout_reset(&sc->sc_timer,
2438	hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2439	&dwc_otg_timer, sc);
2440	}
2441	}
2442
2443	static void
2444	dwc_otg_timer_start(struct dwc_otg_softc *sc)
2445	{
2446	if (sc->sc_timer_active != 0)
2447	return;
2448
2449	sc->sc_timer_active = 1;
2450
2451	/* restart timer */
2452	usb_callout_reset(&sc->sc_timer,
2453	hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2454	&dwc_otg_timer, sc);
2455	}
2456
2457	static void
2458	dwc_otg_timer_stop(struct dwc_otg_softc *sc)
2459	{
2460	if (sc->sc_timer_active == 0)
2461	return;
2462
2463	sc->sc_timer_active = 0;
2464
2465	/* stop timer */
2466	usb_callout_stop(&sc->sc_timer);
2467	}
2468
2469	static uint16_t
2470	dwc_otg_compute_isoc_rx_tt_slot(struct dwc_otg_tt_info *pinfo)
2471	{
2472	if (pinfo->slot_index < DWC_OTG_TT_SLOT_MAX)
2473	pinfo->slot_index++;
2474	return (pinfo->slot_index);
2475	}
2476
2477	static uint8_t
2478	dwc_otg_update_host_transfer_schedule_locked(struct dwc_otg_softc *sc)
2479	{
2480	TAILQ_HEAD(, usb_xfer) head;
2481	struct usb_xfer *xfer;
2482	struct usb_xfer *xfer_next;
2483	struct dwc_otg_td *td;
2484	uint16_t temp;
2485	uint16_t slot;
2486
2487	temp = DWC_OTG_READ_4(sc, DOTG_HFNUM) & DWC_OTG_FRAME_MASK;
2488
2489	if (sc->sc_last_frame_num == temp)
2490	return (0);
2491
2492	sc->sc_last_frame_num = temp;
2493
2494	TAILQ_INIT(&head);
2495
2496	if ((temp & 7) == 0) {
2497
2498	/* reset the schedule */
2499	memset(sc->sc_tt_info, 0, sizeof(sc->sc_tt_info));
2500
2501	TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2502	td = xfer->td_transfer_cache;
2503	if (td == NULL \|\| td->ep_type != UE_ISOCHRONOUS)
2504	continue;
2505
2506	/* check for IN direction */
2507	if ((td->hcchar & HCCHAR_EPDIR_IN) != 0)
2508	continue;
2509
2510	sc->sc_needsof = 1;
2511
2512	if (td->hcsplt == 0 \|\| td->tt_scheduled != 0)
2513	continue;
2514
2515	/* compute slot */
2516	slot = dwc_otg_compute_isoc_rx_tt_slot(
2517	sc->sc_tt_info + td->tt_index);
2518	if (slot > 3) {
2519	/*
2520	* Not enough time to get complete
2521	* split executed.
2522	*/
2523	continue;
2524	}
2525	/* Delayed start */
2526	td->tt_start_slot = temp + slot;
2527	td->tt_scheduled = 1;
2528	TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2529	TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2530	}
2531
2532	TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2533	td = xfer->td_transfer_cache;
2534	if (td == NULL \|\| td->ep_type != UE_ISOCHRONOUS)
2535	continue;
2536
2537	/* check for OUT direction */
2538	if ((td->hcchar & HCCHAR_EPDIR_IN) == 0)
2539	continue;
2540
2541	sc->sc_needsof = 1;
2542
2543	if (td->hcsplt == 0 \|\| td->tt_scheduled != 0)
2544	continue;
2545
2546	/* Start ASAP */
2547	td->tt_start_slot = temp;
2548	td->tt_scheduled = 1;
2549	TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2550	TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2551	}
2552
2553	TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2554	td = xfer->td_transfer_cache;
2555	if (td == NULL \|\| td->ep_type != UE_INTERRUPT)
2556	continue;
2557
2558	if (td->tt_scheduled != 0) {
2559	sc->sc_needsof = 1;
2560	continue;
2561	}
2562
2563	if (dwc_otg_host_rate_check_interrupt(sc, td))
2564	continue;
2565
2566	if (td->hcsplt == 0) {
2567	sc->sc_needsof = 1;
2568	td->tt_scheduled = 1;
2569	continue;
2570	}
2571
2572	/* start ASAP */
2573	td->tt_start_slot = temp;
2574	sc->sc_needsof = 1;
2575	td->tt_scheduled = 1;
2576	TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2577	TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2578	}
2579
2580	TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2581	td = xfer->td_transfer_cache;
2582	if (td == NULL \|\|
2583	td->ep_type != UE_CONTROL) {
2584	continue;
2585	}
2586
2587	sc->sc_needsof = 1;
2588
2589	if (td->hcsplt == 0 \|\| td->tt_scheduled != 0)
2590	continue;
2591
2592	/* start ASAP */
2593	td->tt_start_slot = temp;
2594	td->tt_scheduled = 1;
2595	TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2596	TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2597	}
2598	}
2599	if ((temp & 7) < 6) {
2600	TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2601	td = xfer->td_transfer_cache;
2602	if (td == NULL \|\|
2603	td->ep_type != UE_BULK) {
2604	continue;
2605	}
2606
2607	sc->sc_needsof = 1;
2608
2609	if (td->hcsplt == 0 \|\| td->tt_scheduled != 0)
2610	continue;
2611
2612	/* start ASAP */
2613	td->tt_start_slot = temp;
2614	td->tt_scheduled = 1;
2615	TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2616	TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2617	}
2618	}
2619
2620	/* Put TT transfers in execution order at the end */
2621	TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2622
2623	/* move all TT transfers in front, keeping the current order */
2624	TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2625	td = xfer->td_transfer_cache;
2626	if (td == NULL \|\| td->hcsplt == 0)
2627	continue;
2628	TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2629	TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2630	}
2631	TAILQ_CONCAT(&head, &sc->sc_bus.intr_q.head, wait_entry);
2632	TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2633
2634	/* put non-TT non-ISOCHRONOUS transfers last */
2635	TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2636	td = xfer->td_transfer_cache;
2637	if (td == NULL \|\| td->hcsplt != 0 \|\| td->ep_type == UE_ISOCHRONOUS)
2638	continue;
2639	TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2640	TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2641	}
2642	TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2643
2644	if ((temp & 7) == 0) {
2645
2646	DPRINTFN(12, "SOF interrupt #%d, needsof=%d\n",
2647	(int)temp, (int)sc->sc_needsof);
2648
2649	/* update SOF IRQ mask */
2650	if (sc->sc_irq_mask & GINTMSK_SOFMSK) {
2651	if (sc->sc_needsof == 0) {
2652	sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2653	DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2654	}
2655	} else {
2656	if (sc->sc_needsof != 0) {
2657	sc->sc_irq_mask \|= GINTMSK_SOFMSK;
2658	DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2659	}
2660	}
2661
2662	/* clear need SOF flag */
2663	sc->sc_needsof = 0;
2664	}
2665	return (1);
2666	}
2667
2668	static void
2669	dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *sc)
2670	{
2671	struct usb_xfer *xfer;
2672	uint32_t count;
2673	uint32_t temp;
2674	uint32_t haint;
2675	uint8_t got_rx_status;
2676	uint8_t x;
2677
2678	if (sc->sc_flags.status_device_mode == 0) {
2679	/*
2680	* Update host transfer schedule, so that new
2681	* transfers can be issued:
2682	*/
2683	dwc_otg_update_host_transfer_schedule_locked(sc);
2684	}
2685	count = 0;
2686	repeat:
2687	if (++count == 16) {
2688	/* give other interrupts a chance */
2689	DPRINTF("Yield\n");
2690	return;
2691	}
2692
2693	/* get all host channel interrupts */
2694	haint = DWC_OTG_READ_4(sc, DOTG_HAINT);
2695	while (1) {
2696	x = ffs(haint) - 1;
2697	if (x >= sc->sc_host_ch_max)
2698	break;
2699	temp = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
2700	DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), temp);
2701	temp &= ~HCINT_SOFTWARE_ONLY;
2702	sc->sc_chan_state[x].hcint \|= temp;
2703	haint &= ~(1U << x);
2704	}
2705
2706	if (sc->sc_last_rx_status == 0) {
2707
2708	temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2709	if (temp & GINTSTS_RXFLVL) {
2710	/* pop current status */
2711	sc->sc_last_rx_status =
2712	DWC_OTG_READ_4(sc, DOTG_GRXSTSPD);
2713	}
2714
2715	if (sc->sc_last_rx_status != 0) {
2716
2717	uint8_t ep_no;
2718
2719	temp = sc->sc_last_rx_status &
2720	GRXSTSRD_PKTSTS_MASK;
2721
2722	/* non-data messages we simply skip */
2723	if (temp != GRXSTSRD_STP_DATA &&
2724	temp != GRXSTSRD_STP_COMPLETE &&
2725	temp != GRXSTSRD_OUT_DATA) {
2726	/* check for halted channel */
2727	if (temp == GRXSTSRH_HALTED) {
2728	ep_no = GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status);
2729	sc->sc_chan_state[ep_no].wait_halted = 0;
2730	DPRINTFN(5, "channel halt complete ch=%u\n", ep_no);
2731	}
2732	/* store bytes and FIFO offset */
2733	sc->sc_current_rx_bytes = 0;
2734	sc->sc_current_rx_fifo = 0;
2735
2736	/* acknowledge status */
2737	dwc_otg_common_rx_ack(sc);
2738	goto repeat;
2739	}
2740
2741	temp = GRXSTSRD_BCNT_GET(
2742	sc->sc_last_rx_status);
2743	ep_no = GRXSTSRD_CHNUM_GET(
2744	sc->sc_last_rx_status);
2745
2746	/* store bytes and FIFO offset */
2747	sc->sc_current_rx_bytes = (temp + 3) & ~3;
2748	sc->sc_current_rx_fifo = DOTG_DFIFO(ep_no);
2749
2750	DPRINTF("Reading %d bytes from ep %d\n", temp, ep_no);
2751
2752	/* check if we should dump the data */
2753	if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2754	dwc_otg_common_rx_ack(sc);
2755	goto repeat;
2756	}
2757
2758	got_rx_status = 1;
2759
2760	DPRINTFN(5, "RX status = 0x%08x: ch=%d pid=%d bytes=%d sts=%d\n",
2761	sc->sc_last_rx_status, ep_no,
2762	(sc->sc_last_rx_status >> 15) & 3,
2763	GRXSTSRD_BCNT_GET(sc->sc_last_rx_status),
2764	(sc->sc_last_rx_status >> 17) & 15);
2765	} else {
2766	got_rx_status = 0;
2767	}
2768	} else {
2769	uint8_t ep_no;
2770
2771	ep_no = GRXSTSRD_CHNUM_GET(
2772	sc->sc_last_rx_status);
2773
2774	/* check if we should dump the data */
2775	if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2776	dwc_otg_common_rx_ack(sc);
2777	goto repeat;
2778	}
2779
2780	got_rx_status = 1;
2781	}
2782
2783	/* execute FIFOs */
2784	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry)
2785	dwc_otg_xfer_do_fifo(sc, xfer);
2786
2787	if (got_rx_status) {
2788	/* check if data was consumed */
2789	if (sc->sc_last_rx_status == 0)
2790	goto repeat;
2791
2792	/* disable RX FIFO level interrupt */
2793	sc->sc_irq_mask &= ~GINTMSK_RXFLVLMSK;
2794	DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2795	}
2796	}
2797
2798	static void
2799	dwc_otg_interrupt_complete_locked(struct dwc_otg_softc *sc)
2800	{
2801	struct usb_xfer *xfer;
2802	repeat:
2803	/* scan for completion events */
2804	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
2805	if (dwc_otg_xfer_do_complete_locked(sc, xfer))
2806	goto repeat;
2807	}
2808	}
2809
2810	static void
2811	dwc_otg_vbus_interrupt(struct dwc_otg_softc *sc, uint8_t is_on)
2812	{
2813	DPRINTFN(5, "vbus = %u\n", is_on);
2814
2815	/*
2816	* If the USB host mode is forced, then assume VBUS is always
2817	* present else rely on the input to this function:
2818	*/
2819	if ((is_on != 0) \|\| (sc->sc_mode == DWC_MODE_HOST)) {
2820
2821	if (!sc->sc_flags.status_vbus) {
2822	sc->sc_flags.status_vbus = 1;
2823
2824	/* complete root HUB interrupt endpoint */
2825
2826	dwc_otg_root_intr(sc);
2827	}
2828	} else {
2829	if (sc->sc_flags.status_vbus) {
2830	sc->sc_flags.status_vbus = 0;
2831	sc->sc_flags.status_bus_reset = 0;
2832	sc->sc_flags.status_suspend = 0;
2833	sc->sc_flags.change_suspend = 0;
2834	sc->sc_flags.change_connect = 1;
2835
2836	/* complete root HUB interrupt endpoint */
2837
2838	dwc_otg_root_intr(sc);
2839	}
2840	}
2841	}
2842
2843	int
2844	dwc_otg_filter_interrupt(void *arg)
2845	{
2846	struct dwc_otg_softc *sc = arg;
2847	int retval = FILTER_HANDLED;
2848	uint32_t status;
2849
2850	USB_BUS_SPIN_LOCK(&sc->sc_bus);
2851
2852	/* read and clear interrupt status */
2853	status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2854
2855	/* clear interrupts we are handling here */
2856	DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & ~DWC_OTG_MSK_GINT_THREAD_IRQ);
2857
2858	/* check for USB state change interrupts */
2859	if ((status & DWC_OTG_MSK_GINT_THREAD_IRQ) != 0)
2860	retval = FILTER_SCHEDULE_THREAD;
2861
2862	/* clear FIFO empty interrupts */
2863	if (status & sc->sc_irq_mask &
2864	(GINTSTS_PTXFEMP \| GINTSTS_NPTXFEMP)) {
2865	sc->sc_irq_mask &= ~(GINTSTS_PTXFEMP \| GINTSTS_NPTXFEMP);
2866	DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2867	}
2868	/* clear all IN endpoint interrupts */
2869	if (status & GINTSTS_IEPINT) {
2870	uint32_t temp;
2871	uint8_t x;
2872
2873	for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
2874	temp = DWC_OTG_READ_4(sc, DOTG_DIEPINT(x));
2875	if (temp & DIEPMSK_XFERCOMPLMSK) {
2876	DWC_OTG_WRITE_4(sc, DOTG_DIEPINT(x),
2877	DIEPMSK_XFERCOMPLMSK);
2878	}
2879	}
2880	}
2881
2882	/* poll FIFOs, if any */
2883	dwc_otg_interrupt_poll_locked(sc);
2884
2885	if (sc->sc_xfer_complete != 0)
2886	retval = FILTER_SCHEDULE_THREAD;
2887
2888	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2889
2890	return (retval);
2891	}
2892
2893	void
2894	dwc_otg_interrupt(void *arg)
2895	{
2896	struct dwc_otg_softc *sc = arg;
2897	uint32_t status;
2898
2899	USB_BUS_LOCK(&sc->sc_bus);
2900	USB_BUS_SPIN_LOCK(&sc->sc_bus);
2901
2902	/* read and clear interrupt status */
2903	status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2904
2905	/* clear interrupts we are handling here */
2906	DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & DWC_OTG_MSK_GINT_THREAD_IRQ);
2907
2908	DPRINTFN(14, "GINTSTS=0x%08x HAINT=0x%08x HFNUM=0x%08x\n",
2909	status, DWC_OTG_READ_4(sc, DOTG_HAINT),
2910	DWC_OTG_READ_4(sc, DOTG_HFNUM));
2911
2912	if (status & GINTSTS_USBRST) {
2913
2914	/* set correct state */
2915	sc->sc_flags.status_device_mode = 1;
2916	sc->sc_flags.status_bus_reset = 0;
2917	sc->sc_flags.status_suspend = 0;
2918	sc->sc_flags.change_suspend = 0;
2919	sc->sc_flags.change_connect = 1;
2920
2921	/* Disable SOF interrupt */
2922	sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2923	DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2924
2925	/* complete root HUB interrupt endpoint */
2926	dwc_otg_root_intr(sc);
2927	}
2928
2929	/* check for any bus state change interrupts */
2930	if (status & GINTSTS_ENUMDONE) {
2931
2932	uint32_t temp;
2933
2934	DPRINTFN(5, "end of reset\n");
2935
2936	/* set correct state */
2937	sc->sc_flags.status_device_mode = 1;
2938	sc->sc_flags.status_bus_reset = 1;
2939	sc->sc_flags.status_suspend = 0;
2940	sc->sc_flags.change_suspend = 0;
2941	sc->sc_flags.change_connect = 1;
2942	sc->sc_flags.status_low_speed = 0;
2943	sc->sc_flags.port_enabled = 1;
2944
2945	/* reset FIFOs */
2946	(void) dwc_otg_init_fifo(sc, DWC_MODE_DEVICE);
2947
2948	/* reset function address */
2949	dwc_otg_set_address(sc, 0);
2950
2951	/* figure out enumeration speed */
2952	temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
2953	if (DSTS_ENUMSPD_GET(temp) == DSTS_ENUMSPD_HI)
2954	sc->sc_flags.status_high_speed = 1;
2955	else
2956	sc->sc_flags.status_high_speed = 0;
2957
2958	/*
2959	* Disable resume and SOF interrupt, and enable
2960	* suspend and RX frame interrupt:
2961	*/
2962	sc->sc_irq_mask &= ~(GINTMSK_WKUPINTMSK \| GINTMSK_SOFMSK);
2963	sc->sc_irq_mask \|= GINTMSK_USBSUSPMSK;
2964	DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2965
2966	/* complete root HUB interrupt endpoint */
2967	dwc_otg_root_intr(sc);
2968	}
2969
2970	if (status & GINTSTS_PRTINT) {
2971	uint32_t hprt;
2972
2973	hprt = DWC_OTG_READ_4(sc, DOTG_HPRT);
2974
2975	/* clear change bits */
2976	DWC_OTG_WRITE_4(sc, DOTG_HPRT, (hprt & (
2977	HPRT_PRTPWR \| HPRT_PRTENCHNG \|
2978	HPRT_PRTCONNDET \| HPRT_PRTOVRCURRCHNG)) \|
2979	sc->sc_hprt_val);
2980
2981	DPRINTFN(12, "GINTSTS=0x%08x, HPRT=0x%08x\n", status, hprt);
2982
2983	sc->sc_flags.status_device_mode = 0;
2984
2985	if (hprt & HPRT_PRTCONNSTS)
2986	sc->sc_flags.status_bus_reset = 1;
2987	else
2988	sc->sc_flags.status_bus_reset = 0;
2989
2990	if (hprt & HPRT_PRTENCHNG)
2991	sc->sc_flags.change_enabled = 1;
2992
2993	if (hprt & HPRT_PRTENA)
2994	sc->sc_flags.port_enabled = 1;
2995	else
2996	sc->sc_flags.port_enabled = 0;
2997
2998	if (hprt & HPRT_PRTOVRCURRCHNG)
2999	sc->sc_flags.change_over_current = 1;
3000
3001	if (hprt & HPRT_PRTOVRCURRACT)
3002	sc->sc_flags.port_over_current = 1;
3003	else
3004	sc->sc_flags.port_over_current = 0;
3005
3006	if (hprt & HPRT_PRTPWR)
3007	sc->sc_flags.port_powered = 1;
3008	else
3009	sc->sc_flags.port_powered = 0;
3010
3011	if (((hprt & HPRT_PRTSPD_MASK)
3012	>> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_LOW)
3013	sc->sc_flags.status_low_speed = 1;
3014	else
3015	sc->sc_flags.status_low_speed = 0;
3016
3017	if (((hprt & HPRT_PRTSPD_MASK)
3018	>> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_HIGH)
3019	sc->sc_flags.status_high_speed = 1;
3020	else
3021	sc->sc_flags.status_high_speed = 0;
3022
3023	if (hprt & HPRT_PRTCONNDET)
3024	sc->sc_flags.change_connect = 1;
3025
3026	if (hprt & HPRT_PRTSUSP)
3027	dwc_otg_suspend_irq(sc);
3028	else
3029	dwc_otg_resume_irq(sc);
3030
3031	/* complete root HUB interrupt endpoint */
3032	dwc_otg_root_intr(sc);
3033
3034	/* update host frame interval */
3035	dwc_otg_update_host_frame_interval(sc);
3036	}
3037
3038	/*
3039	* If resume and suspend is set at the same time we interpret
3040	* that like RESUME. Resume is set when there is at least 3
3041	* milliseconds of inactivity on the USB BUS.
3042	*/
3043	if (status & GINTSTS_WKUPINT) {
3044
3045	DPRINTFN(5, "resume interrupt\n");
3046
3047	dwc_otg_resume_irq(sc);
3048
3049	} else if (status & GINTSTS_USBSUSP) {
3050
3051	DPRINTFN(5, "suspend interrupt\n");
3052
3053	dwc_otg_suspend_irq(sc);
3054	}
3055	/* check VBUS */
3056	if (status & (GINTSTS_USBSUSP \|
3057	GINTSTS_USBRST \|
3058	GINTMSK_OTGINTMSK \|
3059	GINTSTS_SESSREQINT)) {
3060	uint32_t temp;
3061
3062	temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
3063
3064	DPRINTFN(5, "GOTGCTL=0x%08x\n", temp);
3065
3066	dwc_otg_vbus_interrupt(sc,
3067	(temp & (GOTGCTL_ASESVLD \| GOTGCTL_BSESVLD)) ? 1 : 0);
3068	}
3069
3070	if (sc->sc_xfer_complete != 0) {
3071	sc->sc_xfer_complete = 0;
3072
3073	/* complete FIFOs, if any */
3074	dwc_otg_interrupt_complete_locked(sc);
3075	}
3076	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3077	USB_BUS_UNLOCK(&sc->sc_bus);
3078	}
3079
3080	static void
3081	dwc_otg_setup_standard_chain_sub(struct dwc_otg_std_temp *temp)
3082	{
3083	struct dwc_otg_td *td;
3084
3085	/* get current Transfer Descriptor */
3086	td = temp->td_next;
3087	temp->td = td;
3088
3089	/* prepare for next TD */
3090	temp->td_next = td->obj_next;
3091
3092	/* fill out the Transfer Descriptor */
3093	td->func = temp->func;
3094	td->pc = temp->pc;
3095	td->offset = temp->offset;
3096	td->remainder = temp->len;
3097	td->tx_bytes = 0;
3098	td->error_any = 0;
3099	td->error_stall = 0;
3100	td->npkt = 0;
3101	td->did_stall = temp->did_stall;
3102	td->short_pkt = temp->short_pkt;
3103	td->alt_next = temp->setup_alt_next;
3104	td->set_toggle = 0;
3105	td->got_short = 0;
3106	td->did_nak = 0;
3107	td->channel[0] = DWC_OTG_MAX_CHANNELS;
3108	td->channel[1] = DWC_OTG_MAX_CHANNELS;
3109	td->channel[2] = DWC_OTG_MAX_CHANNELS;
3110	td->state = 0;
3111	td->errcnt = 0;
3112	td->tt_scheduled = 0;
3113	td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
3114	}
3115
3116	static void
3117	dwc_otg_setup_standard_chain(struct usb_xfer *xfer)
3118	{
3119	struct dwc_otg_std_temp temp;
3120	struct dwc_otg_td *td;
3121	uint32_t x;
3122	uint8_t need_sync;
3123	uint8_t is_host;
3124
3125	DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
3126	xfer->address, UE_GET_ADDR(xfer->endpointno),
3127	xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
3128
3129	temp.max_frame_size = xfer->max_frame_size;
3130
3131	td = xfer->td_start[0];
3132	xfer->td_transfer_first = td;
3133	xfer->td_transfer_cache = td;
3134
3135	/* setup temp */
3136
3137	temp.pc = NULL;
3138	temp.td = NULL;
3139	temp.td_next = xfer->td_start[0];
3140	temp.offset = 0;
3141	temp.setup_alt_next = xfer->flags_int.short_frames_ok \|\|
3142	xfer->flags_int.isochronous_xfr;
3143	temp.did_stall = !xfer->flags_int.control_stall;
3144
3145	is_host = (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST);
3146
3147	/* check if we should prepend a setup message */
3148
3149	if (xfer->flags_int.control_xfr) {
3150	if (xfer->flags_int.control_hdr) {
3151
3152	if (is_host)
3153	temp.func = &dwc_otg_host_setup_tx;
3154	else
3155	temp.func = &dwc_otg_setup_rx;
3156
3157	temp.len = xfer->frlengths[0];
3158	temp.pc = xfer->frbuffers + 0;
3159	temp.short_pkt = temp.len ? 1 : 0;
3160
3161	/* check for last frame */
3162	if (xfer->nframes == 1) {
3163	/* no STATUS stage yet, SETUP is last */
3164	if (xfer->flags_int.control_act)
3165	temp.setup_alt_next = 0;
3166	}
3167
3168	dwc_otg_setup_standard_chain_sub(&temp);
3169	}
3170	x = 1;
3171	} else {
3172	x = 0;
3173	}
3174
3175	if (x != xfer->nframes) {
3176	if (xfer->endpointno & UE_DIR_IN) {
3177	if (is_host) {
3178	temp.func = &dwc_otg_host_data_rx;
3179	need_sync = 0;
3180	} else {
3181	temp.func = &dwc_otg_data_tx;
3182	need_sync = 1;
3183	}
3184	} else {
3185	if (is_host) {
3186	temp.func = &dwc_otg_host_data_tx;
3187	need_sync = 0;
3188	} else {
3189	temp.func = &dwc_otg_data_rx;
3190	need_sync = 0;
3191	}
3192	}
3193
3194	/* setup "pc" pointer */
3195	temp.pc = xfer->frbuffers + x;
3196	} else {
3197	need_sync = 0;
3198	}
3199	while (x != xfer->nframes) {
3200
3201	/* DATA0 / DATA1 message */
3202
3203	temp.len = xfer->frlengths[x];
3204
3205	x++;
3206
3207	if (x == xfer->nframes) {
3208	if (xfer->flags_int.control_xfr) {
3209	if (xfer->flags_int.control_act) {
3210	temp.setup_alt_next = 0;
3211	}
3212	} else {
3213	temp.setup_alt_next = 0;
3214	}
3215	}
3216	if (temp.len == 0) {
3217
3218	/* make sure that we send an USB packet */
3219
3220	temp.short_pkt = 0;
3221
3222	} else {
3223
3224	/* regular data transfer */
3225
3226	temp.short_pkt = (xfer->flags.force_short_xfer ? 0 : 1);
3227	}
3228
3229	dwc_otg_setup_standard_chain_sub(&temp);
3230
3231	if (xfer->flags_int.isochronous_xfr) {
3232	temp.offset += temp.len;
3233	} else {
3234	/* get next Page Cache pointer */
3235	temp.pc = xfer->frbuffers + x;
3236	}
3237	}
3238
3239	if (xfer->flags_int.control_xfr) {
3240
3241	/* always setup a valid "pc" pointer for status and sync */
3242	temp.pc = xfer->frbuffers + 0;
3243	temp.len = 0;
3244	temp.short_pkt = 0;
3245	temp.setup_alt_next = 0;
3246
3247	/* check if we need to sync */
3248	if (need_sync) {
3249	/* we need a SYNC point after TX */
3250	temp.func = &dwc_otg_data_tx_sync;
3251	dwc_otg_setup_standard_chain_sub(&temp);
3252	}
3253
3254	/* check if we should append a status stage */
3255	if (!xfer->flags_int.control_act) {
3256
3257	/*
3258	* Send a DATA1 message and invert the current
3259	* endpoint direction.
3260	*/
3261	if (xfer->endpointno & UE_DIR_IN) {
3262	if (is_host) {
3263	temp.func = &dwc_otg_host_data_tx;
3264	need_sync = 0;
3265	} else {
3266	temp.func = &dwc_otg_data_rx;
3267	need_sync = 0;
3268	}
3269	} else {
3270	if (is_host) {
3271	temp.func = &dwc_otg_host_data_rx;
3272	need_sync = 0;
3273	} else {
3274	temp.func = &dwc_otg_data_tx;
3275	need_sync = 1;
3276	}
3277	}
3278
3279	dwc_otg_setup_standard_chain_sub(&temp);
3280
3281	/* data toggle should be DATA1 */
3282	td = temp.td;
3283	td->set_toggle = 1;
3284
3285	if (need_sync) {
3286	/* we need a SYNC point after TX */
3287	temp.func = &dwc_otg_data_tx_sync;
3288	dwc_otg_setup_standard_chain_sub(&temp);
3289	}
3290	}
3291	} else {
3292	/* check if we need to sync */
3293	if (need_sync) {
3294
3295	temp.pc = xfer->frbuffers + 0;
3296	temp.len = 0;
3297	temp.short_pkt = 0;
3298	temp.setup_alt_next = 0;
3299
3300	/* we need a SYNC point after TX */
3301	temp.func = &dwc_otg_data_tx_sync;
3302	dwc_otg_setup_standard_chain_sub(&temp);
3303	}
3304	}
3305
3306	/* must have at least one frame! */
3307	td = temp.td;
3308	xfer->td_transfer_last = td;
3309
3310	if (is_host) {
3311
3312	struct dwc_otg_softc *sc;
3313	uint32_t hcchar;
3314	uint32_t hcsplt;
3315
3316	sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3317
3318	/* get first again */
3319	td = xfer->td_transfer_first;
3320	td->toggle = (xfer->endpoint->toggle_next ? 1 : 0);
3321
3322	hcchar =
3323	(xfer->address << HCCHAR_DEVADDR_SHIFT) \|
3324	((xfer->endpointno & UE_ADDR) << HCCHAR_EPNUM_SHIFT) \|
3325	(xfer->max_packet_size << HCCHAR_MPS_SHIFT) \|
3326	HCCHAR_CHENA;
3327
3328	/*
3329	* We are not always able to meet the timing
3330	* requirements of the USB interrupt endpoint's
3331	* complete split token, when doing transfers going
3332	* via a transaction translator. Use the CONTROL
3333	* transfer type instead of the INTERRUPT transfer
3334	* type in general, as a means to workaround
3335	* that. This trick should work for both FULL and LOW
3336	* speed USB traffic going through a TT. For non-TT
3337	* traffic it works aswell. The reason for using
3338	* CONTROL type instead of BULK is that some TTs might
3339	* reject LOW speed BULK traffic.
3340	*/
3341	if (td->ep_type == UE_INTERRUPT)
3342	hcchar \|= (UE_CONTROL << HCCHAR_EPTYPE_SHIFT);
3343	else
3344	hcchar \|= (td->ep_type << HCCHAR_EPTYPE_SHIFT);
3345
3346	if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN)
3347	hcchar \|= HCCHAR_EPDIR_IN;
3348
3349	switch (xfer->xroot->udev->speed) {
3350	case USB_SPEED_LOW:
3351	hcchar \|= HCCHAR_LSPDDEV;
3352	/* FALLTHROUGH */
3353	case USB_SPEED_FULL:
3354	/* check if root HUB port is running High Speed */
3355	if (dwc_otg_uses_split(xfer->xroot->udev)) {
3356	hcsplt = HCSPLT_SPLTENA \|
3357	(xfer->xroot->udev->hs_port_no <<
3358	HCSPLT_PRTADDR_SHIFT) \|
3359	(xfer->xroot->udev->hs_hub_addr <<
3360	HCSPLT_HUBADDR_SHIFT);
3361	} else {
3362	hcsplt = 0;
3363	}
3364	if (td->ep_type == UE_INTERRUPT) {
3365	uint32_t ival;
3366	ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3367	if (ival == 0)
3368	ival = 1;
3369	else if (ival > 127)
3370	ival = 127;
3371	td->tmr_val = sc->sc_tmr_val + ival;
3372	td->tmr_res = ival;
3373	} else if (td->ep_type == UE_ISOCHRONOUS) {
3374	td->tmr_res = 1;
3375	td->tmr_val = sc->sc_last_frame_num;
3376	if (td->hcchar & HCCHAR_EPDIR_IN)
3377	td->tmr_val++;
3378	} else {
3379	td->tmr_val = 0;
3380	td->tmr_res = (uint8_t)sc->sc_last_frame_num;
3381	}
3382	break;
3383	case USB_SPEED_HIGH:
3384	hcsplt = 0;
3385	if (td->ep_type == UE_INTERRUPT) {
3386	uint32_t ival;
3387	hcchar \|= ((xfer->max_packet_count & 3)
3388	<< HCCHAR_MC_SHIFT);
3389	ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3390	if (ival == 0)
3391	ival = 1;
3392	else if (ival > 127)
3393	ival = 127;
3394	td->tmr_val = sc->sc_tmr_val + ival;
3395	td->tmr_res = ival;
3396	} else if (td->ep_type == UE_ISOCHRONOUS) {
3397	hcchar \|= ((xfer->max_packet_count & 3)
3398	<< HCCHAR_MC_SHIFT);
3399	td->tmr_res = 1 << usbd_xfer_get_fps_shift(xfer);
3400	td->tmr_val = sc->sc_last_frame_num;
3401	if (td->hcchar & HCCHAR_EPDIR_IN)
3402	td->tmr_val += td->tmr_res;
3403
3404	} else {
3405	td->tmr_val = 0;
3406	td->tmr_res = (uint8_t)sc->sc_last_frame_num;
3407	}
3408	break;
3409	default:
3410	hcsplt = 0;
3411	td->tmr_val = 0;
3412	td->tmr_res = 0;
3413	break;
3414	}
3415
3416	/* store configuration in all TD's */
3417	while (1) {
3418	td->hcchar = hcchar;
3419	td->hcsplt = hcsplt;
3420
3421	if (((void *)td) == xfer->td_transfer_last)
3422	break;
3423
3424	td = td->obj_next;
3425	}
3426	}
3427	}
3428
3429	static void
3430	dwc_otg_timeout(void *arg)
3431	{
3432	struct usb_xfer *xfer = arg;
3433
3434	DPRINTF("xfer=%p\n", xfer);
3435
3436	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
3437
3438	/* transfer is transferred */
3439	dwc_otg_device_done(xfer, USB_ERR_TIMEOUT);
3440	}
3441
3442	static void
3443	dwc_otg_start_standard_chain(struct usb_xfer *xfer)
3444	{
3445	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3446
3447	DPRINTFN(9, "\n");
3448
3449	/*
3450	* Poll one time in device mode, which will turn on the
3451	* endpoint interrupts. Else wait for SOF interrupt in host
3452	* mode.
3453	*/
3454	USB_BUS_SPIN_LOCK(&sc->sc_bus);
3455
3456	if (sc->sc_flags.status_device_mode != 0) {
3457	dwc_otg_xfer_do_fifo(sc, xfer);
3458	if (dwc_otg_xfer_do_complete_locked(sc, xfer))
3459	goto done;
3460	} else {
3461	struct dwc_otg_td *td = xfer->td_transfer_cache;
3462	if (td->ep_type == UE_ISOCHRONOUS &&
3463	(td->hcchar & HCCHAR_EPDIR_IN) == 0) {
3464	/*
3465	* Need to start ISOCHRONOUS OUT transfer ASAP
3466	* because execution is delayed by one 125us
3467	* microframe:
3468	*/
3469	dwc_otg_xfer_do_fifo(sc, xfer);
3470	if (dwc_otg_xfer_do_complete_locked(sc, xfer))
3471	goto done;
3472	}
3473	}
3474
3475	/* put transfer on interrupt queue */
3476	usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
3477
3478	/* start timeout, if any */
3479	if (xfer->timeout != 0) {
3480	usbd_transfer_timeout_ms(xfer,
3481	&dwc_otg_timeout, xfer->timeout);
3482	}
3483
3484	if (sc->sc_flags.status_device_mode != 0)
3485	goto done;
3486
3487	/* enable SOF interrupt, if any */
3488	dwc_otg_enable_sof_irq(sc);
3489	done:
3490	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3491	}
3492
3493	static void
3494	dwc_otg_root_intr(struct dwc_otg_softc *sc)
3495	{
3496	DPRINTFN(9, "\n");
3497
3498	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3499
3500	/* set port bit */
3501	sc->sc_hub_idata[0] = 0x02; /* we only have one port */
3502
3503	uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
3504	sizeof(sc->sc_hub_idata));
3505	}
3506
3507	static usb_error_t
3508	dwc_otg_standard_done_sub(struct usb_xfer *xfer)
3509	{
3510	struct dwc_otg_td *td;
3511	uint32_t len;
3512	usb_error_t error;
3513
3514	DPRINTFN(9, "\n");
3515
3516	td = xfer->td_transfer_cache;
3517
3518	do {
3519	len = td->remainder;
3520
3521	/* store last data toggle */
3522	xfer->endpoint->toggle_next = td->toggle;
3523
3524	if (xfer->aframes != xfer->nframes) {
3525	/*
3526	* Verify the length and subtract
3527	* the remainder from "frlengths[]":
3528	*/
3529	if (len > xfer->frlengths[xfer->aframes]) {
3530	td->error_any = 1;
3531	} else {
3532	xfer->frlengths[xfer->aframes] -= len;
3533	}
3534	}
3535	/* Check for transfer error */
3536	if (td->error_any) {
3537	/* the transfer is finished */
3538	error = (td->error_stall ?
3539	USB_ERR_STALLED : USB_ERR_IOERROR);
3540	td = NULL;
3541	break;
3542	}
3543	/* Check for short transfer */
3544	if (len > 0) {
3545	if (xfer->flags_int.short_frames_ok \|\|
3546	xfer->flags_int.isochronous_xfr) {
3547	/* follow alt next */
3548	if (td->alt_next) {
3549	td = td->obj_next;
3550	} else {
3551	td = NULL;
3552	}
3553	} else {
3554	/* the transfer is finished */
3555	td = NULL;
3556	}
3557	error = 0;
3558	break;
3559	}
3560	td = td->obj_next;
3561
3562	/* this USB frame is complete */
3563	error = 0;
3564	break;
3565
3566	} while (0);
3567
3568	/* update transfer cache */
3569
3570	xfer->td_transfer_cache = td;
3571
3572	return (error);
3573	}
3574
3575	static void
3576	dwc_otg_standard_done(struct usb_xfer *xfer)
3577	{
3578	usb_error_t err = 0;
3579
3580	DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
3581	xfer, xfer->endpoint);
3582
3583	/* reset scanner */
3584
3585	xfer->td_transfer_cache = xfer->td_transfer_first;
3586
3587	if (xfer->flags_int.control_xfr) {
3588
3589	if (xfer->flags_int.control_hdr) {
3590
3591	err = dwc_otg_standard_done_sub(xfer);
3592	}
3593	xfer->aframes = 1;
3594
3595	if (xfer->td_transfer_cache == NULL) {
3596	goto done;
3597	}
3598	}
3599	while (xfer->aframes != xfer->nframes) {
3600
3601	err = dwc_otg_standard_done_sub(xfer);
3602	xfer->aframes++;
3603
3604	if (xfer->td_transfer_cache == NULL) {
3605	goto done;
3606	}
3607	}
3608
3609	if (xfer->flags_int.control_xfr &&
3610	!xfer->flags_int.control_act) {
3611
3612	err = dwc_otg_standard_done_sub(xfer);
3613	}
3614	done:
3615	dwc_otg_device_done(xfer, err);
3616	}
3617
3618	/------------------------------------------------------------------------
3619	* dwc_otg_device_done
3620	*
3621	* NOTE: this function can be called more than one time on the
3622	* same USB transfer!
3623	------------------------------------------------------------------------/
3624	static void
3625	dwc_otg_device_done(struct usb_xfer *xfer, usb_error_t error)
3626	{
3627	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3628
3629	DPRINTFN(9, "xfer=%p, endpoint=%p, error=%d\n",
3630	xfer, xfer->endpoint, error);
3631
3632	USB_BUS_SPIN_LOCK(&sc->sc_bus);
3633
3634	if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
3635	/* Interrupts are cleared by the interrupt handler */
3636	} else {
3637	struct dwc_otg_td *td;
3638
3639	td = xfer->td_transfer_cache;
3640	if (td != NULL)
3641	dwc_otg_host_channel_free(sc, td);
3642	}
3643	/* dequeue transfer and start next transfer */
3644	usbd_transfer_done(xfer, error);
3645
3646	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3647	}
3648
3649	static void
3650	dwc_otg_xfer_stall(struct usb_xfer *xfer)
3651	{
3652	dwc_otg_device_done(xfer, USB_ERR_STALLED);
3653	}
3654
3655	static void
3656	dwc_otg_set_stall(struct usb_device *udev,
3657	struct usb_endpoint ep, uint8_t did_stall)
3658	{
3659	struct dwc_otg_softc *sc;
3660	uint32_t temp;
3661	uint32_t reg;
3662	uint8_t ep_no;
3663
3664	USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3665
3666	/* check mode */
3667	if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3668	/* not supported */
3669	return;
3670	}
3671
3672	sc = DWC_OTG_BUS2SC(udev->bus);
3673
3674	USB_BUS_SPIN_LOCK(&sc->sc_bus);
3675
3676	/* get endpoint address */
3677	ep_no = ep->edesc->bEndpointAddress;
3678
3679	DPRINTFN(5, "endpoint=0x%x\n", ep_no);
3680
3681	if (ep_no & UE_DIR_IN) {
3682	reg = DOTG_DIEPCTL(ep_no & UE_ADDR);
3683	temp = sc->sc_in_ctl[ep_no & UE_ADDR];
3684	} else {
3685	reg = DOTG_DOEPCTL(ep_no & UE_ADDR);
3686	temp = sc->sc_out_ctl[ep_no & UE_ADDR];
3687	}
3688
3689	/* disable and stall endpoint */
3690	DWC_OTG_WRITE_4(sc, reg, temp \| DOEPCTL_EPDIS);
3691	DWC_OTG_WRITE_4(sc, reg, temp \| DOEPCTL_STALL);
3692
3693	/* clear active OUT ep */
3694	if (!(ep_no & UE_DIR_IN)) {
3695
3696	sc->sc_active_rx_ep &= ~(1U << (ep_no & UE_ADDR));
3697
3698	if (sc->sc_last_rx_status != 0 &&
3699	(ep_no & UE_ADDR) == GRXSTSRD_CHNUM_GET(
3700	sc->sc_last_rx_status)) {
3701	/* dump data */
3702	dwc_otg_common_rx_ack(sc);
3703	/* poll interrupt */
3704	dwc_otg_interrupt_poll_locked(sc);
3705	dwc_otg_interrupt_complete_locked(sc);
3706	}
3707	}
3708	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3709	}
3710
3711	static void
3712	dwc_otg_clear_stall_sub_locked(struct dwc_otg_softc *sc, uint32_t mps,
3713	uint8_t ep_no, uint8_t ep_type, uint8_t ep_dir)
3714	{
3715	uint32_t reg;
3716	uint32_t temp;
3717
3718	if (ep_type == UE_CONTROL) {
3719	/* clearing stall is not needed */
3720	return;
3721	}
3722
3723	if (ep_dir) {
3724	reg = DOTG_DIEPCTL(ep_no);
3725	} else {
3726	reg = DOTG_DOEPCTL(ep_no);
3727	sc->sc_active_rx_ep \|= (1U << ep_no);
3728	}
3729
3730	/* round up and mask away the multiplier count */
3731	mps = (mps + 3) & 0x7FC;
3732
3733	if (ep_type == UE_BULK) {
3734	temp = DIEPCTL_EPTYPE_SET(
3735	DIEPCTL_EPTYPE_BULK) \|
3736	DIEPCTL_USBACTEP;
3737	} else if (ep_type == UE_INTERRUPT) {
3738	temp = DIEPCTL_EPTYPE_SET(
3739	DIEPCTL_EPTYPE_INTERRUPT) \|
3740	DIEPCTL_USBACTEP;
3741	} else {
3742	temp = DIEPCTL_EPTYPE_SET(
3743	DIEPCTL_EPTYPE_ISOC) \|
3744	DIEPCTL_USBACTEP;
3745	}
3746
3747	temp \|= DIEPCTL_MPS_SET(mps);
3748	temp \|= DIEPCTL_TXFNUM_SET(ep_no);
3749
3750	if (ep_dir)
3751	sc->sc_in_ctl[ep_no] = temp;
3752	else
3753	sc->sc_out_ctl[ep_no] = temp;
3754
3755	DWC_OTG_WRITE_4(sc, reg, temp \| DOEPCTL_EPDIS);
3756	DWC_OTG_WRITE_4(sc, reg, temp \| DOEPCTL_SETD0PID);
3757	DWC_OTG_WRITE_4(sc, reg, temp \| DIEPCTL_SNAK);
3758
3759	/* we only reset the transmit FIFO */
3760	if (ep_dir) {
3761	dwc_otg_tx_fifo_reset(sc,
3762	GRSTCTL_TXFIFO(ep_no) \|
3763	GRSTCTL_TXFFLSH);
3764
3765	DWC_OTG_WRITE_4(sc,
3766	DOTG_DIEPTSIZ(ep_no), 0);
3767	}
3768
3769	/* poll interrupt */
3770	dwc_otg_interrupt_poll_locked(sc);
3771	dwc_otg_interrupt_complete_locked(sc);
3772	}
3773
3774	static void
3775	dwc_otg_clear_stall(struct usb_device udev, struct usb_endpoint ep)
3776	{
3777	struct dwc_otg_softc *sc;
3778	struct usb_endpoint_descriptor *ed;
3779
3780	DPRINTFN(5, "endpoint=%p\n", ep);
3781
3782	USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3783
3784	/* check mode */
3785	if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3786	/* not supported */
3787	return;
3788	}
3789	/* get softc */
3790	sc = DWC_OTG_BUS2SC(udev->bus);
3791
3792	USB_BUS_SPIN_LOCK(&sc->sc_bus);
3793
3794	/* get endpoint descriptor */
3795	ed = ep->edesc;
3796
3797	/* reset endpoint */
3798	dwc_otg_clear_stall_sub_locked(sc,
3799	UGETW(ed->wMaxPacketSize),
3800	(ed->bEndpointAddress & UE_ADDR),
3801	(ed->bmAttributes & UE_XFERTYPE),
3802	(ed->bEndpointAddress & (UE_DIR_IN \| UE_DIR_OUT)));
3803
3804	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3805	}
3806
3807	static void
3808	dwc_otg_device_state_change(struct usb_device *udev)
3809	{
3810	struct dwc_otg_softc *sc;
3811	uint8_t x;
3812
3813	/* check mode */
3814	if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3815	/* not supported */
3816	return;
3817	}
3818
3819	/* get softc */
3820	sc = DWC_OTG_BUS2SC(udev->bus);
3821
3822	/* deactivate all other endpoint but the control endpoint */
3823	if (udev->state == USB_STATE_CONFIGURED \|\|
3824	udev->state == USB_STATE_ADDRESSED) {
3825
3826	USB_BUS_LOCK(&sc->sc_bus);
3827
3828	for (x = 1; x != sc->sc_dev_ep_max; x++) {
3829
3830	if (x < sc->sc_dev_in_ep_max) {
3831	DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x),
3832	DIEPCTL_EPDIS);
3833	DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x), 0);
3834	}
3835
3836	DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x),
3837	DOEPCTL_EPDIS);
3838	DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x), 0);
3839	}
3840	USB_BUS_UNLOCK(&sc->sc_bus);
3841	}
3842	}
3843
3844	int
3845	dwc_otg_init(struct dwc_otg_softc *sc)
3846	{
3847	uint32_t temp;
3848
3849	DPRINTF("start\n");
3850
3851	/* set up the bus structure */
3852	sc->sc_bus.usbrev = USB_REV_2_0;
3853	sc->sc_bus.methods = &dwc_otg_bus_methods;
3854
3855	usb_callout_init_mtx(&sc->sc_timer,
3856	&sc->sc_bus.bus_mtx, 0);
3857
3858	USB_BUS_LOCK(&sc->sc_bus);
3859
3860	/* turn on clocks */
3861	dwc_otg_clocks_on(sc);
3862
3863	temp = DWC_OTG_READ_4(sc, DOTG_GSNPSID);
3864	DPRINTF("Version = 0x%08x\n", temp);
3865
3866	/* disconnect */
3867	DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3868	DCTL_SFTDISCON);
3869
3870	/* wait for host to detect disconnect */
3871	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 32);
3872
3873	DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
3874	GRSTCTL_CSFTRST);
3875
3876	/* wait a little bit for block to reset */
3877	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 128);
3878
3879	switch (sc->sc_mode) {
3880	case DWC_MODE_DEVICE:
3881	temp = GUSBCFG_FORCEDEVMODE;
3882	break;
3883	case DWC_MODE_HOST:
3884	temp = GUSBCFG_FORCEHOSTMODE;
3885	break;
3886	default:
3887	temp = 0;
3888	break;
3889	}
3890
3891	/* select HSIC, ULPI or internal PHY mode */
3892	switch (dwc_otg_phy_type) {
3893	case DWC_OTG_PHY_HSIC:
3894	DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3895	GUSBCFG_PHYIF \|
3896	GUSBCFG_TRD_TIM_SET(5) \| temp);
3897	DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL,
3898	0x000000EC);
3899
3900	temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3901	DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3902	temp & ~GLPMCFG_HSIC_CONN);
3903	DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3904	temp \| GLPMCFG_HSIC_CONN);
3905	break;
3906	case DWC_OTG_PHY_ULPI:
3907	DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3908	GUSBCFG_ULPI_UTMI_SEL \|
3909	GUSBCFG_TRD_TIM_SET(5) \| temp);
3910	DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3911
3912	temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3913	DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3914	temp & ~GLPMCFG_HSIC_CONN);
3915	break;
3916	case DWC_OTG_PHY_INTERNAL:
3917	DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3918	GUSBCFG_PHYSEL \|
3919	GUSBCFG_TRD_TIM_SET(5) \| temp);
3920	DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3921
3922	temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3923	DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3924	temp & ~GLPMCFG_HSIC_CONN);
3925
3926	temp = DWC_OTG_READ_4(sc, DOTG_GGPIO);
3927	temp &= ~(DOTG_GGPIO_NOVBUSSENS \| DOTG_GGPIO_I2CPADEN);
3928	temp \|= (DOTG_GGPIO_VBUSASEN \| DOTG_GGPIO_VBUSBSEN \|
3929	DOTG_GGPIO_PWRDWN);
3930	DWC_OTG_WRITE_4(sc, DOTG_GGPIO, temp);
3931	break;
3932	default:
3933	break;
3934	}
3935
3936	/* clear global nak */
3937	DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3938	DCTL_CGOUTNAK \|
3939	DCTL_CGNPINNAK);
3940
3941	/* disable USB port */
3942	DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0xFFFFFFFF);
3943
3944	/* wait 10ms */
3945	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3946
3947	/* enable USB port */
3948	DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
3949
3950	/* wait 10ms */
3951	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3952
3953	temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG3);
3954
3955	sc->sc_fifo_size = 4 * GHWCFG3_DFIFODEPTH_GET(temp);
3956
3957	temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
3958
3959	sc->sc_dev_ep_max = GHWCFG2_NUMDEVEPS_GET(temp);
3960
3961	if (sc->sc_dev_ep_max > DWC_OTG_MAX_ENDPOINTS)
3962	sc->sc_dev_ep_max = DWC_OTG_MAX_ENDPOINTS;
3963
3964	sc->sc_host_ch_max = GHWCFG2_NUMHSTCHNL_GET(temp);
3965
3966	if (sc->sc_host_ch_max > DWC_OTG_MAX_CHANNELS)
3967	sc->sc_host_ch_max = DWC_OTG_MAX_CHANNELS;
3968
3969	temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG4);
3970
3971	sc->sc_dev_in_ep_max = GHWCFG4_NUM_IN_EP_GET(temp);
3972
3973	DPRINTF("Total FIFO size = %d bytes, Device EPs = %d/%d Host CHs = %d\n",
3974	sc->sc_fifo_size, sc->sc_dev_ep_max, sc->sc_dev_in_ep_max,
3975	sc->sc_host_ch_max);
3976
3977	/* setup FIFO */
3978	if (dwc_otg_init_fifo(sc, sc->sc_mode)) {
3979	USB_BUS_UNLOCK(&sc->sc_bus);
3980	return (EINVAL);
3981	}
3982
3983	/* enable interrupts */
3984	sc->sc_irq_mask = DWC_OTG_MSK_GINT_ENABLED;
3985	DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
3986
3987	if (sc->sc_mode == DWC_MODE_OTG \|\| sc->sc_mode == DWC_MODE_DEVICE) {
3988
3989	/* enable all endpoint interrupts */
3990	temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
3991	if (temp & GHWCFG2_MPI) {
3992	uint8_t x;
3993
3994	DPRINTF("Disable Multi Process Interrupts\n");
3995
3996	for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
3997	DWC_OTG_WRITE_4(sc, DOTG_DIEPEACHINTMSK(x), 0);
3998	DWC_OTG_WRITE_4(sc, DOTG_DOEPEACHINTMSK(x), 0);
3999	}
4000	DWC_OTG_WRITE_4(sc, DOTG_DEACHINTMSK, 0);
4001	}
4002	DWC_OTG_WRITE_4(sc, DOTG_DIEPMSK,
4003	DIEPMSK_XFERCOMPLMSK);
4004	DWC_OTG_WRITE_4(sc, DOTG_DOEPMSK, 0);
4005	DWC_OTG_WRITE_4(sc, DOTG_DAINTMSK, 0xFFFF);
4006	}
4007
4008	if (sc->sc_mode == DWC_MODE_OTG \|\| sc->sc_mode == DWC_MODE_HOST) {
4009	/* setup clocks */
4010	temp = DWC_OTG_READ_4(sc, DOTG_HCFG);
4011	temp &= ~(HCFG_FSLSSUPP \| HCFG_FSLSPCLKSEL_MASK);
4012	temp \|= (1 << HCFG_FSLSPCLKSEL_SHIFT);
4013	DWC_OTG_WRITE_4(sc, DOTG_HCFG, temp);
4014	}
4015
4016	/* only enable global IRQ */
4017	DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG,
4018	GAHBCFG_GLBLINTRMSK);
4019
4020	/* turn off clocks */
4021	dwc_otg_clocks_off(sc);
4022
4023	/* read initial VBUS state */
4024
4025	temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
4026
4027	DPRINTFN(5, "GOTCTL=0x%08x\n", temp);
4028
4029	dwc_otg_vbus_interrupt(sc,
4030	(temp & (GOTGCTL_ASESVLD \| GOTGCTL_BSESVLD)) ? 1 : 0);
4031
4032	USB_BUS_UNLOCK(&sc->sc_bus);
4033
4034	/* catch any lost interrupts */
4035
4036	dwc_otg_do_poll(&sc->sc_bus);
4037
4038	return (0); /* success */
4039	}
4040
4041	void
4042	dwc_otg_uninit(struct dwc_otg_softc *sc)
4043	{
4044	USB_BUS_LOCK(&sc->sc_bus);
4045
4046	/* stop host timer */
4047	dwc_otg_timer_stop(sc);
4048
4049	/* set disconnect */
4050	DWC_OTG_WRITE_4(sc, DOTG_DCTL,
4051	DCTL_SFTDISCON);
4052
4053	/* turn off global IRQ */
4054	DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG, 0);
4055
4056	sc->sc_flags.port_enabled = 0;
4057	sc->sc_flags.port_powered = 0;
4058	sc->sc_flags.status_vbus = 0;
4059	sc->sc_flags.status_bus_reset = 0;
4060	sc->sc_flags.status_suspend = 0;
4061	sc->sc_flags.change_suspend = 0;
4062	sc->sc_flags.change_connect = 1;
4063
4064	dwc_otg_pull_down(sc);
4065	dwc_otg_clocks_off(sc);
4066
4067	USB_BUS_UNLOCK(&sc->sc_bus);
4068
4069	usb_callout_drain(&sc->sc_timer);
4070	}
4071
4072	static void
4073	dwc_otg_suspend(struct dwc_otg_softc *sc)
4074	{
4075	return;
4076	}
4077
4078	static void
4079	dwc_otg_resume(struct dwc_otg_softc *sc)
4080	{
4081	return;
4082	}
4083
4084	static void
4085	dwc_otg_do_poll(struct usb_bus *bus)
4086	{
4087	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
4088
4089	USB_BUS_LOCK(&sc->sc_bus);
4090	USB_BUS_SPIN_LOCK(&sc->sc_bus);
4091	dwc_otg_interrupt_poll_locked(sc);
4092	dwc_otg_interrupt_complete_locked(sc);
4093	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
4094	USB_BUS_UNLOCK(&sc->sc_bus);
4095	}
4096
4097	/------------------------------------------------------------------------
4098	* DWC OTG bulk support
4099	* DWC OTG control support
4100	* DWC OTG interrupt support
4101	------------------------------------------------------------------------/
4102	static void
4103	dwc_otg_device_non_isoc_open(struct usb_xfer *xfer)
4104	{
4105	}
4106
4107	static void
4108	dwc_otg_device_non_isoc_close(struct usb_xfer *xfer)
4109	{
4110	dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
4111	}
4112
4113	static void
4114	dwc_otg_device_non_isoc_enter(struct usb_xfer *xfer)
4115	{
4116	}
4117
4118	static void
4119	dwc_otg_device_non_isoc_start(struct usb_xfer *xfer)
4120	{
4121	/* setup TDs */
4122	dwc_otg_setup_standard_chain(xfer);
4123	dwc_otg_start_standard_chain(xfer);
4124	}
4125
4126	static const struct usb_pipe_methods dwc_otg_device_non_isoc_methods =
4127	{
4128	.open = dwc_otg_device_non_isoc_open,
4129	.close = dwc_otg_device_non_isoc_close,
4130	.enter = dwc_otg_device_non_isoc_enter,
4131	.start = dwc_otg_device_non_isoc_start,
4132	};
4133
4134	/------------------------------------------------------------------------
4135	* DWC OTG full speed isochronous support
4136	------------------------------------------------------------------------/
4137	static void
4138	dwc_otg_device_isoc_open(struct usb_xfer *xfer)
4139	{
4140	}
4141
4142	static void
4143	dwc_otg_device_isoc_close(struct usb_xfer *xfer)
4144	{
4145	dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
4146	}
4147
4148	static void
4149	dwc_otg_device_isoc_enter(struct usb_xfer *xfer)
4150	{
4151	}
4152
4153	static void
4154	dwc_otg_device_isoc_start(struct usb_xfer *xfer)
4155	{
4156	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
4157	uint32_t temp;
4158	uint32_t msframes;
4159	uint32_t framenum;
4160	uint8_t shift = usbd_xfer_get_fps_shift(xfer);
4161
4162	DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
4163	xfer, xfer->endpoint->isoc_next, xfer->nframes);
4164
4165	if (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST) {
4166	temp = DWC_OTG_READ_4(sc, DOTG_HFNUM);
4167
4168	/* get the current frame index */
4169	framenum = (temp & HFNUM_FRNUM_MASK);
4170	} else {
4171	temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
4172
4173	/* get the current frame index */
4174	framenum = DSTS_SOFFN_GET(temp);
4175	}
4176
4177	/*
4178	* Check if port is doing 8000 or 1000 frames per second:
4179	*/
4180	if (sc->sc_flags.status_high_speed)
4181	framenum /= 8;
4182
4183	framenum &= DWC_OTG_FRAME_MASK;
4184
4185	/*
4186	* Compute number of milliseconds worth of data traffic for
4187	* this USB transfer:
4188	*/
4189	if (xfer->xroot->udev->speed == USB_SPEED_HIGH)
4190	msframes = ((xfer->nframes << shift) + 7) / 8;
4191	else
4192	msframes = xfer->nframes;
4193
4194	/*
4195	* check if the frame index is within the window where the frames
4196	* will be inserted
4197	*/
4198	temp = (framenum - xfer->endpoint->isoc_next) & DWC_OTG_FRAME_MASK;
4199
4200	if ((xfer->endpoint->is_synced == 0) \|\| (temp < msframes)) {
4201	/*
4202	* If there is data underflow or the pipe queue is
4203	* empty we schedule the transfer a few frames ahead
4204	* of the current frame position. Else two isochronous
4205	* transfers might overlap.
4206	*/
4207	xfer->endpoint->isoc_next = (framenum + 3) & DWC_OTG_FRAME_MASK;
4208	xfer->endpoint->is_synced = 1;
4209	DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
4210	}
4211	/*
4212	* compute how many milliseconds the insertion is ahead of the
4213	* current frame position:
4214	*/
4215	temp = (xfer->endpoint->isoc_next - framenum) & DWC_OTG_FRAME_MASK;
4216
4217	/*
4218	* pre-compute when the isochronous transfer will be finished:
4219	*/
4220	xfer->isoc_time_complete =
4221	usb_isoc_time_expand(&sc->sc_bus, framenum) + temp + msframes;
4222
4223	/* setup TDs */
4224	dwc_otg_setup_standard_chain(xfer);
4225
4226	/* compute frame number for next insertion */
4227	xfer->endpoint->isoc_next += msframes;
4228
4229	/* start TD chain */
4230	dwc_otg_start_standard_chain(xfer);
4231	}
4232
4233	static const struct usb_pipe_methods dwc_otg_device_isoc_methods =
4234	{
4235	.open = dwc_otg_device_isoc_open,
4236	.close = dwc_otg_device_isoc_close,
4237	.enter = dwc_otg_device_isoc_enter,
4238	.start = dwc_otg_device_isoc_start,
4239	};
4240
4241	/------------------------------------------------------------------------
4242	* DWC OTG root control support
4243	------------------------------------------------------------------------
4244	* Simulate a hardware HUB by handling all the necessary requests.
4245	------------------------------------------------------------------------/
4246
4247	static const struct usb_device_descriptor dwc_otg_devd = {
4248	.bLength = sizeof(struct usb_device_descriptor),
4249	.bDescriptorType = UDESC_DEVICE,
4250	.bcdUSB = {0x00, 0x02},
4251	.bDeviceClass = UDCLASS_HUB,
4252	.bDeviceSubClass = UDSUBCLASS_HUB,
4253	.bDeviceProtocol = UDPROTO_HSHUBSTT,
4254	.bMaxPacketSize = 64,
4255	.bcdDevice = {0x00, 0x01},
4256	.iManufacturer = 1,
4257	.iProduct = 2,
4258	.bNumConfigurations = 1,
4259	};
4260
4261	static const struct dwc_otg_config_desc dwc_otg_confd = {
4262	.confd = {
4263	.bLength = sizeof(struct usb_config_descriptor),
4264	.bDescriptorType = UDESC_CONFIG,
4265	.wTotalLength[0] = sizeof(dwc_otg_confd),
4266	.bNumInterface = 1,
4267	.bConfigurationValue = 1,
4268	.iConfiguration = 0,
4269	.bmAttributes = UC_SELF_POWERED,
4270	.bMaxPower = 0,
4271	},
4272	.ifcd = {
4273	.bLength = sizeof(struct usb_interface_descriptor),
4274	.bDescriptorType = UDESC_INTERFACE,
4275	.bNumEndpoints = 1,
4276	.bInterfaceClass = UICLASS_HUB,
4277	.bInterfaceSubClass = UISUBCLASS_HUB,
4278	.bInterfaceProtocol = 0,
4279	},
4280	.endpd = {
4281	.bLength = sizeof(struct usb_endpoint_descriptor),
4282	.bDescriptorType = UDESC_ENDPOINT,
4283	.bEndpointAddress = (UE_DIR_IN \| DWC_OTG_INTR_ENDPT),
4284	.bmAttributes = UE_INTERRUPT,
4285	.wMaxPacketSize[0] = 8,
4286	.bInterval = 255,
4287	},
4288	};
4289
4290	#define HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) }
4291
4292	static const struct usb_hub_descriptor_min dwc_otg_hubd = {
4293	.bDescLength = sizeof(dwc_otg_hubd),
4294	.bDescriptorType = UDESC_HUB,
4295	.bNbrPorts = 1,
4296	HSETW(.wHubCharacteristics, (UHD_PWR_NO_SWITCH \| UHD_OC_INDIVIDUAL)),
4297	.bPwrOn2PwrGood = 50,
4298	.bHubContrCurrent = 0,
4299	.DeviceRemovable = {0}, /* port is removable */
4300	};
4301
4302	#define STRING_VENDOR \
4303	"D\0W\0C\0O\0T\0G"
4304
4305	#define STRING_PRODUCT \
4306	"O\0T\0G\0 \0R\0o\0o\0t\0 \0H\0U\0B"
4307
4308	USB_MAKE_STRING_DESC(STRING_VENDOR, dwc_otg_vendor);
4309	USB_MAKE_STRING_DESC(STRING_PRODUCT, dwc_otg_product);
4310
4311	static usb_error_t
4312	dwc_otg_roothub_exec(struct usb_device *udev,
4313	struct usb_device_request req, const void pptr, uint16_t plength)
4314	{
4315	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4316	const void *ptr;
4317	uint16_t len;
4318	uint16_t value;
4319	uint16_t index;
4320	usb_error_t err;
4321
4322	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
4323
4324	/* buffer reset */
4325	ptr = (const void *)&sc->sc_hub_temp;
4326	len = 0;
4327	err = 0;
4328
4329	value = UGETW(req->wValue);
4330	index = UGETW(req->wIndex);
4331
4332	/* demultiplex the control request */
4333
4334	switch (req->bmRequestType) {
4335	case UT_READ_DEVICE:
4336	switch (req->bRequest) {
4337	case UR_GET_DESCRIPTOR:
4338	goto tr_handle_get_descriptor;
4339	case UR_GET_CONFIG:
4340	goto tr_handle_get_config;
4341	case UR_GET_STATUS:
4342	goto tr_handle_get_status;
4343	default:
4344	goto tr_stalled;
4345	}
4346	break;
4347
4348	case UT_WRITE_DEVICE:
4349	switch (req->bRequest) {
4350	case UR_SET_ADDRESS:
4351	goto tr_handle_set_address;
4352	case UR_SET_CONFIG:
4353	goto tr_handle_set_config;
4354	case UR_CLEAR_FEATURE:
4355	goto tr_valid; /* nop */
4356	case UR_SET_DESCRIPTOR:
4357	goto tr_valid; /* nop */
4358	case UR_SET_FEATURE:
4359	default:
4360	goto tr_stalled;
4361	}
4362	break;
4363
4364	case UT_WRITE_ENDPOINT:
4365	switch (req->bRequest) {
4366	case UR_CLEAR_FEATURE:
4367	switch (UGETW(req->wValue)) {
4368	case UF_ENDPOINT_HALT:
4369	goto tr_handle_clear_halt;
4370	case UF_DEVICE_REMOTE_WAKEUP:
4371	goto tr_handle_clear_wakeup;
4372	default:
4373	goto tr_stalled;
4374	}
4375	break;
4376	case UR_SET_FEATURE:
4377	switch (UGETW(req->wValue)) {
4378	case UF_ENDPOINT_HALT:
4379	goto tr_handle_set_halt;
4380	case UF_DEVICE_REMOTE_WAKEUP:
4381	goto tr_handle_set_wakeup;
4382	default:
4383	goto tr_stalled;
4384	}
4385	break;
4386	case UR_SYNCH_FRAME:
4387	goto tr_valid; /* nop */
4388	default:
4389	goto tr_stalled;
4390	}
4391	break;
4392
4393	case UT_READ_ENDPOINT:
4394	switch (req->bRequest) {
4395	case UR_GET_STATUS:
4396	goto tr_handle_get_ep_status;
4397	default:
4398	goto tr_stalled;
4399	}
4400	break;
4401
4402	case UT_WRITE_INTERFACE:
4403	switch (req->bRequest) {
4404	case UR_SET_INTERFACE:
4405	goto tr_handle_set_interface;
4406	case UR_CLEAR_FEATURE:
4407	goto tr_valid; /* nop */
4408	case UR_SET_FEATURE:
4409	default:
4410	goto tr_stalled;
4411	}
4412	break;
4413
4414	case UT_READ_INTERFACE:
4415	switch (req->bRequest) {
4416	case UR_GET_INTERFACE:
4417	goto tr_handle_get_interface;
4418	case UR_GET_STATUS:
4419	goto tr_handle_get_iface_status;
4420	default:
4421	goto tr_stalled;
4422	}
4423	break;
4424
4425	case UT_WRITE_CLASS_INTERFACE:
4426	case UT_WRITE_VENDOR_INTERFACE:
4427	/* XXX forward */
4428	break;
4429
4430	case UT_READ_CLASS_INTERFACE:
4431	case UT_READ_VENDOR_INTERFACE:
4432	/* XXX forward */
4433	break;
4434
4435	case UT_WRITE_CLASS_DEVICE:
4436	switch (req->bRequest) {
4437	case UR_CLEAR_FEATURE:
4438	goto tr_valid;
4439	case UR_SET_DESCRIPTOR:
4440	case UR_SET_FEATURE:
4441	break;
4442	default:
4443	goto tr_stalled;
4444	}
4445	break;
4446
4447	case UT_WRITE_CLASS_OTHER:
4448	switch (req->bRequest) {
4449	case UR_CLEAR_FEATURE:
4450	goto tr_handle_clear_port_feature;
4451	case UR_SET_FEATURE:
4452	goto tr_handle_set_port_feature;
4453	case UR_CLEAR_TT_BUFFER:
4454	case UR_RESET_TT:
4455	case UR_STOP_TT:
4456	goto tr_valid;
4457
4458	default:
4459	goto tr_stalled;
4460	}
4461	break;
4462
4463	case UT_READ_CLASS_OTHER:
4464	switch (req->bRequest) {
4465	case UR_GET_TT_STATE:
4466	goto tr_handle_get_tt_state;
4467	case UR_GET_STATUS:
4468	goto tr_handle_get_port_status;
4469	default:
4470	goto tr_stalled;
4471	}
4472	break;
4473
4474	case UT_READ_CLASS_DEVICE:
4475	switch (req->bRequest) {
4476	case UR_GET_DESCRIPTOR:
4477	goto tr_handle_get_class_descriptor;
4478	case UR_GET_STATUS:
4479	goto tr_handle_get_class_status;
4480
4481	default:
4482	goto tr_stalled;
4483	}
4484	break;
4485	default:
4486	goto tr_stalled;
4487	}
4488	goto tr_valid;
4489
4490	tr_handle_get_descriptor:
4491	switch (value >> 8) {
4492	case UDESC_DEVICE:
4493	if (value & 0xff) {
4494	goto tr_stalled;
4495	}
4496	len = sizeof(dwc_otg_devd);
4497	ptr = (const void *)&dwc_otg_devd;
4498	goto tr_valid;
4499	case UDESC_CONFIG:
4500	if (value & 0xff) {
4501	goto tr_stalled;
4502	}
4503	len = sizeof(dwc_otg_confd);
4504	ptr = (const void *)&dwc_otg_confd;
4505	goto tr_valid;
4506	case UDESC_STRING:
4507	switch (value & 0xff) {
4508	case 0: /* Language table */
4509	len = sizeof(usb_string_lang_en);
4510	ptr = (const void *)&usb_string_lang_en;
4511	goto tr_valid;
4512
4513	case 1: /* Vendor */
4514	len = sizeof(dwc_otg_vendor);
4515	ptr = (const void *)&dwc_otg_vendor;
4516	goto tr_valid;
4517
4518	case 2: /* Product */
4519	len = sizeof(dwc_otg_product);
4520	ptr = (const void *)&dwc_otg_product;
4521	goto tr_valid;
4522	default:
4523	break;
4524	}
4525	break;
4526	default:
4527	goto tr_stalled;
4528	}
4529	goto tr_stalled;
4530
4531	tr_handle_get_config:
4532	len = 1;
4533	sc->sc_hub_temp.wValue[0] = sc->sc_conf;
4534	goto tr_valid;
4535
4536	tr_handle_get_status:
4537	len = 2;
4538	USETW(sc->sc_hub_temp.wValue, UDS_SELF_POWERED);
4539	goto tr_valid;
4540
4541	tr_handle_set_address:
4542	if (value & 0xFF00) {
4543	goto tr_stalled;
4544	}
4545	sc->sc_rt_addr = value;
4546	goto tr_valid;
4547
4548	tr_handle_set_config:
4549	if (value >= 2) {
4550	goto tr_stalled;
4551	}
4552	sc->sc_conf = value;
4553	goto tr_valid;
4554
4555	tr_handle_get_interface:
4556	len = 1;
4557	sc->sc_hub_temp.wValue[0] = 0;
4558	goto tr_valid;
4559
4560	tr_handle_get_tt_state:
4561	tr_handle_get_class_status:
4562	tr_handle_get_iface_status:
4563	tr_handle_get_ep_status:
4564	len = 2;
4565	USETW(sc->sc_hub_temp.wValue, 0);
4566	goto tr_valid;
4567
4568	tr_handle_set_halt:
4569	tr_handle_set_interface:
4570	tr_handle_set_wakeup:
4571	tr_handle_clear_wakeup:
4572	tr_handle_clear_halt:
4573	goto tr_valid;
4574
4575	tr_handle_clear_port_feature:
4576	if (index != 1)
4577	goto tr_stalled;
4578
4579	DPRINTFN(9, "UR_CLEAR_PORT_FEATURE on port %d\n", index);
4580
4581	switch (value) {
4582	case UHF_PORT_SUSPEND:
4583	dwc_otg_wakeup_peer(sc);
4584	break;
4585
4586	case UHF_PORT_ENABLE:
4587	if (sc->sc_flags.status_device_mode == 0) {
4588	DWC_OTG_WRITE_4(sc, DOTG_HPRT,
4589	sc->sc_hprt_val \| HPRT_PRTENA);
4590	}
4591	sc->sc_flags.port_enabled = 0;
4592	break;
4593
4594	case UHF_C_PORT_RESET:
4595	sc->sc_flags.change_reset = 0;
4596	break;
4597
4598	case UHF_C_PORT_ENABLE:
4599	sc->sc_flags.change_enabled = 0;
4600	break;
4601
4602	case UHF_C_PORT_OVER_CURRENT:
4603	sc->sc_flags.change_over_current = 0;
4604	break;
4605
4606	case UHF_PORT_TEST:
4607	case UHF_PORT_INDICATOR:
4608	/* nops */
4609	break;
4610
4611	case UHF_PORT_POWER:
4612	sc->sc_flags.port_powered = 0;
4613	if (sc->sc_mode == DWC_MODE_HOST \|\| sc->sc_mode == DWC_MODE_OTG) {
4614	sc->sc_hprt_val = 0;
4615	DWC_OTG_WRITE_4(sc, DOTG_HPRT, HPRT_PRTENA);
4616	}
4617	dwc_otg_pull_down(sc);
4618	dwc_otg_clocks_off(sc);
4619	break;
4620
4621	case UHF_C_PORT_CONNECTION:
4622	/* clear connect change flag */
4623	sc->sc_flags.change_connect = 0;
4624	break;
4625
4626	case UHF_C_PORT_SUSPEND:
4627	sc->sc_flags.change_suspend = 0;
4628	break;
4629
4630	default:
4631	err = USB_ERR_IOERROR;
4632	goto done;
4633	}
4634	goto tr_valid;
4635
4636	tr_handle_set_port_feature:
4637	if (index != 1) {
4638	goto tr_stalled;
4639	}
4640	DPRINTFN(9, "UR_SET_PORT_FEATURE\n");
4641
4642	switch (value) {
4643	case UHF_PORT_ENABLE:
4644	break;
4645
4646	case UHF_PORT_SUSPEND:
4647	if (sc->sc_flags.status_device_mode == 0) {
4648	/* set suspend BIT */
4649	sc->sc_hprt_val \|= HPRT_PRTSUSP;
4650	DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4651
4652	/* generate HUB suspend event */
4653	dwc_otg_suspend_irq(sc);
4654	}
4655	break;
4656
4657	case UHF_PORT_RESET:
4658	if (sc->sc_flags.status_device_mode == 0) {
4659
4660	DPRINTF("PORT RESET\n");
4661
4662	/* enable PORT reset */
4663	DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val \| HPRT_PRTRST);
4664
4665	/* Wait 62.5ms for reset to complete */
4666	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4667
4668	DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4669
4670	/* Wait 62.5ms for reset to complete */
4671	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4672
4673	/* reset FIFOs */
4674	(void) dwc_otg_init_fifo(sc, DWC_MODE_HOST);
4675
4676	sc->sc_flags.change_reset = 1;
4677	} else {
4678	err = USB_ERR_IOERROR;
4679	}
4680	break;
4681
4682	case UHF_PORT_TEST:
4683	case UHF_PORT_INDICATOR:
4684	/* nops */
4685	break;
4686	case UHF_PORT_POWER:
4687	sc->sc_flags.port_powered = 1;
4688	if (sc->sc_mode == DWC_MODE_HOST \|\| sc->sc_mode == DWC_MODE_OTG) {
4689	sc->sc_hprt_val \|= HPRT_PRTPWR;
4690	DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4691	}
4692	if (sc->sc_mode == DWC_MODE_DEVICE \|\| sc->sc_mode == DWC_MODE_OTG) {
4693	/* pull up D+, if any */
4694	dwc_otg_pull_up(sc);
4695	}
4696	break;
4697	default:
4698	err = USB_ERR_IOERROR;
4699	goto done;
4700	}
4701	goto tr_valid;
4702
4703	tr_handle_get_port_status:
4704
4705	DPRINTFN(9, "UR_GET_PORT_STATUS\n");
4706
4707	if (index != 1)
4708	goto tr_stalled;
4709
4710	if (sc->sc_flags.status_vbus)
4711	dwc_otg_clocks_on(sc);
4712	else
4713	dwc_otg_clocks_off(sc);
4714
4715	/* Select Device Side Mode */
4716
4717	if (sc->sc_flags.status_device_mode) {
4718	value = UPS_PORT_MODE_DEVICE;
4719	dwc_otg_timer_stop(sc);
4720	} else {
4721	value = 0;
4722	dwc_otg_timer_start(sc);
4723	}
4724
4725	if (sc->sc_flags.status_high_speed)
4726	value \|= UPS_HIGH_SPEED;
4727	else if (sc->sc_flags.status_low_speed)
4728	value \|= UPS_LOW_SPEED;
4729
4730	if (sc->sc_flags.port_powered)
4731	value \|= UPS_PORT_POWER;
4732
4733	if (sc->sc_flags.port_enabled)
4734	value \|= UPS_PORT_ENABLED;
4735
4736	if (sc->sc_flags.port_over_current)
4737	value \|= UPS_OVERCURRENT_INDICATOR;
4738
4739	if (sc->sc_flags.status_vbus &&
4740	sc->sc_flags.status_bus_reset)
4741	value \|= UPS_CURRENT_CONNECT_STATUS;
4742
4743	if (sc->sc_flags.status_suspend)
4744	value \|= UPS_SUSPEND;
4745
4746	USETW(sc->sc_hub_temp.ps.wPortStatus, value);
4747
4748	value = 0;
4749
4750	if (sc->sc_flags.change_connect)
4751	value \|= UPS_C_CONNECT_STATUS;
4752	if (sc->sc_flags.change_suspend)
4753	value \|= UPS_C_SUSPEND;
4754	if (sc->sc_flags.change_reset)
4755	value \|= UPS_C_PORT_RESET;
4756	if (sc->sc_flags.change_over_current)
4757	value \|= UPS_C_OVERCURRENT_INDICATOR;
4758
4759	USETW(sc->sc_hub_temp.ps.wPortChange, value);
4760	len = sizeof(sc->sc_hub_temp.ps);
4761	goto tr_valid;
4762
4763	tr_handle_get_class_descriptor:
4764	if (value & 0xFF) {
4765	goto tr_stalled;
4766	}
4767	ptr = (const void *)&dwc_otg_hubd;
4768	len = sizeof(dwc_otg_hubd);
4769	goto tr_valid;
4770
4771	tr_stalled:
4772	err = USB_ERR_STALLED;
4773	tr_valid:
4774	done:
4775	*plength = len;
4776	*pptr = ptr;
4777	return (err);
4778	}
4779
4780	static void
4781	dwc_otg_xfer_setup(struct usb_setup_params *parm)
4782	{
4783	struct usb_xfer *xfer;
4784	void *last_obj;
4785	uint32_t ntd;
4786	uint32_t n;
4787	uint8_t ep_no;
4788	uint8_t ep_type;
4789
4790	xfer = parm->curr_xfer;
4791
4792	/*
4793	* NOTE: This driver does not use any of the parameters that
4794	* are computed from the following values. Just set some
4795	* reasonable dummies:
4796	*/
4797	parm->hc_max_packet_size = 0x500;
4798	parm->hc_max_packet_count = 3;
4799	parm->hc_max_frame_size = 3 * 0x500;
4800
4801	usbd_transfer_setup_sub(parm);
4802
4803	/*
4804	* compute maximum number of TDs
4805	*/
4806	ep_type = (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE);
4807
4808	if (ep_type == UE_CONTROL) {
4809
4810	ntd = xfer->nframes + 1 /* STATUS / + 1 / SYNC 1 */
4811	+ 1 /* SYNC 2 / + 1 / SYNC 3 */;
4812	} else {
4813
4814	ntd = xfer->nframes + 1 /* SYNC */ ;
4815	}
4816
4817	/*
4818	* check if "usbd_transfer_setup_sub" set an error
4819	*/
4820	if (parm->err)
4821	return;
4822
4823	/*
4824	* allocate transfer descriptors
4825	*/
4826	last_obj = NULL;
4827
4828	ep_no = xfer->endpointno & UE_ADDR;
4829
4830	/*
4831	* Check for a valid endpoint profile in USB device mode:
4832	*/
4833	if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
4834	const struct usb_hw_ep_profile *pf;
4835
4836	dwc_otg_get_hw_ep_profile(parm->udev, &pf, ep_no);
4837
4838	if (pf == NULL) {
4839	/* should not happen */
4840	parm->err = USB_ERR_INVAL;
4841	return;
4842	}
4843	}
4844
4845	/* align data */
4846	parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
4847
4848	for (n = 0; n != ntd; n++) {
4849
4850	struct dwc_otg_td *td;
4851
4852	if (parm->buf) {
4853
4854	td = USB_ADD_BYTES(parm->buf, parm->size[0]);
4855
4856	/* compute shared bandwidth resource index for TT */
4857	if (dwc_otg_uses_split(parm->udev)) {
4858	if (parm->udev->parent_hs_hub->ddesc.bDeviceProtocol == UDPROTO_HSHUBMTT)
4859	td->tt_index = parm->udev->device_index;
4860	else
4861	td->tt_index = parm->udev->parent_hs_hub->device_index;
4862	} else {
4863	td->tt_index = parm->udev->device_index;
4864	}
4865
4866	/* init TD */
4867	td->max_packet_size = xfer->max_packet_size;
4868	td->max_packet_count = xfer->max_packet_count;
4869	/* range check */
4870	if (td->max_packet_count == 0 \|\| td->max_packet_count > 3)
4871	td->max_packet_count = 1;
4872	td->ep_no = ep_no;
4873	td->ep_type = ep_type;
4874	td->obj_next = last_obj;
4875
4876	last_obj = td;
4877	}
4878	parm->size[0] += sizeof(*td);
4879	}
4880
4881	xfer->td_start[0] = last_obj;
4882	}
4883
4884	static void
4885	dwc_otg_xfer_unsetup(struct usb_xfer *xfer)
4886	{
4887	return;
4888	}
4889
4890	static void
4891	dwc_otg_ep_init(struct usb_device udev, struct usb_endpoint_descriptor edesc,
4892	struct usb_endpoint *ep)
4893	{
4894	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4895
4896	DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d,%d)\n",
4897	ep, udev->address,
4898	edesc->bEndpointAddress, udev->flags.usb_mode,
4899	sc->sc_rt_addr, udev->device_index);
4900
4901	if (udev->device_index != sc->sc_rt_addr) {
4902
4903	if (udev->flags.usb_mode == USB_MODE_DEVICE) {
4904	if (udev->speed != USB_SPEED_FULL &&
4905	udev->speed != USB_SPEED_HIGH) {
4906	/* not supported */
4907	return;
4908	}
4909	} else {
4910	if (udev->speed == USB_SPEED_HIGH &&
4911	(edesc->wMaxPacketSize[1] & 0x18) != 0 &&
4912	(edesc->bmAttributes & UE_XFERTYPE) != UE_ISOCHRONOUS) {
4913	/* not supported */
4914	DPRINTFN(-1, "Non-isochronous high bandwidth "
4915	"endpoint not supported\n");
4916	return;
4917	}
4918	}
4919	if ((edesc->bmAttributes & UE_XFERTYPE) == UE_ISOCHRONOUS)
4920	ep->methods = &dwc_otg_device_isoc_methods;
4921	else
4922	ep->methods = &dwc_otg_device_non_isoc_methods;
4923	}
4924	}
4925
4926	static void
4927	dwc_otg_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
4928	{
4929	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
4930
4931	switch (state) {
4932	case USB_HW_POWER_SUSPEND:
4933	dwc_otg_suspend(sc);
4934	break;
4935	case USB_HW_POWER_SHUTDOWN:
4936	dwc_otg_uninit(sc);
4937	break;
4938	case USB_HW_POWER_RESUME:
4939	dwc_otg_resume(sc);
4940	break;
4941	default:
4942	break;
4943	}
4944	}
4945
4946	static void
4947	dwc_otg_get_dma_delay(struct usb_device udev, uint32_t pus)
4948	{
4949	/* DMA delay - wait until any use of memory is finished */
4950	pus = (2125); / microseconds */
4951	}
4952
4953	static void
4954	dwc_otg_device_resume(struct usb_device *udev)
4955	{
4956	DPRINTF("\n");
4957
4958	/* poll all transfers again to restart resumed ones */
4959	dwc_otg_do_poll(udev->bus);
4960	}
4961
4962	static void
4963	dwc_otg_device_suspend(struct usb_device *udev)
4964	{
4965	DPRINTF("\n");
4966	}
4967
4968	static const struct usb_bus_methods dwc_otg_bus_methods =
4969	{
4970	.endpoint_init = &dwc_otg_ep_init,
4971	.xfer_setup = &dwc_otg_xfer_setup,
4972	.xfer_unsetup = &dwc_otg_xfer_unsetup,
4973	.get_hw_ep_profile = &dwc_otg_get_hw_ep_profile,
4974	.xfer_stall = &dwc_otg_xfer_stall,
4975	.set_stall = &dwc_otg_set_stall,
4976	.clear_stall = &dwc_otg_clear_stall,
4977	.roothub_exec = &dwc_otg_roothub_exec,
4978	.xfer_poll = &dwc_otg_do_poll,
4979	.device_state_change = &dwc_otg_device_state_change,
4980	.set_hw_power_sleep = &dwc_otg_set_hw_power_sleep,
4981	.get_dma_delay = &dwc_otg_get_dma_delay,
4982	.device_resume = &dwc_otg_device_resume,
4983	.device_suspend = &dwc_otg_device_suspend,
4984	};

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: rtems-libbsd/freebsd/sys/dev/usb/controller/dwc_otg.c @ c04e7c2

Download in other formats: