Context Navigation

source: rtems-libbsd/freebsd/sys/dev/usb/controller/ehci.c @ 02279d6

55-freebsd-126-freebsd-12freebsd-9.3

Last change on this file since 02279d6 was 02279d6, checked in by Sebastian Huber <sebastian.huber@…>, on 11/12/15 at 12:01:12
USB: Update to FreeBSD trunk 2015-11-10
Property mode set to `100644`
File size: 94.2 KB

Line
1	#include <machine/rtems-bsd-kernel-space.h>
2
3	/* $FreeBSD$ */
4	/*-
5	* Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
6	* Copyright (c) 2004 The NetBSD Foundation, Inc. All rights reserved.
7	* Copyright (c) 2004 Lennart Augustsson. All rights reserved.
8	* Copyright (c) 2004 Charles M. Hannum. All rights reserved.
9	*
10	* Redistribution and use in source and binary forms, with or without
11	* modification, are permitted provided that the following conditions
12	* are met:
13	* 1. Redistributions of source code must retain the above copyright
14	* notice, this list of conditions and the following disclaimer.
15	* 2. Redistributions in binary form must reproduce the above copyright
16	* notice, this list of conditions and the following disclaimer in the
17	* documentation and/or other materials provided with the distribution.
18	*
19	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29	* SUCH DAMAGE.
30	*/
31
32	/*
33	* USB Enhanced Host Controller Driver, a.k.a. USB 2.0 controller.
34	*
35	* The EHCI 0.96 spec can be found at
36	* http://developer.intel.com/technology/usb/download/ehci-r096.pdf
37	* The EHCI 1.0 spec can be found at
38	* http://developer.intel.com/technology/usb/download/ehci-r10.pdf
39	* and the USB 2.0 spec at
40	* http://www.usb.org/developers/docs/usb_20.zip
41	*
42	*/
43
44	/*
45	* TODO:
46	* 1) command failures are not recovered correctly
47	*/
48
49	#ifdef USB_GLOBAL_INCLUDE_FILE
50	#include USB_GLOBAL_INCLUDE_FILE
51	#else
52	#include <sys/stdint.h>
53	#include <sys/stddef.h>
54	#include <rtems/bsd/sys/param.h>
55	#include <sys/queue.h>
56	#include <rtems/bsd/sys/types.h>
57	#include <sys/systm.h>
58	#include <sys/kernel.h>
59	#include <sys/bus.h>
60	#include <sys/module.h>
61	#include <rtems/bsd/sys/lock.h>
62	#include <sys/mutex.h>
63	#include <sys/condvar.h>
64	#include <sys/sysctl.h>
65	#include <sys/sx.h>
66	#include <rtems/bsd/sys/unistd.h>
67	#include <sys/callout.h>
68	#include <sys/malloc.h>
69	#include <sys/priv.h>
70
71	#include <dev/usb/usb.h>
72	#include <dev/usb/usbdi.h>
73
74	#define USB_DEBUG_VAR ehcidebug
75
76	#include <dev/usb/usb_core.h>
77	#include <dev/usb/usb_debug.h>
78	#include <dev/usb/usb_busdma.h>
79	#include <dev/usb/usb_process.h>
80	#include <dev/usb/usb_transfer.h>
81	#include <dev/usb/usb_device.h>
82	#include <dev/usb/usb_hub.h>
83	#include <dev/usb/usb_util.h>
84
85	#include <dev/usb/usb_controller.h>
86	#include <dev/usb/usb_bus.h>
87	#endif /* USB_GLOBAL_INCLUDE_FILE */
88
89	#include <dev/usb/controller/ehci.h>
90	#include <dev/usb/controller/ehcireg.h>
91
92	#define EHCI_BUS2SC(bus) \
93	((ehci_softc_t )(((uint8_t )(bus)) - \
94	((uint8_t )&(((ehci_softc_t )0)->sc_bus))))
95
96	#ifdef USB_DEBUG
97	static int ehcidebug = 0;
98	static int ehcinohighspeed = 0;
99	static int ehciiaadbug = 0;
100	static int ehcilostintrbug = 0;
101
102	static SYSCTL_NODE(_hw_usb, OID_AUTO, ehci, CTLFLAG_RW, 0, "USB ehci");
103	SYSCTL_INT(_hw_usb_ehci, OID_AUTO, debug, CTLFLAG_RWTUN,
104	&ehcidebug, 0, "Debug level");
105	SYSCTL_INT(_hw_usb_ehci, OID_AUTO, no_hs, CTLFLAG_RWTUN,
106	&ehcinohighspeed, 0, "Disable High Speed USB");
107	SYSCTL_INT(_hw_usb_ehci, OID_AUTO, iaadbug, CTLFLAG_RWTUN,
108	&ehciiaadbug, 0, "Enable doorbell bug workaround");
109	SYSCTL_INT(_hw_usb_ehci, OID_AUTO, lostintrbug, CTLFLAG_RWTUN,
110	&ehcilostintrbug, 0, "Enable lost interrupt bug workaround");
111
112	static void ehci_dump_regs(ehci_softc_t *sc);
113	static void ehci_dump_sqh(ehci_softc_t sc, ehci_qh_t sqh);
114
115	#endif
116
117	#define EHCI_INTR_ENDPT 1
118
119	static const struct usb_bus_methods ehci_bus_methods;
120	static const struct usb_pipe_methods ehci_device_bulk_methods;
121	static const struct usb_pipe_methods ehci_device_ctrl_methods;
122	static const struct usb_pipe_methods ehci_device_intr_methods;
123	static const struct usb_pipe_methods ehci_device_isoc_fs_methods;
124	static const struct usb_pipe_methods ehci_device_isoc_hs_methods;
125
126	static void ehci_do_poll(struct usb_bus *);
127	static void ehci_device_done(struct usb_xfer *, usb_error_t);
128	static uint8_t ehci_check_transfer(struct usb_xfer *);
129	static void ehci_timeout(void *);
130	static void ehci_poll_timeout(void *);
131
132	static void ehci_root_intr(ehci_softc_t *sc);
133
134	struct ehci_std_temp {
135	ehci_softc_t *sc;
136	struct usb_page_cache *pc;
137	ehci_qtd_t *td;
138	ehci_qtd_t *td_next;
139	uint32_t average;
140	uint32_t qtd_status;
141	uint32_t len;
142	uint16_t max_frame_size;
143	uint8_t shortpkt;
144	uint8_t auto_data_toggle;
145	uint8_t setup_alt_next;
146	uint8_t last_frame;
147	};
148
149	void
150	ehci_iterate_hw_softc(struct usb_bus bus, usb_bus_mem_sub_cb_t cb)
151	{
152	ehci_softc_t *sc = EHCI_BUS2SC(bus);
153	uint32_t i;
154
155	cb(bus, &sc->sc_hw.pframes_pc, &sc->sc_hw.pframes_pg,
156	sizeof(uint32_t) * EHCI_FRAMELIST_COUNT, EHCI_FRAMELIST_ALIGN);
157
158	cb(bus, &sc->sc_hw.terminate_pc, &sc->sc_hw.terminate_pg,
159	sizeof(struct ehci_qh_sub), EHCI_QH_ALIGN);
160
161	cb(bus, &sc->sc_hw.async_start_pc, &sc->sc_hw.async_start_pg,
162	sizeof(ehci_qh_t), EHCI_QH_ALIGN);
163
164	for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
165	cb(bus, sc->sc_hw.intr_start_pc + i,
166	sc->sc_hw.intr_start_pg + i,
167	sizeof(ehci_qh_t), EHCI_QH_ALIGN);
168	}
169
170	for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
171	cb(bus, sc->sc_hw.isoc_hs_start_pc + i,
172	sc->sc_hw.isoc_hs_start_pg + i,
173	sizeof(ehci_itd_t), EHCI_ITD_ALIGN);
174	}
175
176	for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
177	cb(bus, sc->sc_hw.isoc_fs_start_pc + i,
178	sc->sc_hw.isoc_fs_start_pg + i,
179	sizeof(ehci_sitd_t), EHCI_SITD_ALIGN);
180	}
181	}
182
183	usb_error_t
184	ehci_reset(ehci_softc_t *sc)
185	{
186	uint32_t hcr;
187	int i;
188
189	EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
190	for (i = 0; i < 100; i++) {
191	usb_pause_mtx(NULL, hz / 128);
192	hcr = EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_HCRESET;
193	if (!hcr) {
194	if (sc->sc_flags & (EHCI_SCFLG_SETMODE \| EHCI_SCFLG_BIGEMMIO)) {
195	/*
196	* Force USBMODE as requested. Controllers
197	* may have multiple operating modes.
198	*/
199	uint32_t usbmode = EOREAD4(sc, EHCI_USBMODE);
200	if (sc->sc_flags & EHCI_SCFLG_SETMODE) {
201	usbmode = (usbmode &~ EHCI_UM_CM) \| EHCI_UM_CM_HOST;
202	device_printf(sc->sc_bus.bdev,
203	"set host controller mode\n");
204	}
205	if (sc->sc_flags & EHCI_SCFLG_BIGEMMIO) {
206	usbmode = (usbmode &~ EHCI_UM_ES) \| EHCI_UM_ES_BE;
207	device_printf(sc->sc_bus.bdev,
208	"set big-endian mode\n");
209	}
210	EOWRITE4(sc, EHCI_USBMODE, usbmode);
211	}
212	return (0);
213	}
214	}
215	device_printf(sc->sc_bus.bdev, "reset timeout\n");
216	return (USB_ERR_IOERROR);
217	}
218
219	static usb_error_t
220	ehci_hcreset(ehci_softc_t *sc)
221	{
222	uint32_t hcr;
223	int i;
224
225	EOWRITE4(sc, EHCI_USBCMD, 0); /* Halt controller */
226	for (i = 0; i < 100; i++) {
227	usb_pause_mtx(NULL, hz / 128);
228	hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
229	if (hcr)
230	break;
231	}
232	if (!hcr)
233	/*
234	* Fall through and try reset anyway even though
235	* Table 2-9 in the EHCI spec says this will result
236	* in undefined behavior.
237	*/
238	device_printf(sc->sc_bus.bdev, "stop timeout\n");
239
240	return (ehci_reset(sc));
241	}
242
243	static int
244	ehci_init_sub(struct ehci_softc *sc)
245	{
246	struct usb_page_search buf_res;
247	uint32_t cparams;
248	uint32_t hcr;
249	uint8_t i;
250
251	cparams = EREAD4(sc, EHCI_HCCPARAMS);
252
253	DPRINTF("cparams=0x%x\n", cparams);
254
255	if (EHCI_HCC_64BIT(cparams)) {
256	DPRINTF("HCC uses 64-bit structures\n");
257
258	/* MUST clear segment register if 64 bit capable */
259	EOWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
260	}
261
262	usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
263	EOWRITE4(sc, EHCI_PERIODICLISTBASE, buf_res.physaddr);
264
265	usbd_get_page(&sc->sc_hw.async_start_pc, 0, &buf_res);
266	EOWRITE4(sc, EHCI_ASYNCLISTADDR, buf_res.physaddr \| EHCI_LINK_QH);
267
268	/* enable interrupts */
269	EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
270
271	/* turn on controller */
272	EOWRITE4(sc, EHCI_USBCMD,
273	EHCI_CMD_ITC_1 \| /* 1 microframes interrupt delay */
274	(EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_FLS_M) \|
275	EHCI_CMD_ASE \|
276	EHCI_CMD_PSE \|
277	EHCI_CMD_RS);
278
279	/* Take over port ownership */
280	EOWRITE4(sc, EHCI_CONFIGFLAG, EHCI_CONF_CF);
281
282	for (i = 0; i < 100; i++) {
283	usb_pause_mtx(NULL, hz / 128);
284	hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
285	if (!hcr) {
286	break;
287	}
288	}
289	if (hcr) {
290	device_printf(sc->sc_bus.bdev, "run timeout\n");
291	return (USB_ERR_IOERROR);
292	}
293	return (USB_ERR_NORMAL_COMPLETION);
294	}
295
296	usb_error_t
297	ehci_init(ehci_softc_t *sc)
298	{
299	struct usb_page_search buf_res;
300	uint32_t version;
301	uint32_t sparams;
302	uint16_t i;
303	uint16_t x;
304	uint16_t y;
305	uint16_t bit;
306	usb_error_t err = 0;
307
308	DPRINTF("start\n");
309
310	usb_callout_init_mtx(&sc->sc_tmo_pcd, &sc->sc_bus.bus_mtx, 0);
311	usb_callout_init_mtx(&sc->sc_tmo_poll, &sc->sc_bus.bus_mtx, 0);
312
313	sc->sc_offs = EHCI_CAPLENGTH(EREAD4(sc, EHCI_CAPLEN_HCIVERSION));
314
315	#ifdef USB_DEBUG
316	if (ehciiaadbug)
317	sc->sc_flags \|= EHCI_SCFLG_IAADBUG;
318	if (ehcilostintrbug)
319	sc->sc_flags \|= EHCI_SCFLG_LOSTINTRBUG;
320	if (ehcidebug > 2) {
321	ehci_dump_regs(sc);
322	}
323	#endif
324
325	version = EHCI_HCIVERSION(EREAD4(sc, EHCI_CAPLEN_HCIVERSION));
326	device_printf(sc->sc_bus.bdev, "EHCI version %x.%x\n",
327	version >> 8, version & 0xff);
328
329	sparams = EREAD4(sc, EHCI_HCSPARAMS);
330	DPRINTF("sparams=0x%x\n", sparams);
331
332	sc->sc_noport = EHCI_HCS_N_PORTS(sparams);
333	sc->sc_bus.usbrev = USB_REV_2_0;
334
335	if (!(sc->sc_flags & EHCI_SCFLG_DONTRESET)) {
336	/* Reset the controller */
337	DPRINTF("%s: resetting\n",
338	device_get_nameunit(sc->sc_bus.bdev));
339
340	err = ehci_hcreset(sc);
341	if (err) {
342	device_printf(sc->sc_bus.bdev, "reset timeout\n");
343	return (err);
344	}
345	}
346
347	/*
348	* use current frame-list-size selection 0: 10244 bytes 1: 5124
349	* bytes 2: 256*4 bytes 3: unknown
350	*/
351	if (EHCI_CMD_FLS(EOREAD4(sc, EHCI_USBCMD)) == 3) {
352	device_printf(sc->sc_bus.bdev, "invalid frame-list-size\n");
353	return (USB_ERR_IOERROR);
354	}
355	/* set up the bus struct */
356	sc->sc_bus.methods = &ehci_bus_methods;
357
358	sc->sc_eintrs = EHCI_NORMAL_INTRS;
359
360	if (1) {
361	struct ehci_qh_sub *qh;
362
363	usbd_get_page(&sc->sc_hw.terminate_pc, 0, &buf_res);
364
365	qh = buf_res.buffer;
366
367	sc->sc_terminate_self = htohc32(sc, buf_res.physaddr);
368
369	/* init terminate TD */
370	qh->qtd_next =
371	htohc32(sc, EHCI_LINK_TERMINATE);
372	qh->qtd_altnext =
373	htohc32(sc, EHCI_LINK_TERMINATE);
374	qh->qtd_status =
375	htohc32(sc, EHCI_QTD_HALTED);
376	}
377
378	for (i = 0; i < EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
379	ehci_qh_t *qh;
380
381	usbd_get_page(sc->sc_hw.intr_start_pc + i, 0, &buf_res);
382
383	qh = buf_res.buffer;
384
385	/* initialize page cache pointer */
386
387	qh->page_cache = sc->sc_hw.intr_start_pc + i;
388
389	/* store a pointer to queue head */
390
391	sc->sc_intr_p_last[i] = qh;
392
393	qh->qh_self =
394	htohc32(sc, buf_res.physaddr) \|
395	htohc32(sc, EHCI_LINK_QH);
396
397	qh->qh_endp =
398	htohc32(sc, EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH));
399	qh->qh_endphub =
400	htohc32(sc, EHCI_QH_SET_MULT(1));
401	qh->qh_curqtd = 0;
402
403	qh->qh_qtd.qtd_next =
404	htohc32(sc, EHCI_LINK_TERMINATE);
405	qh->qh_qtd.qtd_altnext =
406	htohc32(sc, EHCI_LINK_TERMINATE);
407	qh->qh_qtd.qtd_status =
408	htohc32(sc, EHCI_QTD_HALTED);
409	}
410
411	/*
412	* the QHs are arranged to give poll intervals that are
413	* powers of 2 times 1ms
414	*/
415	bit = EHCI_VIRTUAL_FRAMELIST_COUNT / 2;
416	while (bit) {
417	x = bit;
418	while (x & bit) {
419	ehci_qh_t *qh_x;
420	ehci_qh_t *qh_y;
421
422	y = (x ^ bit) \| (bit / 2);
423
424	qh_x = sc->sc_intr_p_last[x];
425	qh_y = sc->sc_intr_p_last[y];
426
427	/*
428	* the next QH has half the poll interval
429	*/
430	qh_x->qh_link = qh_y->qh_self;
431
432	x++;
433	}
434	bit >>= 1;
435	}
436
437	if (1) {
438	ehci_qh_t *qh;
439
440	qh = sc->sc_intr_p_last[0];
441
442	/* the last (1ms) QH terminates */
443	qh->qh_link = htohc32(sc, EHCI_LINK_TERMINATE);
444	}
445	for (i = 0; i < EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
446	ehci_sitd_t *sitd;
447	ehci_itd_t *itd;
448
449	usbd_get_page(sc->sc_hw.isoc_fs_start_pc + i, 0, &buf_res);
450
451	sitd = buf_res.buffer;
452
453	/* initialize page cache pointer */
454
455	sitd->page_cache = sc->sc_hw.isoc_fs_start_pc + i;
456
457	/* store a pointer to the transfer descriptor */
458
459	sc->sc_isoc_fs_p_last[i] = sitd;
460
461	/* initialize full speed isochronous */
462
463	sitd->sitd_self =
464	htohc32(sc, buf_res.physaddr) \|
465	htohc32(sc, EHCI_LINK_SITD);
466
467	sitd->sitd_back =
468	htohc32(sc, EHCI_LINK_TERMINATE);
469
470	sitd->sitd_next =
471	sc->sc_intr_p_last[i \| (EHCI_VIRTUAL_FRAMELIST_COUNT / 2)]->qh_self;
472
473
474	usbd_get_page(sc->sc_hw.isoc_hs_start_pc + i, 0, &buf_res);
475
476	itd = buf_res.buffer;
477
478	/* initialize page cache pointer */
479
480	itd->page_cache = sc->sc_hw.isoc_hs_start_pc + i;
481
482	/* store a pointer to the transfer descriptor */
483
484	sc->sc_isoc_hs_p_last[i] = itd;
485
486	/* initialize high speed isochronous */
487
488	itd->itd_self =
489	htohc32(sc, buf_res.physaddr) \|
490	htohc32(sc, EHCI_LINK_ITD);
491
492	itd->itd_next =
493	sitd->sitd_self;
494	}
495
496	usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
497
498	if (1) {
499	uint32_t *pframes;
500
501	pframes = buf_res.buffer;
502
503	/*
504	* execution order:
505	* pframes -> high speed isochronous ->
506	* full speed isochronous -> interrupt QH's
507	*/
508	for (i = 0; i < EHCI_FRAMELIST_COUNT; i++) {
509	pframes[i] = sc->sc_isoc_hs_p_last
510	[i & (EHCI_VIRTUAL_FRAMELIST_COUNT - 1)]->itd_self;
511	}
512	}
513	usbd_get_page(&sc->sc_hw.async_start_pc, 0, &buf_res);
514
515	if (1) {
516
517	ehci_qh_t *qh;
518
519	qh = buf_res.buffer;
520
521	/* initialize page cache pointer */
522
523	qh->page_cache = &sc->sc_hw.async_start_pc;
524
525	/* store a pointer to the queue head */
526
527	sc->sc_async_p_last = qh;
528
529	/* init dummy QH that starts the async list */
530
531	qh->qh_self =
532	htohc32(sc, buf_res.physaddr) \|
533	htohc32(sc, EHCI_LINK_QH);
534
535	/* fill the QH */
536	qh->qh_endp =
537	htohc32(sc, EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH) \| EHCI_QH_HRECL);
538	qh->qh_endphub = htohc32(sc, EHCI_QH_SET_MULT(1));
539	qh->qh_link = qh->qh_self;
540	qh->qh_curqtd = 0;
541
542	/* fill the overlay qTD */
543	qh->qh_qtd.qtd_next = htohc32(sc, EHCI_LINK_TERMINATE);
544	qh->qh_qtd.qtd_altnext = htohc32(sc, EHCI_LINK_TERMINATE);
545	qh->qh_qtd.qtd_status = htohc32(sc, EHCI_QTD_HALTED);
546	}
547	/* flush all cache into memory */
548
549	usb_bus_mem_flush_all(&sc->sc_bus, &ehci_iterate_hw_softc);
550
551	#ifdef USB_DEBUG
552	if (ehcidebug) {
553	ehci_dump_sqh(sc, sc->sc_async_p_last);
554	}
555	#endif
556
557	/* finial setup */
558	err = ehci_init_sub(sc);
559
560	if (!err) {
561	/* catch any lost interrupts */
562	ehci_do_poll(&sc->sc_bus);
563	}
564	return (err);
565	}
566
567	/*
568	* shut down the controller when the system is going down
569	*/
570	void
571	ehci_detach(ehci_softc_t *sc)
572	{
573	USB_BUS_LOCK(&sc->sc_bus);
574
575	usb_callout_stop(&sc->sc_tmo_pcd);
576	usb_callout_stop(&sc->sc_tmo_poll);
577
578	EOWRITE4(sc, EHCI_USBINTR, 0);
579	USB_BUS_UNLOCK(&sc->sc_bus);
580
581	if (ehci_hcreset(sc)) {
582	DPRINTF("reset failed!\n");
583	}
584
585	/* XXX let stray task complete */
586	usb_pause_mtx(NULL, hz / 20);
587
588	usb_callout_drain(&sc->sc_tmo_pcd);
589	usb_callout_drain(&sc->sc_tmo_poll);
590	}
591
592	#ifndef __rtems__
593	static void
594	#else /* __rtems__ */
595	void
596	#endif /* __rtems__ */
597	ehci_suspend(ehci_softc_t *sc)
598	{
599	DPRINTF("stopping the HC\n");
600
601	/* reset HC */
602	ehci_hcreset(sc);
603	}
604
605	#ifndef __rtems__
606	static void
607	#else /* __rtems__ */
608	void
609	#endif /* __rtems__ */
610	ehci_resume(ehci_softc_t *sc)
611	{
612	/* reset HC */
613	ehci_hcreset(sc);
614
615	/* setup HC */
616	ehci_init_sub(sc);
617
618	/* catch any lost interrupts */
619	ehci_do_poll(&sc->sc_bus);
620	}
621
622	#ifdef USB_DEBUG
623	static void
624	ehci_dump_regs(ehci_softc_t *sc)
625	{
626	uint32_t i;
627
628	i = EOREAD4(sc, EHCI_USBCMD);
629	printf("cmd=0x%08x\n", i);
630
631	if (i & EHCI_CMD_ITC_1)
632	printf(" EHCI_CMD_ITC_1\n");
633	if (i & EHCI_CMD_ITC_2)
634	printf(" EHCI_CMD_ITC_2\n");
635	if (i & EHCI_CMD_ITC_4)
636	printf(" EHCI_CMD_ITC_4\n");
637	if (i & EHCI_CMD_ITC_8)
638	printf(" EHCI_CMD_ITC_8\n");
639	if (i & EHCI_CMD_ITC_16)
640	printf(" EHCI_CMD_ITC_16\n");
641	if (i & EHCI_CMD_ITC_32)
642	printf(" EHCI_CMD_ITC_32\n");
643	if (i & EHCI_CMD_ITC_64)
644	printf(" EHCI_CMD_ITC_64\n");
645	if (i & EHCI_CMD_ASPME)
646	printf(" EHCI_CMD_ASPME\n");
647	if (i & EHCI_CMD_ASPMC)
648	printf(" EHCI_CMD_ASPMC\n");
649	if (i & EHCI_CMD_LHCR)
650	printf(" EHCI_CMD_LHCR\n");
651	if (i & EHCI_CMD_IAAD)
652	printf(" EHCI_CMD_IAAD\n");
653	if (i & EHCI_CMD_ASE)
654	printf(" EHCI_CMD_ASE\n");
655	if (i & EHCI_CMD_PSE)
656	printf(" EHCI_CMD_PSE\n");
657	if (i & EHCI_CMD_FLS_M)
658	printf(" EHCI_CMD_FLS_M\n");
659	if (i & EHCI_CMD_HCRESET)
660	printf(" EHCI_CMD_HCRESET\n");
661	if (i & EHCI_CMD_RS)
662	printf(" EHCI_CMD_RS\n");
663
664	i = EOREAD4(sc, EHCI_USBSTS);
665
666	printf("sts=0x%08x\n", i);
667
668	if (i & EHCI_STS_ASS)
669	printf(" EHCI_STS_ASS\n");
670	if (i & EHCI_STS_PSS)
671	printf(" EHCI_STS_PSS\n");
672	if (i & EHCI_STS_REC)
673	printf(" EHCI_STS_REC\n");
674	if (i & EHCI_STS_HCH)
675	printf(" EHCI_STS_HCH\n");
676	if (i & EHCI_STS_IAA)
677	printf(" EHCI_STS_IAA\n");
678	if (i & EHCI_STS_HSE)
679	printf(" EHCI_STS_HSE\n");
680	if (i & EHCI_STS_FLR)
681	printf(" EHCI_STS_FLR\n");
682	if (i & EHCI_STS_PCD)
683	printf(" EHCI_STS_PCD\n");
684	if (i & EHCI_STS_ERRINT)
685	printf(" EHCI_STS_ERRINT\n");
686	if (i & EHCI_STS_INT)
687	printf(" EHCI_STS_INT\n");
688
689	printf("ien=0x%08x\n",
690	EOREAD4(sc, EHCI_USBINTR));
691	printf("frindex=0x%08x ctrdsegm=0x%08x periodic=0x%08x async=0x%08x\n",
692	EOREAD4(sc, EHCI_FRINDEX),
693	EOREAD4(sc, EHCI_CTRLDSSEGMENT),
694	EOREAD4(sc, EHCI_PERIODICLISTBASE),
695	EOREAD4(sc, EHCI_ASYNCLISTADDR));
696	for (i = 1; i <= sc->sc_noport; i++) {
697	printf("port %d status=0x%08x\n", i,
698	EOREAD4(sc, EHCI_PORTSC(i)));
699	}
700	}
701
702	static void
703	ehci_dump_link(ehci_softc_t *sc, uint32_t link, int type)
704	{
705	link = hc32toh(sc, link);
706	printf("0x%08x", link);
707	if (link & EHCI_LINK_TERMINATE)
708	printf("<T>");
709	else {
710	printf("<");
711	if (type) {
712	switch (EHCI_LINK_TYPE(link)) {
713	case EHCI_LINK_ITD:
714	printf("ITD");
715	break;
716	case EHCI_LINK_QH:
717	printf("QH");
718	break;
719	case EHCI_LINK_SITD:
720	printf("SITD");
721	break;
722	case EHCI_LINK_FSTN:
723	printf("FSTN");
724	break;
725	}
726	}
727	printf(">");
728	}
729	}
730
731	static void
732	ehci_dump_qtd(ehci_softc_t sc, ehci_qtd_t qtd)
733	{
734	uint32_t s;
735
736	printf(" next=");
737	ehci_dump_link(sc, qtd->qtd_next, 0);
738	printf(" altnext=");
739	ehci_dump_link(sc, qtd->qtd_altnext, 0);
740	printf("\n");
741	s = hc32toh(sc, qtd->qtd_status);
742	printf(" status=0x%08x: toggle=%d bytes=0x%x ioc=%d c_page=0x%x\n",
743	s, EHCI_QTD_GET_TOGGLE(s), EHCI_QTD_GET_BYTES(s),
744	EHCI_QTD_GET_IOC(s), EHCI_QTD_GET_C_PAGE(s));
745	printf(" cerr=%d pid=%d stat=%s%s%s%s%s%s%s%s\n",
746	EHCI_QTD_GET_CERR(s), EHCI_QTD_GET_PID(s),
747	(s & EHCI_QTD_ACTIVE) ? "ACTIVE" : "NOT_ACTIVE",
748	(s & EHCI_QTD_HALTED) ? "-HALTED" : "",
749	(s & EHCI_QTD_BUFERR) ? "-BUFERR" : "",
750	(s & EHCI_QTD_BABBLE) ? "-BABBLE" : "",
751	(s & EHCI_QTD_XACTERR) ? "-XACTERR" : "",
752	(s & EHCI_QTD_MISSEDMICRO) ? "-MISSED" : "",
753	(s & EHCI_QTD_SPLITXSTATE) ? "-SPLIT" : "",
754	(s & EHCI_QTD_PINGSTATE) ? "-PING" : "");
755
756	for (s = 0; s < 5; s++) {
757	printf(" buffer[%d]=0x%08x\n", s,
758	hc32toh(sc, qtd->qtd_buffer[s]));
759	}
760	for (s = 0; s < 5; s++) {
761	printf(" buffer_hi[%d]=0x%08x\n", s,
762	hc32toh(sc, qtd->qtd_buffer_hi[s]));
763	}
764	}
765
766	static uint8_t
767	ehci_dump_sqtd(ehci_softc_t sc, ehci_qtd_t sqtd)
768	{
769	uint8_t temp;
770
771	usb_pc_cpu_invalidate(sqtd->page_cache);
772	printf("QTD(%p) at 0x%08x:\n", sqtd, hc32toh(sc, sqtd->qtd_self));
773	ehci_dump_qtd(sc, sqtd);
774	temp = (sqtd->qtd_next & htohc32(sc, EHCI_LINK_TERMINATE)) ? 1 : 0;
775	return (temp);
776	}
777
778	static void
779	ehci_dump_sqtds(ehci_softc_t sc, ehci_qtd_t sqtd)
780	{
781	uint16_t i;
782	uint8_t stop;
783
784	stop = 0;
785	for (i = 0; sqtd && (i < 20) && !stop; sqtd = sqtd->obj_next, i++) {
786	stop = ehci_dump_sqtd(sc, sqtd);
787	}
788	if (sqtd) {
789	printf("dump aborted, too many TDs\n");
790	}
791	}
792
793	static void
794	ehci_dump_sqh(ehci_softc_t sc, ehci_qh_t qh)
795	{
796	uint32_t endp;
797	uint32_t endphub;
798
799	usb_pc_cpu_invalidate(qh->page_cache);
800	printf("QH(%p) at 0x%08x:\n", qh, hc32toh(sc, qh->qh_self) & ~0x1F);
801	printf(" link=");
802	ehci_dump_link(sc, qh->qh_link, 1);
803	printf("\n");
804	endp = hc32toh(sc, qh->qh_endp);
805	printf(" endp=0x%08x\n", endp);
806	printf(" addr=0x%02x inact=%d endpt=%d eps=%d dtc=%d hrecl=%d\n",
807	EHCI_QH_GET_ADDR(endp), EHCI_QH_GET_INACT(endp),
808	EHCI_QH_GET_ENDPT(endp), EHCI_QH_GET_EPS(endp),
809	EHCI_QH_GET_DTC(endp), EHCI_QH_GET_HRECL(endp));
810	printf(" mpl=0x%x ctl=%d nrl=%d\n",
811	EHCI_QH_GET_MPL(endp), EHCI_QH_GET_CTL(endp),
812	EHCI_QH_GET_NRL(endp));
813	endphub = hc32toh(sc, qh->qh_endphub);
814	printf(" endphub=0x%08x\n", endphub);
815	printf(" smask=0x%02x cmask=0x%02x huba=0x%02x port=%d mult=%d\n",
816	EHCI_QH_GET_SMASK(endphub), EHCI_QH_GET_CMASK(endphub),
817	EHCI_QH_GET_HUBA(endphub), EHCI_QH_GET_PORT(endphub),
818	EHCI_QH_GET_MULT(endphub));
819	printf(" curqtd=");
820	ehci_dump_link(sc, qh->qh_curqtd, 0);
821	printf("\n");
822	printf("Overlay qTD:\n");
823	ehci_dump_qtd(sc, (void *)&qh->qh_qtd);
824	}
825
826	static void
827	ehci_dump_sitd(ehci_softc_t sc, ehci_sitd_t sitd)
828	{
829	usb_pc_cpu_invalidate(sitd->page_cache);
830	printf("SITD(%p) at 0x%08x\n", sitd, hc32toh(sc, sitd->sitd_self) & ~0x1F);
831	printf(" next=0x%08x\n", hc32toh(sc, sitd->sitd_next));
832	printf(" portaddr=0x%08x dir=%s addr=%d endpt=0x%x port=0x%x huba=0x%x\n",
833	hc32toh(sc, sitd->sitd_portaddr),
834	(sitd->sitd_portaddr & htohc32(sc, EHCI_SITD_SET_DIR_IN))
835	? "in" : "out",
836	EHCI_SITD_GET_ADDR(hc32toh(sc, sitd->sitd_portaddr)),
837	EHCI_SITD_GET_ENDPT(hc32toh(sc, sitd->sitd_portaddr)),
838	EHCI_SITD_GET_PORT(hc32toh(sc, sitd->sitd_portaddr)),
839	EHCI_SITD_GET_HUBA(hc32toh(sc, sitd->sitd_portaddr)));
840	printf(" mask=0x%08x\n", hc32toh(sc, sitd->sitd_mask));
841	printf(" status=0x%08x <%s> len=0x%x\n", hc32toh(sc, sitd->sitd_status),
842	(sitd->sitd_status & htohc32(sc, EHCI_SITD_ACTIVE)) ? "ACTIVE" : "",
843	EHCI_SITD_GET_LEN(hc32toh(sc, sitd->sitd_status)));
844	printf(" back=0x%08x, bp=0x%08x,0x%08x,0x%08x,0x%08x\n",
845	hc32toh(sc, sitd->sitd_back),
846	hc32toh(sc, sitd->sitd_bp[0]),
847	hc32toh(sc, sitd->sitd_bp[1]),
848	hc32toh(sc, sitd->sitd_bp_hi[0]),
849	hc32toh(sc, sitd->sitd_bp_hi[1]));
850	}
851
852	static void
853	ehci_dump_itd(ehci_softc_t sc, ehci_itd_t itd)
854	{
855	usb_pc_cpu_invalidate(itd->page_cache);
856	printf("ITD(%p) at 0x%08x\n", itd, hc32toh(sc, itd->itd_self) & ~0x1F);
857	printf(" next=0x%08x\n", hc32toh(sc, itd->itd_next));
858	printf(" status[0]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[0]),
859	(itd->itd_status[0] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
860	printf(" status[1]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[1]),
861	(itd->itd_status[1] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
862	printf(" status[2]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[2]),
863	(itd->itd_status[2] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
864	printf(" status[3]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[3]),
865	(itd->itd_status[3] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
866	printf(" status[4]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[4]),
867	(itd->itd_status[4] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
868	printf(" status[5]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[5]),
869	(itd->itd_status[5] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
870	printf(" status[6]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[6]),
871	(itd->itd_status[6] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
872	printf(" status[7]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[7]),
873	(itd->itd_status[7] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
874	printf(" bp[0]=0x%08x\n", hc32toh(sc, itd->itd_bp[0]));
875	printf(" addr=0x%02x; endpt=0x%01x\n",
876	EHCI_ITD_GET_ADDR(hc32toh(sc, itd->itd_bp[0])),
877	EHCI_ITD_GET_ENDPT(hc32toh(sc, itd->itd_bp[0])));
878	printf(" bp[1]=0x%08x\n", hc32toh(sc, itd->itd_bp[1]));
879	printf(" dir=%s; mpl=0x%02x\n",
880	(hc32toh(sc, itd->itd_bp[1]) & EHCI_ITD_SET_DIR_IN) ? "in" : "out",
881	EHCI_ITD_GET_MPL(hc32toh(sc, itd->itd_bp[1])));
882	printf(" bp[2..6]=0x%08x,0x%08x,0x%08x,0x%08x,0x%08x\n",
883	hc32toh(sc, itd->itd_bp[2]),
884	hc32toh(sc, itd->itd_bp[3]),
885	hc32toh(sc, itd->itd_bp[4]),
886	hc32toh(sc, itd->itd_bp[5]),
887	hc32toh(sc, itd->itd_bp[6]));
888	printf(" bp_hi=0x%08x,0x%08x,0x%08x,0x%08x,\n"
889	" 0x%08x,0x%08x,0x%08x\n",
890	hc32toh(sc, itd->itd_bp_hi[0]),
891	hc32toh(sc, itd->itd_bp_hi[1]),
892	hc32toh(sc, itd->itd_bp_hi[2]),
893	hc32toh(sc, itd->itd_bp_hi[3]),
894	hc32toh(sc, itd->itd_bp_hi[4]),
895	hc32toh(sc, itd->itd_bp_hi[5]),
896	hc32toh(sc, itd->itd_bp_hi[6]));
897	}
898
899	static void
900	ehci_dump_isoc(ehci_softc_t *sc)
901	{
902	ehci_itd_t *itd;
903	ehci_sitd_t *sitd;
904	uint16_t max = 1000;
905	uint16_t pos;
906
907	pos = (EOREAD4(sc, EHCI_FRINDEX) / 8) &
908	(EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
909
910	printf("%s: isochronous dump from frame 0x%03x:\n",
911	__FUNCTION__, pos);
912
913	itd = sc->sc_isoc_hs_p_last[pos];
914	sitd = sc->sc_isoc_fs_p_last[pos];
915
916	while (itd && max && max--) {
917	ehci_dump_itd(sc, itd);
918	itd = itd->prev;
919	}
920
921	while (sitd && max && max--) {
922	ehci_dump_sitd(sc, sitd);
923	sitd = sitd->prev;
924	}
925	}
926
927	#endif
928
929	static void
930	ehci_transfer_intr_enqueue(struct usb_xfer *xfer)
931	{
932	/* check for early completion */
933	if (ehci_check_transfer(xfer)) {
934	return;
935	}
936	/* put transfer on interrupt queue */
937	usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
938
939	/* start timeout, if any */
940	if (xfer->timeout != 0) {
941	usbd_transfer_timeout_ms(xfer, &ehci_timeout, xfer->timeout);
942	}
943	}
944
945	#define EHCI_APPEND_FS_TD(std,last) (last) = _ehci_append_fs_td(std,last)
946	static ehci_sitd_t *
947	_ehci_append_fs_td(ehci_sitd_t std, ehci_sitd_t last)
948	{
949	DPRINTFN(11, "%p to %p\n", std, last);
950
951	/* (sc->sc_bus.mtx) must be locked */
952
953	std->next = last->next;
954	std->sitd_next = last->sitd_next;
955
956	std->prev = last;
957
958	usb_pc_cpu_flush(std->page_cache);
959
960	/*
961	* the last->next->prev is never followed: std->next->prev = std;
962	*/
963	last->next = std;
964	last->sitd_next = std->sitd_self;
965
966	usb_pc_cpu_flush(last->page_cache);
967
968	return (std);
969	}
970
971	#define EHCI_APPEND_HS_TD(std,last) (last) = _ehci_append_hs_td(std,last)
972	static ehci_itd_t *
973	_ehci_append_hs_td(ehci_itd_t std, ehci_itd_t last)
974	{
975	DPRINTFN(11, "%p to %p\n", std, last);
976
977	/* (sc->sc_bus.mtx) must be locked */
978
979	std->next = last->next;
980	std->itd_next = last->itd_next;
981
982	std->prev = last;
983
984	usb_pc_cpu_flush(std->page_cache);
985
986	/*
987	* the last->next->prev is never followed: std->next->prev = std;
988	*/
989	last->next = std;
990	last->itd_next = std->itd_self;
991
992	usb_pc_cpu_flush(last->page_cache);
993
994	return (std);
995	}
996
997	#define EHCI_APPEND_QH(sqh,last) (last) = _ehci_append_qh(sqh,last)
998	static ehci_qh_t *
999	_ehci_append_qh(ehci_qh_t sqh, ehci_qh_t last)
1000	{
1001	DPRINTFN(11, "%p to %p\n", sqh, last);
1002
1003	if (sqh->prev != NULL) {
1004	/* should not happen */
1005	DPRINTFN(0, "QH already linked!\n");
1006	return (last);
1007	}
1008	/* (sc->sc_bus.mtx) must be locked */
1009
1010	sqh->next = last->next;
1011	sqh->qh_link = last->qh_link;
1012
1013	sqh->prev = last;
1014
1015	usb_pc_cpu_flush(sqh->page_cache);
1016
1017	/*
1018	* the last->next->prev is never followed: sqh->next->prev = sqh;
1019	*/
1020
1021	last->next = sqh;
1022	last->qh_link = sqh->qh_self;
1023
1024	usb_pc_cpu_flush(last->page_cache);
1025
1026	return (sqh);
1027	}
1028
1029	#define EHCI_REMOVE_FS_TD(std,last) (last) = _ehci_remove_fs_td(std,last)
1030	static ehci_sitd_t *
1031	_ehci_remove_fs_td(ehci_sitd_t std, ehci_sitd_t last)
1032	{
1033	DPRINTFN(11, "%p from %p\n", std, last);
1034
1035	/* (sc->sc_bus.mtx) must be locked */
1036
1037	std->prev->next = std->next;
1038	std->prev->sitd_next = std->sitd_next;
1039
1040	usb_pc_cpu_flush(std->prev->page_cache);
1041
1042	if (std->next) {
1043	std->next->prev = std->prev;
1044	usb_pc_cpu_flush(std->next->page_cache);
1045	}
1046	return ((last == std) ? std->prev : last);
1047	}
1048
1049	#define EHCI_REMOVE_HS_TD(std,last) (last) = _ehci_remove_hs_td(std,last)
1050	static ehci_itd_t *
1051	_ehci_remove_hs_td(ehci_itd_t std, ehci_itd_t last)
1052	{
1053	DPRINTFN(11, "%p from %p\n", std, last);
1054
1055	/* (sc->sc_bus.mtx) must be locked */
1056
1057	std->prev->next = std->next;
1058	std->prev->itd_next = std->itd_next;
1059
1060	usb_pc_cpu_flush(std->prev->page_cache);
1061
1062	if (std->next) {
1063	std->next->prev = std->prev;
1064	usb_pc_cpu_flush(std->next->page_cache);
1065	}
1066	return ((last == std) ? std->prev : last);
1067	}
1068
1069	#define EHCI_REMOVE_QH(sqh,last) (last) = _ehci_remove_qh(sqh,last)
1070	static ehci_qh_t *
1071	_ehci_remove_qh(ehci_qh_t sqh, ehci_qh_t last)
1072	{
1073	DPRINTFN(11, "%p from %p\n", sqh, last);
1074
1075	/* (sc->sc_bus.mtx) must be locked */
1076
1077	/* only remove if not removed from a queue */
1078	if (sqh->prev) {
1079
1080	sqh->prev->next = sqh->next;
1081	sqh->prev->qh_link = sqh->qh_link;
1082
1083	usb_pc_cpu_flush(sqh->prev->page_cache);
1084
1085	if (sqh->next) {
1086	sqh->next->prev = sqh->prev;
1087	usb_pc_cpu_flush(sqh->next->page_cache);
1088	}
1089	last = ((last == sqh) ? sqh->prev : last);
1090
1091	sqh->prev = 0;
1092
1093	usb_pc_cpu_flush(sqh->page_cache);
1094	}
1095	return (last);
1096	}
1097
1098	static void
1099	ehci_data_toggle_update(struct usb_xfer *xfer, uint16_t actlen, uint16_t xlen)
1100	{
1101	uint16_t rem;
1102	uint8_t dt;
1103
1104	/* count number of full packets */
1105	dt = (actlen / xfer->max_packet_size) & 1;
1106
1107	/* compute remainder */
1108	rem = actlen % xfer->max_packet_size;
1109
1110	if (rem > 0)
1111	dt ^= 1; /* short packet at the end */
1112	else if (actlen != xlen)
1113	dt ^= 1; /* zero length packet at the end */
1114	else if (xlen == 0)
1115	dt ^= 1; /* zero length transfer */
1116
1117	xfer->endpoint->toggle_next ^= dt;
1118	}
1119
1120	static usb_error_t
1121	ehci_non_isoc_done_sub(struct usb_xfer *xfer)
1122	{
1123	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1124	ehci_qtd_t *td;
1125	ehci_qtd_t *td_alt_next;
1126	uint32_t status;
1127	uint16_t len;
1128
1129	td = xfer->td_transfer_cache;
1130	td_alt_next = td->alt_next;
1131
1132	if (xfer->aframes != xfer->nframes) {
1133	usbd_xfer_set_frame_len(xfer, xfer->aframes, 0);
1134	}
1135	while (1) {
1136
1137	usb_pc_cpu_invalidate(td->page_cache);
1138	status = hc32toh(sc, td->qtd_status);
1139
1140	len = EHCI_QTD_GET_BYTES(status);
1141
1142	/*
1143	* Verify the status length and
1144	* add the length to "frlengths[]":
1145	*/
1146	if (len > td->len) {
1147	/* should not happen */
1148	DPRINTF("Invalid status length, "
1149	"0x%04x/0x%04x bytes\n", len, td->len);
1150	status \|= EHCI_QTD_HALTED;
1151	} else if (xfer->aframes != xfer->nframes) {
1152	xfer->frlengths[xfer->aframes] += td->len - len;
1153	/* manually update data toggle */
1154	ehci_data_toggle_update(xfer, td->len - len, td->len);
1155	}
1156
1157	/* Check for last transfer */
1158	if (((void *)td) == xfer->td_transfer_last) {
1159	td = NULL;
1160	break;
1161	}
1162	/* Check for transfer error */
1163	if (status & EHCI_QTD_HALTED) {
1164	/* the transfer is finished */
1165	td = NULL;
1166	break;
1167	}
1168	/* Check for short transfer */
1169	if (len > 0) {
1170	if (xfer->flags_int.short_frames_ok) {
1171	/* follow alt next */
1172	td = td->alt_next;
1173	} else {
1174	/* the transfer is finished */
1175	td = NULL;
1176	}
1177	break;
1178	}
1179	td = td->obj_next;
1180
1181	if (td->alt_next != td_alt_next) {
1182	/* this USB frame is complete */
1183	break;
1184	}
1185	}
1186
1187	/* update transfer cache */
1188
1189	xfer->td_transfer_cache = td;
1190
1191	#ifdef USB_DEBUG
1192	if (status & EHCI_QTD_STATERRS) {
1193	DPRINTFN(11, "error, addr=%d, endpt=0x%02x, frame=0x%02x"
1194	"status=%s%s%s%s%s%s%s%s\n",
1195	xfer->address, xfer->endpointno, xfer->aframes,
1196	(status & EHCI_QTD_ACTIVE) ? "[ACTIVE]" : "[NOT_ACTIVE]",
1197	(status & EHCI_QTD_HALTED) ? "[HALTED]" : "",
1198	(status & EHCI_QTD_BUFERR) ? "[BUFERR]" : "",
1199	(status & EHCI_QTD_BABBLE) ? "[BABBLE]" : "",
1200	(status & EHCI_QTD_XACTERR) ? "[XACTERR]" : "",
1201	(status & EHCI_QTD_MISSEDMICRO) ? "[MISSED]" : "",
1202	(status & EHCI_QTD_SPLITXSTATE) ? "[SPLIT]" : "",
1203	(status & EHCI_QTD_PINGSTATE) ? "[PING]" : "");
1204	}
1205	#endif
1206	if (status & EHCI_QTD_HALTED) {
1207	if ((xfer->xroot->udev->parent_hs_hub != NULL) \|\|
1208	(xfer->xroot->udev->address != 0)) {
1209	/* try to separate I/O errors from STALL */
1210	if (EHCI_QTD_GET_CERR(status) == 0)
1211	return (USB_ERR_IOERROR);
1212	}
1213	return (USB_ERR_STALLED);
1214	}
1215	return (USB_ERR_NORMAL_COMPLETION);
1216	}
1217
1218	static void
1219	ehci_non_isoc_done(struct usb_xfer *xfer)
1220	{
1221	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1222	ehci_qh_t *qh;
1223	uint32_t status;
1224	usb_error_t err = 0;
1225
1226	DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
1227	xfer, xfer->endpoint);
1228
1229	#ifdef USB_DEBUG
1230	if (ehcidebug > 10) {
1231	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1232
1233	ehci_dump_sqtds(sc, xfer->td_transfer_first);
1234	}
1235	#endif
1236
1237	/* extract data toggle directly from the QH's overlay area */
1238
1239	qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1240
1241	usb_pc_cpu_invalidate(qh->page_cache);
1242
1243	status = hc32toh(sc, qh->qh_qtd.qtd_status);
1244
1245	/* reset scanner */
1246
1247	xfer->td_transfer_cache = xfer->td_transfer_first;
1248
1249	if (xfer->flags_int.control_xfr) {
1250
1251	if (xfer->flags_int.control_hdr) {
1252
1253	err = ehci_non_isoc_done_sub(xfer);
1254	}
1255	xfer->aframes = 1;
1256
1257	if (xfer->td_transfer_cache == NULL) {
1258	goto done;
1259	}
1260	}
1261	while (xfer->aframes != xfer->nframes) {
1262
1263	err = ehci_non_isoc_done_sub(xfer);
1264	xfer->aframes++;
1265
1266	if (xfer->td_transfer_cache == NULL) {
1267	goto done;
1268	}
1269	}
1270
1271	if (xfer->flags_int.control_xfr &&
1272	!xfer->flags_int.control_act) {
1273
1274	err = ehci_non_isoc_done_sub(xfer);
1275	}
1276	done:
1277	ehci_device_done(xfer, err);
1278	}
1279
1280	/------------------------------------------------------------------------
1281	* ehci_check_transfer
1282	*
1283	* Return values:
1284	* 0: USB transfer is not finished
1285	* Else: USB transfer is finished
1286	------------------------------------------------------------------------/
1287	static uint8_t
1288	ehci_check_transfer(struct usb_xfer *xfer)
1289	{
1290	const struct usb_pipe_methods *methods = xfer->endpoint->methods;
1291	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1292
1293	uint32_t status;
1294
1295	DPRINTFN(13, "xfer=%p checking transfer\n", xfer);
1296
1297	if (methods == &ehci_device_isoc_fs_methods) {
1298	ehci_sitd_t *td;
1299
1300	/* isochronous full speed transfer */
1301
1302	td = xfer->td_transfer_last;
1303	usb_pc_cpu_invalidate(td->page_cache);
1304	status = hc32toh(sc, td->sitd_status);
1305
1306	/* also check if first is complete */
1307
1308	td = xfer->td_transfer_first;
1309	usb_pc_cpu_invalidate(td->page_cache);
1310	status \|= hc32toh(sc, td->sitd_status);
1311
1312	if (!(status & EHCI_SITD_ACTIVE)) {
1313	ehci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1314	goto transferred;
1315	}
1316	} else if (methods == &ehci_device_isoc_hs_methods) {
1317	ehci_itd_t *td;
1318
1319	/* isochronous high speed transfer */
1320
1321	/* check last transfer */
1322	td = xfer->td_transfer_last;
1323	usb_pc_cpu_invalidate(td->page_cache);
1324	status = td->itd_status[0];
1325	status \|= td->itd_status[1];
1326	status \|= td->itd_status[2];
1327	status \|= td->itd_status[3];
1328	status \|= td->itd_status[4];
1329	status \|= td->itd_status[5];
1330	status \|= td->itd_status[6];
1331	status \|= td->itd_status[7];
1332
1333	/* also check first transfer */
1334	td = xfer->td_transfer_first;
1335	usb_pc_cpu_invalidate(td->page_cache);
1336	status \|= td->itd_status[0];
1337	status \|= td->itd_status[1];
1338	status \|= td->itd_status[2];
1339	status \|= td->itd_status[3];
1340	status \|= td->itd_status[4];
1341	status \|= td->itd_status[5];
1342	status \|= td->itd_status[6];
1343	status \|= td->itd_status[7];
1344
1345	/* if no transactions are active we continue */
1346	if (!(status & htohc32(sc, EHCI_ITD_ACTIVE))) {
1347	ehci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1348	goto transferred;
1349	}
1350	} else {
1351	ehci_qtd_t *td;
1352	ehci_qh_t *qh;
1353
1354	/* non-isochronous transfer */
1355
1356	/*
1357	* check whether there is an error somewhere in the middle,
1358	* or whether there was a short packet (SPD and not ACTIVE)
1359	*/
1360	td = xfer->td_transfer_cache;
1361
1362	qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1363
1364	usb_pc_cpu_invalidate(qh->page_cache);
1365
1366	status = hc32toh(sc, qh->qh_qtd.qtd_status);
1367	if (status & EHCI_QTD_ACTIVE) {
1368	/* transfer is pending */
1369	goto done;
1370	}
1371
1372	while (1) {
1373	usb_pc_cpu_invalidate(td->page_cache);
1374	status = hc32toh(sc, td->qtd_status);
1375
1376	/*
1377	* Check if there is an active TD which
1378	* indicates that the transfer isn't done.
1379	*/
1380	if (status & EHCI_QTD_ACTIVE) {
1381	/* update cache */
1382	xfer->td_transfer_cache = td;
1383	goto done;
1384	}
1385	/*
1386	* last transfer descriptor makes the transfer done
1387	*/
1388	if (((void *)td) == xfer->td_transfer_last) {
1389	break;
1390	}
1391	/*
1392	* any kind of error makes the transfer done
1393	*/
1394	if (status & EHCI_QTD_HALTED) {
1395	break;
1396	}
1397	/*
1398	* if there is no alternate next transfer, a short
1399	* packet also makes the transfer done
1400	*/
1401	if (EHCI_QTD_GET_BYTES(status)) {
1402	if (xfer->flags_int.short_frames_ok) {
1403	/* follow alt next */
1404	if (td->alt_next) {
1405	td = td->alt_next;
1406	continue;
1407	}
1408	}
1409	/* transfer is done */
1410	break;
1411	}
1412	td = td->obj_next;
1413	}
1414	ehci_non_isoc_done(xfer);
1415	goto transferred;
1416	}
1417
1418	done:
1419	DPRINTFN(13, "xfer=%p is still active\n", xfer);
1420	return (0);
1421
1422	transferred:
1423	return (1);
1424	}
1425
1426	static void
1427	ehci_pcd_enable(ehci_softc_t *sc)
1428	{
1429	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
1430
1431	sc->sc_eintrs \|= EHCI_STS_PCD;
1432	EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1433
1434	/* acknowledge any PCD interrupt */
1435	EOWRITE4(sc, EHCI_USBSTS, EHCI_STS_PCD);
1436
1437	ehci_root_intr(sc);
1438	}
1439
1440	static void
1441	ehci_interrupt_poll(ehci_softc_t *sc)
1442	{
1443	struct usb_xfer *xfer;
1444
1445	repeat:
1446	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
1447	/*
1448	* check if transfer is transferred
1449	*/
1450	if (ehci_check_transfer(xfer)) {
1451	/* queue has been modified */
1452	goto repeat;
1453	}
1454	}
1455	}
1456
1457	/*
1458	* Some EHCI chips from VIA / ATI seem to trigger interrupts before
1459	* writing back the qTD status, or miss signalling occasionally under
1460	* heavy load. If the host machine is too fast, we can miss
1461	* transaction completion - when we scan the active list the
1462	* transaction still seems to be active. This generally exhibits
1463	* itself as a umass stall that never recovers.
1464	*
1465	* We work around this behaviour by setting up this callback after any
1466	* softintr that completes with transactions still pending, giving us
1467	* another chance to check for completion after the writeback has
1468	* taken place.
1469	*/
1470	static void
1471	ehci_poll_timeout(void *arg)
1472	{
1473	ehci_softc_t *sc = arg;
1474
1475	DPRINTFN(3, "\n");
1476	ehci_interrupt_poll(sc);
1477	}
1478
1479	/------------------------------------------------------------------------
1480	* ehci_interrupt - EHCI interrupt handler
1481	*
1482	* NOTE: Do not access "sc->sc_bus.bdev" inside the interrupt handler,
1483	* hence the interrupt handler will be setup before "sc->sc_bus.bdev"
1484	* is present !
1485	------------------------------------------------------------------------/
1486	void
1487	ehci_interrupt(ehci_softc_t *sc)
1488	{
1489	uint32_t status;
1490
1491	USB_BUS_LOCK(&sc->sc_bus);
1492
1493	DPRINTFN(16, "real interrupt\n");
1494
1495	#ifdef USB_DEBUG
1496	if (ehcidebug > 15) {
1497	ehci_dump_regs(sc);
1498	}
1499	#endif
1500
1501	status = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
1502	if (status == 0) {
1503	/* the interrupt was not for us */
1504	goto done;
1505	}
1506	if (!(status & sc->sc_eintrs)) {
1507	goto done;
1508	}
1509	EOWRITE4(sc, EHCI_USBSTS, status); /* acknowledge */
1510
1511	status &= sc->sc_eintrs;
1512
1513	if (status & EHCI_STS_HSE) {
1514	printf("%s: unrecoverable error, "
1515	"controller halted\n", __FUNCTION__);
1516	#ifdef USB_DEBUG
1517	ehci_dump_regs(sc);
1518	ehci_dump_isoc(sc);
1519	#endif
1520	}
1521	if (status & EHCI_STS_PCD) {
1522	/*
1523	* Disable PCD interrupt for now, because it will be
1524	* on until the port has been reset.
1525	*/
1526	sc->sc_eintrs &= ~EHCI_STS_PCD;
1527	EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1528
1529	ehci_root_intr(sc);
1530
1531	/* do not allow RHSC interrupts > 1 per second */
1532	usb_callout_reset(&sc->sc_tmo_pcd, hz,
1533	(void *)&ehci_pcd_enable, sc);
1534	}
1535	status &= ~(EHCI_STS_INT \| EHCI_STS_ERRINT \| EHCI_STS_PCD \| EHCI_STS_IAA);
1536
1537	if (status != 0) {
1538	/* block unprocessed interrupts */
1539	sc->sc_eintrs &= ~status;
1540	EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1541	printf("%s: blocking interrupts 0x%x\n", __FUNCTION__, status);
1542	}
1543	/* poll all the USB transfers */
1544	ehci_interrupt_poll(sc);
1545
1546	if (sc->sc_flags & EHCI_SCFLG_LOSTINTRBUG) {
1547	usb_callout_reset(&sc->sc_tmo_poll, hz / 128,
1548	(void *)&ehci_poll_timeout, sc);
1549	}
1550
1551	done:
1552	USB_BUS_UNLOCK(&sc->sc_bus);
1553	}
1554
1555	/*
1556	* called when a request does not complete
1557	*/
1558	static void
1559	ehci_timeout(void *arg)
1560	{
1561	struct usb_xfer *xfer = arg;
1562
1563	DPRINTF("xfer=%p\n", xfer);
1564
1565	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
1566
1567	/* transfer is transferred */
1568	ehci_device_done(xfer, USB_ERR_TIMEOUT);
1569	}
1570
1571	static void
1572	ehci_do_poll(struct usb_bus *bus)
1573	{
1574	ehci_softc_t *sc = EHCI_BUS2SC(bus);
1575
1576	USB_BUS_LOCK(&sc->sc_bus);
1577	ehci_interrupt_poll(sc);
1578	USB_BUS_UNLOCK(&sc->sc_bus);
1579	}
1580
1581	static void
1582	ehci_setup_standard_chain_sub(struct ehci_std_temp *temp)
1583	{
1584	struct usb_page_search buf_res;
1585	ehci_qtd_t *td;
1586	ehci_qtd_t *td_next;
1587	ehci_qtd_t *td_alt_next;
1588	uint32_t buf_offset;
1589	uint32_t average;
1590	uint32_t len_old;
1591	uint32_t terminate;
1592	uint32_t qtd_altnext;
1593	uint8_t shortpkt_old;
1594	uint8_t precompute;
1595
1596	terminate = temp->sc->sc_terminate_self;
1597	qtd_altnext = temp->sc->sc_terminate_self;
1598	td_alt_next = NULL;
1599	buf_offset = 0;
1600	shortpkt_old = temp->shortpkt;
1601	len_old = temp->len;
1602	precompute = 1;
1603
1604	restart:
1605
1606	td = temp->td;
1607	td_next = temp->td_next;
1608
1609	while (1) {
1610
1611	if (temp->len == 0) {
1612
1613	if (temp->shortpkt) {
1614	break;
1615	}
1616	/* send a Zero Length Packet, ZLP, last */
1617
1618	temp->shortpkt = 1;
1619	average = 0;
1620
1621	} else {
1622
1623	average = temp->average;
1624
1625	if (temp->len < average) {
1626	if (temp->len % temp->max_frame_size) {
1627	temp->shortpkt = 1;
1628	}
1629	average = temp->len;
1630	}
1631	}
1632
1633	if (td_next == NULL) {
1634	panic("%s: out of EHCI transfer descriptors!", __FUNCTION__);
1635	}
1636	/* get next TD */
1637
1638	td = td_next;
1639	td_next = td->obj_next;
1640
1641	/* check if we are pre-computing */
1642
1643	if (precompute) {
1644
1645	/* update remaining length */
1646
1647	temp->len -= average;
1648
1649	continue;
1650	}
1651	/* fill out current TD */
1652
1653	td->qtd_status =
1654	temp->qtd_status \|
1655	htohc32(temp->sc, EHCI_QTD_IOC \|
1656	EHCI_QTD_SET_BYTES(average));
1657
1658	if (average == 0) {
1659
1660	if (temp->auto_data_toggle == 0) {
1661
1662	/* update data toggle, ZLP case */
1663
1664	temp->qtd_status ^=
1665	htohc32(temp->sc, EHCI_QTD_TOGGLE_MASK);
1666	}
1667	td->len = 0;
1668
1669	/* properly reset reserved fields */
1670	td->qtd_buffer[0] = 0;
1671	td->qtd_buffer[1] = 0;
1672	td->qtd_buffer[2] = 0;
1673	td->qtd_buffer[3] = 0;
1674	td->qtd_buffer[4] = 0;
1675	td->qtd_buffer_hi[0] = 0;
1676	td->qtd_buffer_hi[1] = 0;
1677	td->qtd_buffer_hi[2] = 0;
1678	td->qtd_buffer_hi[3] = 0;
1679	td->qtd_buffer_hi[4] = 0;
1680	} else {
1681
1682	uint8_t x;
1683
1684	if (temp->auto_data_toggle == 0) {
1685
1686	/* update data toggle */
1687
1688	if (((average + temp->max_frame_size - 1) /
1689	temp->max_frame_size) & 1) {
1690	temp->qtd_status ^=
1691	htohc32(temp->sc, EHCI_QTD_TOGGLE_MASK);
1692	}
1693	}
1694	td->len = average;
1695
1696	/* update remaining length */
1697
1698	temp->len -= average;
1699
1700	/* fill out buffer pointers */
1701
1702	usbd_get_page(temp->pc, buf_offset, &buf_res);
1703	td->qtd_buffer[0] =
1704	htohc32(temp->sc, buf_res.physaddr);
1705	td->qtd_buffer_hi[0] = 0;
1706
1707	x = 1;
1708
1709	while (average > EHCI_PAGE_SIZE) {
1710	average -= EHCI_PAGE_SIZE;
1711	buf_offset += EHCI_PAGE_SIZE;
1712	usbd_get_page(temp->pc, buf_offset, &buf_res);
1713	td->qtd_buffer[x] =
1714	htohc32(temp->sc,
1715	buf_res.physaddr & (~0xFFF));
1716	td->qtd_buffer_hi[x] = 0;
1717	x++;
1718	}
1719
1720	/*
1721	* NOTE: The "average" variable is never zero after
1722	* exiting the loop above !
1723	*
1724	* NOTE: We have to subtract one from the offset to
1725	* ensure that we are computing the physical address
1726	* of a valid page !
1727	*/
1728	buf_offset += average;
1729	usbd_get_page(temp->pc, buf_offset - 1, &buf_res);
1730	td->qtd_buffer[x] =
1731	htohc32(temp->sc,
1732	buf_res.physaddr & (~0xFFF));
1733	td->qtd_buffer_hi[x] = 0;
1734
1735	/* properly reset reserved fields */
1736	while (++x < EHCI_QTD_NBUFFERS) {
1737	td->qtd_buffer[x] = 0;
1738	td->qtd_buffer_hi[x] = 0;
1739	}
1740	}
1741
1742	if (td_next) {
1743	/* link the current TD with the next one */
1744	td->qtd_next = td_next->qtd_self;
1745	}
1746	td->qtd_altnext = qtd_altnext;
1747	td->alt_next = td_alt_next;
1748
1749	usb_pc_cpu_flush(td->page_cache);
1750	}
1751
1752	if (precompute) {
1753	precompute = 0;
1754
1755	/* setup alt next pointer, if any */
1756	if (temp->last_frame) {
1757	td_alt_next = NULL;
1758	qtd_altnext = terminate;
1759	} else {
1760	/* we use this field internally */
1761	td_alt_next = td_next;
1762	if (temp->setup_alt_next) {
1763	qtd_altnext = td_next->qtd_self;
1764	} else {
1765	qtd_altnext = terminate;
1766	}
1767	}
1768
1769	/* restore */
1770	temp->shortpkt = shortpkt_old;
1771	temp->len = len_old;
1772	goto restart;
1773	}
1774	temp->td = td;
1775	temp->td_next = td_next;
1776	}
1777
1778	static void
1779	ehci_setup_standard_chain(struct usb_xfer xfer, ehci_qh_t *qh_last)
1780	{
1781	struct ehci_std_temp temp;
1782	const struct usb_pipe_methods *methods;
1783	ehci_qh_t *qh;
1784	ehci_qtd_t *td;
1785	uint32_t qh_endp;
1786	uint32_t qh_endphub;
1787	uint32_t x;
1788
1789	DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
1790	xfer->address, UE_GET_ADDR(xfer->endpointno),
1791	xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
1792
1793	temp.average = xfer->max_hc_frame_size;
1794	temp.max_frame_size = xfer->max_frame_size;
1795	temp.sc = EHCI_BUS2SC(xfer->xroot->bus);
1796
1797	/* toggle the DMA set we are using */
1798	xfer->flags_int.curr_dma_set ^= 1;
1799
1800	/* get next DMA set */
1801	td = xfer->td_start[xfer->flags_int.curr_dma_set];
1802
1803	xfer->td_transfer_first = td;
1804	xfer->td_transfer_cache = td;
1805
1806	temp.td = NULL;
1807	temp.td_next = td;
1808	temp.qtd_status = 0;
1809	temp.last_frame = 0;
1810	temp.setup_alt_next = xfer->flags_int.short_frames_ok;
1811
1812	if (xfer->flags_int.control_xfr) {
1813	if (xfer->endpoint->toggle_next) {
1814	/* DATA1 is next */
1815	temp.qtd_status \|=
1816	htohc32(temp.sc, EHCI_QTD_SET_TOGGLE(1));
1817	}
1818	temp.auto_data_toggle = 0;
1819	} else {
1820	temp.auto_data_toggle = 1;
1821	}
1822
1823	if ((xfer->xroot->udev->parent_hs_hub != NULL) \|\|
1824	(xfer->xroot->udev->address != 0)) {
1825	/* max 3 retries */
1826	temp.qtd_status \|=
1827	htohc32(temp.sc, EHCI_QTD_SET_CERR(3));
1828	}
1829	/* check if we should prepend a setup message */
1830
1831	if (xfer->flags_int.control_xfr) {
1832	if (xfer->flags_int.control_hdr) {
1833
1834	xfer->endpoint->toggle_next = 0;
1835
1836	temp.qtd_status &=
1837	htohc32(temp.sc, EHCI_QTD_SET_CERR(3));
1838	temp.qtd_status \|= htohc32(temp.sc,
1839	EHCI_QTD_ACTIVE \|
1840	EHCI_QTD_SET_PID(EHCI_QTD_PID_SETUP) \|
1841	EHCI_QTD_SET_TOGGLE(0));
1842
1843	temp.len = xfer->frlengths[0];
1844	temp.pc = xfer->frbuffers + 0;
1845	temp.shortpkt = temp.len ? 1 : 0;
1846	/* check for last frame */
1847	if (xfer->nframes == 1) {
1848	/* no STATUS stage yet, SETUP is last */
1849	if (xfer->flags_int.control_act) {
1850	temp.last_frame = 1;
1851	temp.setup_alt_next = 0;
1852	}
1853	}
1854	ehci_setup_standard_chain_sub(&temp);
1855	}
1856	x = 1;
1857	} else {
1858	x = 0;
1859	}
1860
1861	while (x != xfer->nframes) {
1862
1863	/* DATA0 / DATA1 message */
1864
1865	temp.len = xfer->frlengths[x];
1866	temp.pc = xfer->frbuffers + x;
1867
1868	x++;
1869
1870	if (x == xfer->nframes) {
1871	if (xfer->flags_int.control_xfr) {
1872	/* no STATUS stage yet, DATA is last */
1873	if (xfer->flags_int.control_act) {
1874	temp.last_frame = 1;
1875	temp.setup_alt_next = 0;
1876	}
1877	} else {
1878	temp.last_frame = 1;
1879	temp.setup_alt_next = 0;
1880	}
1881	}
1882	/* keep previous data toggle and error count */
1883
1884	temp.qtd_status &=
1885	htohc32(temp.sc, EHCI_QTD_SET_CERR(3) \|
1886	EHCI_QTD_SET_TOGGLE(1));
1887
1888	if (temp.len == 0) {
1889
1890	/* make sure that we send an USB packet */
1891
1892	temp.shortpkt = 0;
1893
1894	} else {
1895
1896	/* regular data transfer */
1897
1898	temp.shortpkt = (xfer->flags.force_short_xfer) ? 0 : 1;
1899	}
1900
1901	/* set endpoint direction */
1902
1903	temp.qtd_status \|=
1904	(UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) ?
1905	htohc32(temp.sc, EHCI_QTD_ACTIVE \|
1906	EHCI_QTD_SET_PID(EHCI_QTD_PID_IN)) :
1907	htohc32(temp.sc, EHCI_QTD_ACTIVE \|
1908	EHCI_QTD_SET_PID(EHCI_QTD_PID_OUT));
1909
1910	ehci_setup_standard_chain_sub(&temp);
1911	}
1912
1913	/* check if we should append a status stage */
1914
1915	if (xfer->flags_int.control_xfr &&
1916	!xfer->flags_int.control_act) {
1917
1918	/*
1919	* Send a DATA1 message and invert the current endpoint
1920	* direction.
1921	*/
1922
1923	temp.qtd_status &= htohc32(temp.sc, EHCI_QTD_SET_CERR(3) \|
1924	EHCI_QTD_SET_TOGGLE(1));
1925	temp.qtd_status \|=
1926	(UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) ?
1927	htohc32(temp.sc, EHCI_QTD_ACTIVE \|
1928	EHCI_QTD_SET_PID(EHCI_QTD_PID_IN) \|
1929	EHCI_QTD_SET_TOGGLE(1)) :
1930	htohc32(temp.sc, EHCI_QTD_ACTIVE \|
1931	EHCI_QTD_SET_PID(EHCI_QTD_PID_OUT) \|
1932	EHCI_QTD_SET_TOGGLE(1));
1933
1934	temp.len = 0;
1935	temp.pc = NULL;
1936	temp.shortpkt = 0;
1937	temp.last_frame = 1;
1938	temp.setup_alt_next = 0;
1939
1940	ehci_setup_standard_chain_sub(&temp);
1941	}
1942	td = temp.td;
1943
1944	/* the last TD terminates the transfer: */
1945	td->qtd_next = htohc32(temp.sc, EHCI_LINK_TERMINATE);
1946	td->qtd_altnext = htohc32(temp.sc, EHCI_LINK_TERMINATE);
1947
1948	usb_pc_cpu_flush(td->page_cache);
1949
1950	/* must have at least one frame! */
1951
1952	xfer->td_transfer_last = td;
1953
1954	#ifdef USB_DEBUG
1955	if (ehcidebug > 8) {
1956	DPRINTF("nexttog=%d; data before transfer:\n",
1957	xfer->endpoint->toggle_next);
1958	ehci_dump_sqtds(temp.sc,
1959	xfer->td_transfer_first);
1960	}
1961	#endif
1962
1963	methods = xfer->endpoint->methods;
1964
1965	qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1966
1967	/* the "qh_link" field is filled when the QH is added */
1968
1969	qh_endp =
1970	(EHCI_QH_SET_ADDR(xfer->address) \|
1971	EHCI_QH_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)) \|
1972	EHCI_QH_SET_MPL(xfer->max_packet_size));
1973
1974	if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_HIGH) {
1975	qh_endp \|= EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH);
1976	if (methods != &ehci_device_intr_methods)
1977	qh_endp \|= EHCI_QH_SET_NRL(8);
1978	} else {
1979
1980	if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_FULL) {
1981	qh_endp \|= EHCI_QH_SET_EPS(EHCI_QH_SPEED_FULL);
1982	} else {
1983	qh_endp \|= EHCI_QH_SET_EPS(EHCI_QH_SPEED_LOW);
1984	}
1985
1986	if (methods == &ehci_device_ctrl_methods) {
1987	qh_endp \|= EHCI_QH_CTL;
1988	}
1989	if (methods != &ehci_device_intr_methods) {
1990	/* Only try one time per microframe! */
1991	qh_endp \|= EHCI_QH_SET_NRL(1);
1992	}
1993	}
1994
1995	if (temp.auto_data_toggle == 0) {
1996	/* software computes the data toggle */
1997	qh_endp \|= EHCI_QH_DTC;
1998	}
1999
2000	qh->qh_endp = htohc32(temp.sc, qh_endp);
2001
2002	qh_endphub =
2003	(EHCI_QH_SET_MULT(xfer->max_packet_count & 3) \|
2004	EHCI_QH_SET_CMASK(xfer->endpoint->usb_cmask) \|
2005	EHCI_QH_SET_SMASK(xfer->endpoint->usb_smask) \|
2006	EHCI_QH_SET_HUBA(xfer->xroot->udev->hs_hub_addr) \|
2007	EHCI_QH_SET_PORT(xfer->xroot->udev->hs_port_no));
2008
2009	qh->qh_endphub = htohc32(temp.sc, qh_endphub);
2010	qh->qh_curqtd = 0;
2011
2012	/* fill the overlay qTD */
2013
2014	if (temp.auto_data_toggle && xfer->endpoint->toggle_next) {
2015	/* DATA1 is next */
2016	qh->qh_qtd.qtd_status = htohc32(temp.sc, EHCI_QTD_SET_TOGGLE(1));
2017	} else {
2018	qh->qh_qtd.qtd_status = 0;
2019	}
2020
2021	td = xfer->td_transfer_first;
2022
2023	qh->qh_qtd.qtd_next = td->qtd_self;
2024	qh->qh_qtd.qtd_altnext =
2025	htohc32(temp.sc, EHCI_LINK_TERMINATE);
2026
2027	/* properly reset reserved fields */
2028	qh->qh_qtd.qtd_buffer[0] = 0;
2029	qh->qh_qtd.qtd_buffer[1] = 0;
2030	qh->qh_qtd.qtd_buffer[2] = 0;
2031	qh->qh_qtd.qtd_buffer[3] = 0;
2032	qh->qh_qtd.qtd_buffer[4] = 0;
2033	qh->qh_qtd.qtd_buffer_hi[0] = 0;
2034	qh->qh_qtd.qtd_buffer_hi[1] = 0;
2035	qh->qh_qtd.qtd_buffer_hi[2] = 0;
2036	qh->qh_qtd.qtd_buffer_hi[3] = 0;
2037	qh->qh_qtd.qtd_buffer_hi[4] = 0;
2038
2039	usb_pc_cpu_flush(qh->page_cache);
2040
2041	if (xfer->xroot->udev->flags.self_suspended == 0) {
2042	EHCI_APPEND_QH(qh, *qh_last);
2043	}
2044	}
2045
2046	static void
2047	ehci_root_intr(ehci_softc_t *sc)
2048	{
2049	uint16_t i;
2050	uint16_t m;
2051
2052	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2053
2054	/* clear any old interrupt data */
2055	memset(sc->sc_hub_idata, 0, sizeof(sc->sc_hub_idata));
2056
2057	/* set bits */
2058	m = (sc->sc_noport + 1);
2059	if (m > (8 * sizeof(sc->sc_hub_idata))) {
2060	m = (8 * sizeof(sc->sc_hub_idata));
2061	}
2062	for (i = 1; i < m; i++) {
2063	/* pick out CHANGE bits from the status register */
2064	if (EOREAD4(sc, EHCI_PORTSC(i)) & EHCI_PS_CLEAR) {
2065	sc->sc_hub_idata[i / 8] \|= 1 << (i % 8);
2066	DPRINTF("port %d changed\n", i);
2067	}
2068	}
2069	uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
2070	sizeof(sc->sc_hub_idata));
2071	}
2072
2073	static void
2074	ehci_isoc_fs_done(ehci_softc_t sc, struct usb_xfer xfer)
2075	{
2076	uint32_t nframes = xfer->nframes;
2077	uint32_t status;
2078	uint32_t *plen = xfer->frlengths;
2079	uint16_t len = 0;
2080	ehci_sitd_t *td = xfer->td_transfer_first;
2081	ehci_sitd_t **pp_last = &sc->sc_isoc_fs_p_last[xfer->qh_pos];
2082
2083	DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
2084	xfer, xfer->endpoint);
2085
2086	while (nframes--) {
2087	if (td == NULL) {
2088	panic("%s:%d: out of TD's\n",
2089	__FUNCTION__, __LINE__);
2090	}
2091	if (pp_last >= &sc->sc_isoc_fs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2092	pp_last = &sc->sc_isoc_fs_p_last[0];
2093	}
2094	#ifdef USB_DEBUG
2095	if (ehcidebug > 15) {
2096	DPRINTF("isoc FS-TD\n");
2097	ehci_dump_sitd(sc, td);
2098	}
2099	#endif
2100	usb_pc_cpu_invalidate(td->page_cache);
2101	status = hc32toh(sc, td->sitd_status);
2102
2103	len = EHCI_SITD_GET_LEN(status);
2104
2105	DPRINTFN(2, "status=0x%08x, rem=%u\n", status, len);
2106
2107	if (*plen >= len) {
2108	len = *plen - len;
2109	} else {
2110	len = 0;
2111	}
2112
2113	*plen = len;
2114
2115	/* remove FS-TD from schedule */
2116	EHCI_REMOVE_FS_TD(td, *pp_last);
2117
2118	pp_last++;
2119	plen++;
2120	td = td->obj_next;
2121	}
2122
2123	xfer->aframes = xfer->nframes;
2124	}
2125
2126	static void
2127	ehci_isoc_hs_done(ehci_softc_t sc, struct usb_xfer xfer)
2128	{
2129	uint32_t nframes = xfer->nframes;
2130	uint32_t status;
2131	uint32_t *plen = xfer->frlengths;
2132	uint16_t len = 0;
2133	uint8_t td_no = 0;
2134	ehci_itd_t *td = xfer->td_transfer_first;
2135	ehci_itd_t **pp_last = &sc->sc_isoc_hs_p_last[xfer->qh_pos];
2136
2137	DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
2138	xfer, xfer->endpoint);
2139
2140	while (nframes) {
2141	if (td == NULL) {
2142	panic("%s:%d: out of TD's\n",
2143	__FUNCTION__, __LINE__);
2144	}
2145	if (pp_last >= &sc->sc_isoc_hs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2146	pp_last = &sc->sc_isoc_hs_p_last[0];
2147	}
2148	#ifdef USB_DEBUG
2149	if (ehcidebug > 15) {
2150	DPRINTF("isoc HS-TD\n");
2151	ehci_dump_itd(sc, td);
2152	}
2153	#endif
2154
2155	usb_pc_cpu_invalidate(td->page_cache);
2156	status = hc32toh(sc, td->itd_status[td_no]);
2157
2158	len = EHCI_ITD_GET_LEN(status);
2159
2160	DPRINTFN(2, "status=0x%08x, len=%u\n", status, len);
2161
2162	if (xfer->endpoint->usb_smask & (1 << td_no)) {
2163
2164	if (*plen >= len) {
2165	/*
2166	* The length is valid. NOTE: The
2167	* complete length is written back
2168	* into the status field, and not the
2169	* remainder like with other transfer
2170	* descriptor types.
2171	*/
2172	} else {
2173	/* Invalid length - truncate */
2174	len = 0;
2175	}
2176
2177	*plen = len;
2178	plen++;
2179	nframes--;
2180	}
2181
2182	td_no++;
2183
2184	if ((td_no == 8) \|\| (nframes == 0)) {
2185	/* remove HS-TD from schedule */
2186	EHCI_REMOVE_HS_TD(td, *pp_last);
2187	pp_last++;
2188
2189	td_no = 0;
2190	td = td->obj_next;
2191	}
2192	}
2193	xfer->aframes = xfer->nframes;
2194	}
2195
2196	/* NOTE: "done" can be run two times in a row,
2197	* from close and from interrupt
2198	*/
2199	static void
2200	ehci_device_done(struct usb_xfer *xfer, usb_error_t error)
2201	{
2202	const struct usb_pipe_methods *methods = xfer->endpoint->methods;
2203	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2204
2205	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2206
2207	DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n",
2208	xfer, xfer->endpoint, error);
2209
2210	if ((methods == &ehci_device_bulk_methods) \|\|
2211	(methods == &ehci_device_ctrl_methods)) {
2212	#ifdef USB_DEBUG
2213	if (ehcidebug > 8) {
2214	DPRINTF("nexttog=%d; data after transfer:\n",
2215	xfer->endpoint->toggle_next);
2216	ehci_dump_sqtds(sc,
2217	xfer->td_transfer_first);
2218	}
2219	#endif
2220
2221	EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
2222	sc->sc_async_p_last);
2223	}
2224	if (methods == &ehci_device_intr_methods) {
2225	EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
2226	sc->sc_intr_p_last[xfer->qh_pos]);
2227	}
2228	/*
2229	* Only finish isochronous transfers once which will update
2230	* "xfer->frlengths".
2231	*/
2232	if (xfer->td_transfer_first &&
2233	xfer->td_transfer_last) {
2234	if (methods == &ehci_device_isoc_fs_methods) {
2235	ehci_isoc_fs_done(sc, xfer);
2236	}
2237	if (methods == &ehci_device_isoc_hs_methods) {
2238	ehci_isoc_hs_done(sc, xfer);
2239	}
2240	xfer->td_transfer_first = NULL;
2241	xfer->td_transfer_last = NULL;
2242	}
2243	/* dequeue transfer and start next transfer */
2244	usbd_transfer_done(xfer, error);
2245	}
2246
2247	/------------------------------------------------------------------------
2248	* ehci bulk support
2249	------------------------------------------------------------------------/
2250	static void
2251	ehci_device_bulk_open(struct usb_xfer *xfer)
2252	{
2253	return;
2254	}
2255
2256	static void
2257	ehci_device_bulk_close(struct usb_xfer *xfer)
2258	{
2259	ehci_device_done(xfer, USB_ERR_CANCELLED);
2260	}
2261
2262	static void
2263	ehci_device_bulk_enter(struct usb_xfer *xfer)
2264	{
2265	return;
2266	}
2267
2268	static void
2269	ehci_doorbell_async(struct ehci_softc *sc)
2270	{
2271	uint32_t temp;
2272
2273	/*
2274	* XXX Performance quirk: Some Host Controllers have a too low
2275	* interrupt rate. Issue an IAAD to stimulate the Host
2276	* Controller after queueing the BULK transfer.
2277	*
2278	* XXX Force the host controller to refresh any QH caches.
2279	*/
2280	temp = EOREAD4(sc, EHCI_USBCMD);
2281	if (!(temp & EHCI_CMD_IAAD))
2282	EOWRITE4(sc, EHCI_USBCMD, temp \| EHCI_CMD_IAAD);
2283	}
2284
2285	static void
2286	ehci_device_bulk_start(struct usb_xfer *xfer)
2287	{
2288	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2289
2290	/* setup TD's and QH */
2291	ehci_setup_standard_chain(xfer, &sc->sc_async_p_last);
2292
2293	/* put transfer on interrupt queue */
2294	ehci_transfer_intr_enqueue(xfer);
2295
2296	/*
2297	* XXX Certain nVidia chipsets choke when using the IAAD
2298	* feature too frequently.
2299	*/
2300	if (sc->sc_flags & EHCI_SCFLG_IAADBUG)
2301	return;
2302
2303	ehci_doorbell_async(sc);
2304	}
2305
2306	static const struct usb_pipe_methods ehci_device_bulk_methods =
2307	{
2308	.open = ehci_device_bulk_open,
2309	.close = ehci_device_bulk_close,
2310	.enter = ehci_device_bulk_enter,
2311	.start = ehci_device_bulk_start,
2312	};
2313
2314	/------------------------------------------------------------------------
2315	* ehci control support
2316	------------------------------------------------------------------------/
2317	static void
2318	ehci_device_ctrl_open(struct usb_xfer *xfer)
2319	{
2320	return;
2321	}
2322
2323	static void
2324	ehci_device_ctrl_close(struct usb_xfer *xfer)
2325	{
2326	ehci_device_done(xfer, USB_ERR_CANCELLED);
2327	}
2328
2329	static void
2330	ehci_device_ctrl_enter(struct usb_xfer *xfer)
2331	{
2332	return;
2333	}
2334
2335	static void
2336	ehci_device_ctrl_start(struct usb_xfer *xfer)
2337	{
2338	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2339
2340	/* setup TD's and QH */
2341	ehci_setup_standard_chain(xfer, &sc->sc_async_p_last);
2342
2343	/* put transfer on interrupt queue */
2344	ehci_transfer_intr_enqueue(xfer);
2345	}
2346
2347	static const struct usb_pipe_methods ehci_device_ctrl_methods =
2348	{
2349	.open = ehci_device_ctrl_open,
2350	.close = ehci_device_ctrl_close,
2351	.enter = ehci_device_ctrl_enter,
2352	.start = ehci_device_ctrl_start,
2353	};
2354
2355	/------------------------------------------------------------------------
2356	* ehci interrupt support
2357	------------------------------------------------------------------------/
2358	static void
2359	ehci_device_intr_open(struct usb_xfer *xfer)
2360	{
2361	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2362	uint16_t best;
2363	uint16_t bit;
2364	uint16_t x;
2365
2366	usb_hs_bandwidth_alloc(xfer);
2367
2368	/*
2369	* Find the best QH position corresponding to the given interval:
2370	*/
2371
2372	best = 0;
2373	bit = EHCI_VIRTUAL_FRAMELIST_COUNT / 2;
2374	while (bit) {
2375	if (xfer->interval >= bit) {
2376	x = bit;
2377	best = bit;
2378	while (x & bit) {
2379	if (sc->sc_intr_stat[x] <
2380	sc->sc_intr_stat[best]) {
2381	best = x;
2382	}
2383	x++;
2384	}
2385	break;
2386	}
2387	bit >>= 1;
2388	}
2389
2390	sc->sc_intr_stat[best]++;
2391	xfer->qh_pos = best;
2392
2393	DPRINTFN(3, "best=%d interval=%d\n",
2394	best, xfer->interval);
2395	}
2396
2397	static void
2398	ehci_device_intr_close(struct usb_xfer *xfer)
2399	{
2400	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2401
2402	sc->sc_intr_stat[xfer->qh_pos]--;
2403
2404	ehci_device_done(xfer, USB_ERR_CANCELLED);
2405
2406	/* bandwidth must be freed after device done */
2407	usb_hs_bandwidth_free(xfer);
2408	}
2409
2410	static void
2411	ehci_device_intr_enter(struct usb_xfer *xfer)
2412	{
2413	return;
2414	}
2415
2416	static void
2417	ehci_device_intr_start(struct usb_xfer *xfer)
2418	{
2419	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2420
2421	/* setup TD's and QH */
2422	ehci_setup_standard_chain(xfer, &sc->sc_intr_p_last[xfer->qh_pos]);
2423
2424	/* put transfer on interrupt queue */
2425	ehci_transfer_intr_enqueue(xfer);
2426	}
2427
2428	static const struct usb_pipe_methods ehci_device_intr_methods =
2429	{
2430	.open = ehci_device_intr_open,
2431	.close = ehci_device_intr_close,
2432	.enter = ehci_device_intr_enter,
2433	.start = ehci_device_intr_start,
2434	};
2435
2436	/------------------------------------------------------------------------
2437	* ehci full speed isochronous support
2438	------------------------------------------------------------------------/
2439	static void
2440	ehci_device_isoc_fs_open(struct usb_xfer *xfer)
2441	{
2442	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2443	ehci_sitd_t *td;
2444	uint32_t sitd_portaddr;
2445	uint8_t ds;
2446
2447	sitd_portaddr =
2448	EHCI_SITD_SET_ADDR(xfer->address) \|
2449	EHCI_SITD_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)) \|
2450	EHCI_SITD_SET_HUBA(xfer->xroot->udev->hs_hub_addr) \|
2451	EHCI_SITD_SET_PORT(xfer->xroot->udev->hs_port_no);
2452
2453	if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN)
2454	sitd_portaddr \|= EHCI_SITD_SET_DIR_IN;
2455
2456	sitd_portaddr = htohc32(sc, sitd_portaddr);
2457
2458	/* initialize all TD's */
2459
2460	for (ds = 0; ds != 2; ds++) {
2461
2462	for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2463
2464	td->sitd_portaddr = sitd_portaddr;
2465
2466	/*
2467	* TODO: make some kind of automatic
2468	* SMASK/CMASK selection based on micro-frame
2469	* usage
2470	*
2471	* micro-frame usage (8 microframes per 1ms)
2472	*/
2473	td->sitd_back = htohc32(sc, EHCI_LINK_TERMINATE);
2474
2475	usb_pc_cpu_flush(td->page_cache);
2476	}
2477	}
2478	}
2479
2480	static void
2481	ehci_device_isoc_fs_close(struct usb_xfer *xfer)
2482	{
2483	ehci_device_done(xfer, USB_ERR_CANCELLED);
2484	}
2485
2486	static void
2487	ehci_device_isoc_fs_enter(struct usb_xfer *xfer)
2488	{
2489	struct usb_page_search buf_res;
2490	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2491	ehci_sitd_t *td;
2492	ehci_sitd_t *td_last = NULL;
2493	ehci_sitd_t **pp_last;
2494	uint32_t *plen;
2495	uint32_t buf_offset;
2496	uint32_t nframes;
2497	uint32_t temp;
2498	uint32_t sitd_mask;
2499	uint16_t tlen;
2500	uint8_t sa;
2501	uint8_t sb;
2502
2503	#ifdef USB_DEBUG
2504	uint8_t once = 1;
2505
2506	#endif
2507
2508	DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
2509	xfer, xfer->endpoint->isoc_next, xfer->nframes);
2510
2511	/* get the current frame index */
2512
2513	nframes = EOREAD4(sc, EHCI_FRINDEX) / 8;
2514
2515	/*
2516	* check if the frame index is within the window where the frames
2517	* will be inserted
2518	*/
2519	buf_offset = (nframes - xfer->endpoint->isoc_next) &
2520	(EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2521
2522	if ((xfer->endpoint->is_synced == 0) \|\|
2523	(buf_offset < xfer->nframes)) {
2524	/*
2525	* If there is data underflow or the pipe queue is empty we
2526	* schedule the transfer a few frames ahead of the current
2527	* frame position. Else two isochronous transfers might
2528	* overlap.
2529	*/
2530	xfer->endpoint->isoc_next = (nframes + 3) &
2531	(EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2532	xfer->endpoint->is_synced = 1;
2533	DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2534	}
2535	/*
2536	* compute how many milliseconds the insertion is ahead of the
2537	* current frame position:
2538	*/
2539	buf_offset = (xfer->endpoint->isoc_next - nframes) &
2540	(EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2541
2542	/*
2543	* pre-compute when the isochronous transfer will be finished:
2544	*/
2545	xfer->isoc_time_complete =
2546	usb_isoc_time_expand(&sc->sc_bus, nframes) +
2547	buf_offset + xfer->nframes;
2548
2549	/* get the real number of frames */
2550
2551	nframes = xfer->nframes;
2552
2553	buf_offset = 0;
2554
2555	plen = xfer->frlengths;
2556
2557	/* toggle the DMA set we are using */
2558	xfer->flags_int.curr_dma_set ^= 1;
2559
2560	/* get next DMA set */
2561	td = xfer->td_start[xfer->flags_int.curr_dma_set];
2562	xfer->td_transfer_first = td;
2563
2564	pp_last = &sc->sc_isoc_fs_p_last[xfer->endpoint->isoc_next];
2565
2566	/* store starting position */
2567
2568	xfer->qh_pos = xfer->endpoint->isoc_next;
2569
2570	while (nframes--) {
2571	if (td == NULL) {
2572	panic("%s:%d: out of TD's\n",
2573	__FUNCTION__, __LINE__);
2574	}
2575	if (pp_last >= &sc->sc_isoc_fs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT])
2576	pp_last = &sc->sc_isoc_fs_p_last[0];
2577
2578	/* reuse sitd_portaddr and sitd_back from last transfer */
2579
2580	if (*plen > xfer->max_frame_size) {
2581	#ifdef USB_DEBUG
2582	if (once) {
2583	once = 0;
2584	printf("%s: frame length(%d) exceeds %d "
2585	"bytes (frame truncated)\n",
2586	__FUNCTION__, *plen,
2587	xfer->max_frame_size);
2588	}
2589	#endif
2590	*plen = xfer->max_frame_size;
2591	}
2592
2593	/* allocate a slot */
2594
2595	sa = usbd_fs_isoc_schedule_alloc_slot(xfer,
2596	xfer->isoc_time_complete - nframes - 1);
2597
2598	if (sa == 255) {
2599	/*
2600	* Schedule is FULL, set length to zero:
2601	*/
2602
2603	*plen = 0;
2604	sa = USB_FS_ISOC_UFRAME_MAX - 1;
2605	}
2606	if (*plen) {
2607	/*
2608	* only call "usbd_get_page()" when we have a
2609	* non-zero length
2610	*/
2611	usbd_get_page(xfer->frbuffers, buf_offset, &buf_res);
2612	td->sitd_bp[0] = htohc32(sc, buf_res.physaddr);
2613	buf_offset += *plen;
2614	/*
2615	* NOTE: We need to subtract one from the offset so
2616	* that we are on a valid page!
2617	*/
2618	usbd_get_page(xfer->frbuffers, buf_offset - 1,
2619	&buf_res);
2620	temp = buf_res.physaddr & ~0xFFF;
2621	} else {
2622	td->sitd_bp[0] = 0;
2623	temp = 0;
2624	}
2625
2626	if (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) {
2627	tlen = *plen;
2628	if (tlen <= 188) {
2629	temp \|= 1; /* T-count = 1, TP = ALL */
2630	tlen = 1;
2631	} else {
2632	tlen += 187;
2633	tlen /= 188;
2634	temp \|= tlen; /* T-count = [1..6] */
2635	temp \|= 8; /* TP = Begin */
2636	}
2637
2638	tlen += sa;
2639
2640	if (tlen >= 8) {
2641	sb = 0;
2642	} else {
2643	sb = (1 << tlen);
2644	}
2645
2646	sa = (1 << sa);
2647	sa = (sb - sa) & 0x3F;
2648	sb = 0;
2649	} else {
2650	sb = (-(4 << sa)) & 0xFE;
2651	sa = (1 << sa) & 0x3F;
2652	}
2653
2654	sitd_mask = (EHCI_SITD_SET_SMASK(sa) \|
2655	EHCI_SITD_SET_CMASK(sb));
2656
2657	td->sitd_bp[1] = htohc32(sc, temp);
2658
2659	td->sitd_mask = htohc32(sc, sitd_mask);
2660
2661	if (nframes == 0) {
2662	td->sitd_status = htohc32(sc,
2663	EHCI_SITD_IOC \|
2664	EHCI_SITD_ACTIVE \|
2665	EHCI_SITD_SET_LEN(*plen));
2666	} else {
2667	td->sitd_status = htohc32(sc,
2668	EHCI_SITD_ACTIVE \|
2669	EHCI_SITD_SET_LEN(*plen));
2670	}
2671	usb_pc_cpu_flush(td->page_cache);
2672
2673	#ifdef USB_DEBUG
2674	if (ehcidebug > 15) {
2675	DPRINTF("FS-TD %d\n", nframes);
2676	ehci_dump_sitd(sc, td);
2677	}
2678	#endif
2679	/* insert TD into schedule */
2680	EHCI_APPEND_FS_TD(td, *pp_last);
2681	pp_last++;
2682
2683	plen++;
2684	td_last = td;
2685	td = td->obj_next;
2686	}
2687
2688	xfer->td_transfer_last = td_last;
2689
2690	/* update isoc_next */
2691	xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_fs_p_last[0]) &
2692	(EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2693
2694	/*
2695	* We don't allow cancelling of the SPLIT transaction USB FULL
2696	* speed transfer, because it disturbs the bandwidth
2697	* computation algorithm.
2698	*/
2699	xfer->flags_int.can_cancel_immed = 0;
2700	}
2701
2702	static void
2703	ehci_device_isoc_fs_start(struct usb_xfer *xfer)
2704	{
2705	/*
2706	* We don't allow cancelling of the SPLIT transaction USB FULL
2707	* speed transfer, because it disturbs the bandwidth
2708	* computation algorithm.
2709	*/
2710	xfer->flags_int.can_cancel_immed = 0;
2711
2712	/* set a default timeout */
2713	if (xfer->timeout == 0)
2714	xfer->timeout = 500; /* ms */
2715
2716	/* put transfer on interrupt queue */
2717	ehci_transfer_intr_enqueue(xfer);
2718	}
2719
2720	static const struct usb_pipe_methods ehci_device_isoc_fs_methods =
2721	{
2722	.open = ehci_device_isoc_fs_open,
2723	.close = ehci_device_isoc_fs_close,
2724	.enter = ehci_device_isoc_fs_enter,
2725	.start = ehci_device_isoc_fs_start,
2726	};
2727
2728	/------------------------------------------------------------------------
2729	* ehci high speed isochronous support
2730	------------------------------------------------------------------------/
2731	static void
2732	ehci_device_isoc_hs_open(struct usb_xfer *xfer)
2733	{
2734	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2735	ehci_itd_t *td;
2736	uint32_t temp;
2737	uint8_t ds;
2738
2739	usb_hs_bandwidth_alloc(xfer);
2740
2741	/* initialize all TD's */
2742
2743	for (ds = 0; ds != 2; ds++) {
2744
2745	for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2746
2747	/* set TD inactive */
2748	td->itd_status[0] = 0;
2749	td->itd_status[1] = 0;
2750	td->itd_status[2] = 0;
2751	td->itd_status[3] = 0;
2752	td->itd_status[4] = 0;
2753	td->itd_status[5] = 0;
2754	td->itd_status[6] = 0;
2755	td->itd_status[7] = 0;
2756
2757	/* set endpoint and address */
2758	td->itd_bp[0] = htohc32(sc,
2759	EHCI_ITD_SET_ADDR(xfer->address) \|
2760	EHCI_ITD_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)));
2761
2762	temp =
2763	EHCI_ITD_SET_MPL(xfer->max_packet_size & 0x7FF);
2764
2765	/* set direction */
2766	if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) {
2767	temp \|= EHCI_ITD_SET_DIR_IN;
2768	}
2769	/* set maximum packet size */
2770	td->itd_bp[1] = htohc32(sc, temp);
2771
2772	/* set transfer multiplier */
2773	td->itd_bp[2] = htohc32(sc, xfer->max_packet_count & 3);
2774
2775	usb_pc_cpu_flush(td->page_cache);
2776	}
2777	}
2778	}
2779
2780	static void
2781	ehci_device_isoc_hs_close(struct usb_xfer *xfer)
2782	{
2783	ehci_device_done(xfer, USB_ERR_CANCELLED);
2784
2785	/* bandwidth must be freed after device done */
2786	usb_hs_bandwidth_free(xfer);
2787	}
2788
2789	static void
2790	ehci_device_isoc_hs_enter(struct usb_xfer *xfer)
2791	{
2792	struct usb_page_search buf_res;
2793	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2794	ehci_itd_t *td;
2795	ehci_itd_t *td_last = NULL;
2796	ehci_itd_t **pp_last;
2797	bus_size_t page_addr;
2798	uint32_t *plen;
2799	uint32_t status;
2800	uint32_t buf_offset;
2801	uint32_t nframes;
2802	uint32_t itd_offset[8 + 1];
2803	uint8_t x;
2804	uint8_t td_no;
2805	uint8_t page_no;
2806	uint8_t shift = usbd_xfer_get_fps_shift(xfer);
2807
2808	#ifdef USB_DEBUG
2809	uint8_t once = 1;
2810
2811	#endif
2812
2813	DPRINTFN(6, "xfer=%p next=%d nframes=%d shift=%d\n",
2814	xfer, xfer->endpoint->isoc_next, xfer->nframes, (int)shift);
2815
2816	/* get the current frame index */
2817
2818	nframes = EOREAD4(sc, EHCI_FRINDEX) / 8;
2819
2820	/*
2821	* check if the frame index is within the window where the frames
2822	* will be inserted
2823	*/
2824	buf_offset = (nframes - xfer->endpoint->isoc_next) &
2825	(EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2826
2827	if ((xfer->endpoint->is_synced == 0) \|\|
2828	(buf_offset < (((xfer->nframes << shift) + 7) / 8))) {
2829	/*
2830	* If there is data underflow or the pipe queue is empty we
2831	* schedule the transfer a few frames ahead of the current
2832	* frame position. Else two isochronous transfers might
2833	* overlap.
2834	*/
2835	xfer->endpoint->isoc_next = (nframes + 3) &
2836	(EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2837	xfer->endpoint->is_synced = 1;
2838	DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2839	}
2840	/*
2841	* compute how many milliseconds the insertion is ahead of the
2842	* current frame position:
2843	*/
2844	buf_offset = (xfer->endpoint->isoc_next - nframes) &
2845	(EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2846
2847	/*
2848	* pre-compute when the isochronous transfer will be finished:
2849	*/
2850	xfer->isoc_time_complete =
2851	usb_isoc_time_expand(&sc->sc_bus, nframes) + buf_offset +
2852	(((xfer->nframes << shift) + 7) / 8);
2853
2854	/* get the real number of frames */
2855
2856	nframes = xfer->nframes;
2857
2858	buf_offset = 0;
2859	td_no = 0;
2860
2861	plen = xfer->frlengths;
2862
2863	/* toggle the DMA set we are using */
2864	xfer->flags_int.curr_dma_set ^= 1;
2865
2866	/* get next DMA set */
2867	td = xfer->td_start[xfer->flags_int.curr_dma_set];
2868	xfer->td_transfer_first = td;
2869
2870	pp_last = &sc->sc_isoc_hs_p_last[xfer->endpoint->isoc_next];
2871
2872	/* store starting position */
2873
2874	xfer->qh_pos = xfer->endpoint->isoc_next;
2875
2876	while (nframes) {
2877	if (td == NULL) {
2878	panic("%s:%d: out of TD's\n",
2879	__FUNCTION__, __LINE__);
2880	}
2881	if (pp_last >= &sc->sc_isoc_hs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2882	pp_last = &sc->sc_isoc_hs_p_last[0];
2883	}
2884	/* range check */
2885	if (*plen > xfer->max_frame_size) {
2886	#ifdef USB_DEBUG
2887	if (once) {
2888	once = 0;
2889	printf("%s: frame length(%d) exceeds %d bytes "
2890	"(frame truncated)\n",
2891	__FUNCTION__, *plen, xfer->max_frame_size);
2892	}
2893	#endif
2894	*plen = xfer->max_frame_size;
2895	}
2896
2897	if (xfer->endpoint->usb_smask & (1 << td_no)) {
2898	status = (EHCI_ITD_SET_LEN(*plen) \|
2899	EHCI_ITD_ACTIVE \|
2900	EHCI_ITD_SET_PG(0));
2901	td->itd_status[td_no] = htohc32(sc, status);
2902	itd_offset[td_no] = buf_offset;
2903	buf_offset += *plen;
2904	plen++;
2905	nframes --;
2906	} else {
2907	td->itd_status[td_no] = 0; /* not active */
2908	itd_offset[td_no] = buf_offset;
2909	}
2910
2911	td_no++;
2912
2913	if ((td_no == 8) \|\| (nframes == 0)) {
2914
2915	/* the rest of the transfers are not active, if any */
2916	for (x = td_no; x != 8; x++) {
2917	td->itd_status[x] = 0; /* not active */
2918	}
2919
2920	/* check if there is any data to be transferred */
2921	if (itd_offset[0] != buf_offset) {
2922	page_no = 0;
2923	itd_offset[td_no] = buf_offset;
2924
2925	/* get first page offset */
2926	usbd_get_page(xfer->frbuffers, itd_offset[0], &buf_res);
2927	/* get page address */
2928	page_addr = buf_res.physaddr & ~0xFFF;
2929	/* update page address */
2930	td->itd_bp[0] &= htohc32(sc, 0xFFF);
2931	td->itd_bp[0] \|= htohc32(sc, page_addr);
2932
2933	for (x = 0; x != td_no; x++) {
2934	/* set page number and page offset */
2935	status = (EHCI_ITD_SET_PG(page_no) \|
2936	(buf_res.physaddr & 0xFFF));
2937	td->itd_status[x] \|= htohc32(sc, status);
2938
2939	/* get next page offset */
2940	if (itd_offset[x + 1] == buf_offset) {
2941	/*
2942	* We subtract one so that
2943	* we don't go off the last
2944	* page!
2945	*/
2946	usbd_get_page(xfer->frbuffers, buf_offset - 1, &buf_res);
2947	} else {
2948	usbd_get_page(xfer->frbuffers, itd_offset[x + 1], &buf_res);
2949	}
2950
2951	/* check if we need a new page */
2952	if ((buf_res.physaddr ^ page_addr) & ~0xFFF) {
2953	/* new page needed */
2954	page_addr = buf_res.physaddr & ~0xFFF;
2955	if (page_no == 6) {
2956	panic("%s: too many pages\n", __FUNCTION__);
2957	}
2958	page_no++;
2959	/* update page address */
2960	td->itd_bp[page_no] &= htohc32(sc, 0xFFF);
2961	td->itd_bp[page_no] \|= htohc32(sc, page_addr);
2962	}
2963	}
2964	}
2965	/* set IOC bit if we are complete */
2966	if (nframes == 0) {
2967	td->itd_status[td_no - 1] \|= htohc32(sc, EHCI_ITD_IOC);
2968	}
2969	usb_pc_cpu_flush(td->page_cache);
2970	#ifdef USB_DEBUG
2971	if (ehcidebug > 15) {
2972	DPRINTF("HS-TD %d\n", nframes);
2973	ehci_dump_itd(sc, td);
2974	}
2975	#endif
2976	/* insert TD into schedule */
2977	EHCI_APPEND_HS_TD(td, *pp_last);
2978	pp_last++;
2979
2980	td_no = 0;
2981	td_last = td;
2982	td = td->obj_next;
2983	}
2984	}
2985
2986	xfer->td_transfer_last = td_last;
2987
2988	/* update isoc_next */
2989	xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_hs_p_last[0]) &
2990	(EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2991	}
2992
2993	static void
2994	ehci_device_isoc_hs_start(struct usb_xfer *xfer)
2995	{
2996	/* put transfer on interrupt queue */
2997	ehci_transfer_intr_enqueue(xfer);
2998	}
2999
3000	static const struct usb_pipe_methods ehci_device_isoc_hs_methods =
3001	{
3002	.open = ehci_device_isoc_hs_open,
3003	.close = ehci_device_isoc_hs_close,
3004	.enter = ehci_device_isoc_hs_enter,
3005	.start = ehci_device_isoc_hs_start,
3006	};
3007
3008	/------------------------------------------------------------------------
3009	* ehci root control support
3010	------------------------------------------------------------------------
3011	* Simulate a hardware hub by handling all the necessary requests.
3012	------------------------------------------------------------------------/
3013
3014	static const
3015	struct usb_device_descriptor ehci_devd =
3016	{
3017	sizeof(struct usb_device_descriptor),
3018	UDESC_DEVICE, /* type */
3019	{0x00, 0x02}, /* USB version */
3020	UDCLASS_HUB, /* class */
3021	UDSUBCLASS_HUB, /* subclass */
3022	UDPROTO_HSHUBSTT, /* protocol */
3023	64, /* max packet */
3024	{0}, {0}, {0x00, 0x01}, /* device id */
3025	1, 2, 0, /* string indicies */
3026	1 /* # of configurations */
3027	};
3028
3029	static const
3030	struct usb_device_qualifier ehci_odevd =
3031	{
3032	sizeof(struct usb_device_qualifier),
3033	UDESC_DEVICE_QUALIFIER, /* type */
3034	{0x00, 0x02}, /* USB version */
3035	UDCLASS_HUB, /* class */
3036	UDSUBCLASS_HUB, /* subclass */
3037	UDPROTO_FSHUB, /* protocol */
3038	0, /* max packet */
3039	0, /* # of configurations */
3040	0
3041	};
3042
3043	static const struct ehci_config_desc ehci_confd = {
3044	.confd = {
3045	.bLength = sizeof(struct usb_config_descriptor),
3046	.bDescriptorType = UDESC_CONFIG,
3047	.wTotalLength[0] = sizeof(ehci_confd),
3048	.bNumInterface = 1,
3049	.bConfigurationValue = 1,
3050	.iConfiguration = 0,
3051	.bmAttributes = UC_SELF_POWERED,
3052	.bMaxPower = 0 /* max power */
3053	},
3054	.ifcd = {
3055	.bLength = sizeof(struct usb_interface_descriptor),
3056	.bDescriptorType = UDESC_INTERFACE,
3057	.bNumEndpoints = 1,
3058	.bInterfaceClass = UICLASS_HUB,
3059	.bInterfaceSubClass = UISUBCLASS_HUB,
3060	.bInterfaceProtocol = 0,
3061	},
3062	.endpd = {
3063	.bLength = sizeof(struct usb_endpoint_descriptor),
3064	.bDescriptorType = UDESC_ENDPOINT,
3065	.bEndpointAddress = UE_DIR_IN \| EHCI_INTR_ENDPT,
3066	.bmAttributes = UE_INTERRUPT,
3067	.wMaxPacketSize[0] = 8, /* max packet (63 ports) */
3068	.bInterval = 255,
3069	},
3070	};
3071
3072	static const
3073	struct usb_hub_descriptor ehci_hubd =
3074	{
3075	.bDescLength = 0, /* dynamic length */
3076	.bDescriptorType = UDESC_HUB,
3077	};
3078
3079	static void
3080	ehci_disown(ehci_softc_t *sc, uint16_t index, uint8_t lowspeed)
3081	{
3082	uint32_t port;
3083	uint32_t v;
3084
3085	DPRINTF("index=%d lowspeed=%d\n", index, lowspeed);
3086
3087	port = EHCI_PORTSC(index);
3088	v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3089	EOWRITE4(sc, port, v \| EHCI_PS_PO);
3090	}
3091
3092	static usb_error_t
3093	ehci_roothub_exec(struct usb_device *udev,
3094	struct usb_device_request req, const void pptr, uint16_t plength)
3095	{
3096	ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3097	const char *str_ptr;
3098	const void *ptr;
3099	uint32_t port;
3100	uint32_t v;
3101	uint16_t len;
3102	uint16_t i;
3103	uint16_t value;
3104	uint16_t index;
3105	usb_error_t err;
3106
3107	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3108
3109	/* buffer reset */
3110	ptr = (const void *)&sc->sc_hub_desc;
3111	len = 0;
3112	err = 0;
3113
3114	value = UGETW(req->wValue);
3115	index = UGETW(req->wIndex);
3116
3117	DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x "
3118	"wValue=0x%04x wIndex=0x%04x\n",
3119	req->bmRequestType, req->bRequest,
3120	UGETW(req->wLength), value, index);
3121
3122	#define C(x,y) ((x) \| ((y) << 8))
3123	switch (C(req->bRequest, req->bmRequestType)) {
3124	case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
3125	case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
3126	case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
3127	/*
3128	* DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
3129	* for the integrated root hub.
3130	*/
3131	break;
3132	case C(UR_GET_CONFIG, UT_READ_DEVICE):
3133	len = 1;
3134	sc->sc_hub_desc.temp[0] = sc->sc_conf;
3135	break;
3136	case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
3137	switch (value >> 8) {
3138	case UDESC_DEVICE:
3139	if ((value & 0xff) != 0) {
3140	err = USB_ERR_IOERROR;
3141	goto done;
3142	}
3143	len = sizeof(ehci_devd);
3144	ptr = (const void *)&ehci_devd;
3145	break;
3146	/*
3147	* We can't really operate at another speed,
3148	* but the specification says we need this
3149	* descriptor:
3150	*/
3151	case UDESC_DEVICE_QUALIFIER:
3152	if ((value & 0xff) != 0) {
3153	err = USB_ERR_IOERROR;
3154	goto done;
3155	}
3156	len = sizeof(ehci_odevd);
3157	ptr = (const void *)&ehci_odevd;
3158	break;
3159
3160	case UDESC_CONFIG:
3161	if ((value & 0xff) != 0) {
3162	err = USB_ERR_IOERROR;
3163	goto done;
3164	}
3165	len = sizeof(ehci_confd);
3166	ptr = (const void *)&ehci_confd;
3167	break;
3168
3169	case UDESC_STRING:
3170	switch (value & 0xff) {
3171	case 0: /* Language table */
3172	str_ptr = "\001";
3173	break;
3174
3175	case 1: /* Vendor */
3176	str_ptr = sc->sc_vendor;
3177	break;
3178
3179	case 2: /* Product */
3180	str_ptr = "EHCI root HUB";
3181	break;
3182
3183	default:
3184	str_ptr = "";
3185	break;
3186	}
3187
3188	len = usb_make_str_desc(
3189	sc->sc_hub_desc.temp,
3190	sizeof(sc->sc_hub_desc.temp),
3191	str_ptr);
3192	break;
3193	default:
3194	err = USB_ERR_IOERROR;
3195	goto done;
3196	}
3197	break;
3198	case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
3199	len = 1;
3200	sc->sc_hub_desc.temp[0] = 0;
3201	break;
3202	case C(UR_GET_STATUS, UT_READ_DEVICE):
3203	len = 2;
3204	USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED);
3205	break;
3206	case C(UR_GET_STATUS, UT_READ_INTERFACE):
3207	case C(UR_GET_STATUS, UT_READ_ENDPOINT):
3208	len = 2;
3209	USETW(sc->sc_hub_desc.stat.wStatus, 0);
3210	break;
3211	case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
3212	if (value >= EHCI_MAX_DEVICES) {
3213	err = USB_ERR_IOERROR;
3214	goto done;
3215	}
3216	sc->sc_addr = value;
3217	break;
3218	case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
3219	if ((value != 0) && (value != 1)) {
3220	err = USB_ERR_IOERROR;
3221	goto done;
3222	}
3223	sc->sc_conf = value;
3224	break;
3225	case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
3226	break;
3227	case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
3228	case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
3229	case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
3230	err = USB_ERR_IOERROR;
3231	goto done;
3232	case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
3233	break;
3234	case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
3235	break;
3236	/* Hub requests */
3237	case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
3238	break;
3239	case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
3240	DPRINTFN(9, "UR_CLEAR_PORT_FEATURE\n");
3241
3242	if ((index < 1) \|\|
3243	(index > sc->sc_noport)) {
3244	err = USB_ERR_IOERROR;
3245	goto done;
3246	}
3247	port = EHCI_PORTSC(index);
3248	v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3249	switch (value) {
3250	case UHF_PORT_ENABLE:
3251	EOWRITE4(sc, port, v & ~EHCI_PS_PE);
3252	break;
3253	case UHF_PORT_SUSPEND:
3254	if ((v & EHCI_PS_SUSP) && (!(v & EHCI_PS_FPR))) {
3255
3256	/*
3257	* waking up a High Speed device is rather
3258	* complicated if
3259	*/
3260	EOWRITE4(sc, port, v \| EHCI_PS_FPR);
3261	}
3262	/* wait 20ms for resume sequence to complete */
3263	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50);
3264
3265	EOWRITE4(sc, port, v & ~(EHCI_PS_SUSP \|
3266	EHCI_PS_FPR \| (3 << 10) /* High Speed */ ));
3267
3268	/* 4ms settle time */
3269	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
3270	break;
3271	case UHF_PORT_POWER:
3272	EOWRITE4(sc, port, v & ~EHCI_PS_PP);
3273	break;
3274	case UHF_PORT_TEST:
3275	DPRINTFN(3, "clear port test "
3276	"%d\n", index);
3277	break;
3278	case UHF_PORT_INDICATOR:
3279	DPRINTFN(3, "clear port ind "
3280	"%d\n", index);
3281	EOWRITE4(sc, port, v & ~EHCI_PS_PIC);
3282	break;
3283	case UHF_C_PORT_CONNECTION:
3284	EOWRITE4(sc, port, v \| EHCI_PS_CSC);
3285	break;
3286	case UHF_C_PORT_ENABLE:
3287	EOWRITE4(sc, port, v \| EHCI_PS_PEC);
3288	break;
3289	case UHF_C_PORT_SUSPEND:
3290	EOWRITE4(sc, port, v \| EHCI_PS_SUSP);
3291	break;
3292	case UHF_C_PORT_OVER_CURRENT:
3293	EOWRITE4(sc, port, v \| EHCI_PS_OCC);
3294	break;
3295	case UHF_C_PORT_RESET:
3296	sc->sc_isreset = 0;
3297	break;
3298	default:
3299	err = USB_ERR_IOERROR;
3300	goto done;
3301	}
3302	break;
3303	case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
3304	if ((value & 0xff) != 0) {
3305	err = USB_ERR_IOERROR;
3306	goto done;
3307	}
3308	v = EREAD4(sc, EHCI_HCSPARAMS);
3309
3310	sc->sc_hub_desc.hubd = ehci_hubd;
3311	sc->sc_hub_desc.hubd.bNbrPorts = sc->sc_noport;
3312
3313	if (EHCI_HCS_PPC(v))
3314	i = UHD_PWR_INDIVIDUAL;
3315	else
3316	i = UHD_PWR_NO_SWITCH;
3317
3318	if (EHCI_HCS_P_INDICATOR(v))
3319	i \|= UHD_PORT_IND;
3320
3321	USETW(sc->sc_hub_desc.hubd.wHubCharacteristics, i);
3322	/* XXX can't find out? */
3323	sc->sc_hub_desc.hubd.bPwrOn2PwrGood = 200;
3324	/* XXX don't know if ports are removable or not */
3325	sc->sc_hub_desc.hubd.bDescLength =
3326	8 + ((sc->sc_noport + 7) / 8);
3327	len = sc->sc_hub_desc.hubd.bDescLength;
3328	break;
3329	case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
3330	len = 16;
3331	memset(sc->sc_hub_desc.temp, 0, 16);
3332	break;
3333	case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
3334	DPRINTFN(9, "get port status i=%d\n",
3335	index);
3336	if ((index < 1) \|\|
3337	(index > sc->sc_noport)) {
3338	err = USB_ERR_IOERROR;
3339	goto done;
3340	}
3341	v = EOREAD4(sc, EHCI_PORTSC(index));
3342	DPRINTFN(9, "port status=0x%04x\n", v);
3343	if (sc->sc_flags & (EHCI_SCFLG_FORCESPEED \| EHCI_SCFLG_TT)) {
3344	if ((v & 0xc000000) == 0x8000000)
3345	i = UPS_HIGH_SPEED;
3346	else if ((v & 0xc000000) == 0x4000000)
3347	i = UPS_LOW_SPEED;
3348	else
3349	i = 0;
3350	} else {
3351	i = UPS_HIGH_SPEED;
3352	}
3353	if (v & EHCI_PS_CS)
3354	i \|= UPS_CURRENT_CONNECT_STATUS;
3355	if (v & EHCI_PS_PE)
3356	i \|= UPS_PORT_ENABLED;
3357	if ((v & EHCI_PS_SUSP) && !(v & EHCI_PS_FPR))
3358	i \|= UPS_SUSPEND;
3359	if (v & EHCI_PS_OCA)
3360	i \|= UPS_OVERCURRENT_INDICATOR;
3361	if (v & EHCI_PS_PR)
3362	i \|= UPS_RESET;
3363	if (v & EHCI_PS_PP)
3364	i \|= UPS_PORT_POWER;
3365	USETW(sc->sc_hub_desc.ps.wPortStatus, i);
3366	i = 0;
3367	if (v & EHCI_PS_CSC)
3368	i \|= UPS_C_CONNECT_STATUS;
3369	if (v & EHCI_PS_PEC)
3370	i \|= UPS_C_PORT_ENABLED;
3371	if (v & EHCI_PS_OCC)
3372	i \|= UPS_C_OVERCURRENT_INDICATOR;
3373	if (v & EHCI_PS_FPR)
3374	i \|= UPS_C_SUSPEND;
3375	if (sc->sc_isreset)
3376	i \|= UPS_C_PORT_RESET;
3377	USETW(sc->sc_hub_desc.ps.wPortChange, i);
3378	len = sizeof(sc->sc_hub_desc.ps);
3379	break;
3380	case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
3381	err = USB_ERR_IOERROR;
3382	goto done;
3383	case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
3384	break;
3385	case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
3386	if ((index < 1) \|\|
3387	(index > sc->sc_noport)) {
3388	err = USB_ERR_IOERROR;
3389	goto done;
3390	}
3391	port = EHCI_PORTSC(index);
3392	v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3393	switch (value) {
3394	case UHF_PORT_ENABLE:
3395	EOWRITE4(sc, port, v \| EHCI_PS_PE);
3396	break;
3397	case UHF_PORT_SUSPEND:
3398	EOWRITE4(sc, port, v \| EHCI_PS_SUSP);
3399	break;
3400	case UHF_PORT_RESET:
3401	DPRINTFN(6, "reset port %d\n", index);
3402	#ifdef USB_DEBUG
3403	if (ehcinohighspeed) {
3404	/*
3405	* Connect USB device to companion
3406	* controller.
3407	*/
3408	ehci_disown(sc, index, 1);
3409	break;
3410	}
3411	#endif
3412	if (EHCI_PS_IS_LOWSPEED(v) &&
3413	(sc->sc_flags & EHCI_SCFLG_TT) == 0) {
3414	/* Low speed device, give up ownership. */
3415	ehci_disown(sc, index, 1);
3416	break;
3417	}
3418	/* Start reset sequence. */
3419	v &= ~(EHCI_PS_PE \| EHCI_PS_PR);
3420	EOWRITE4(sc, port, v \| EHCI_PS_PR);
3421
3422	/* Wait for reset to complete. */
3423	usb_pause_mtx(&sc->sc_bus.bus_mtx,
3424	USB_MS_TO_TICKS(usb_port_root_reset_delay));
3425
3426	/* Terminate reset sequence. */
3427	if (!(sc->sc_flags & EHCI_SCFLG_NORESTERM))
3428	EOWRITE4(sc, port, v);
3429
3430	/* Wait for HC to complete reset. */
3431	usb_pause_mtx(&sc->sc_bus.bus_mtx,
3432	USB_MS_TO_TICKS(EHCI_PORT_RESET_COMPLETE));
3433
3434	v = EOREAD4(sc, port);
3435	DPRINTF("ehci after reset, status=0x%08x\n", v);
3436	if (v & EHCI_PS_PR) {
3437	device_printf(sc->sc_bus.bdev,
3438	"port reset timeout\n");
3439	err = USB_ERR_TIMEOUT;
3440	goto done;
3441	}
3442	if (!(v & EHCI_PS_PE) &&
3443	(sc->sc_flags & EHCI_SCFLG_TT) == 0) {
3444	/* Not a high speed device, give up ownership.*/
3445	ehci_disown(sc, index, 0);
3446	break;
3447	}
3448	sc->sc_isreset = 1;
3449	DPRINTF("ehci port %d reset, status = 0x%08x\n",
3450	index, v);
3451	break;
3452
3453	case UHF_PORT_POWER:
3454	DPRINTFN(3, "set port power %d\n", index);
3455	EOWRITE4(sc, port, v \| EHCI_PS_PP);
3456	break;
3457
3458	case UHF_PORT_TEST:
3459	DPRINTFN(3, "set port test %d\n", index);
3460	break;
3461
3462	case UHF_PORT_INDICATOR:
3463	DPRINTFN(3, "set port ind %d\n", index);
3464	EOWRITE4(sc, port, v \| EHCI_PS_PIC);
3465	break;
3466
3467	default:
3468	err = USB_ERR_IOERROR;
3469	goto done;
3470	}
3471	break;
3472	case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
3473	case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
3474	case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
3475	case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
3476	break;
3477	default:
3478	err = USB_ERR_IOERROR;
3479	goto done;
3480	}
3481	done:
3482	*plength = len;
3483	*pptr = ptr;
3484	return (err);
3485	}
3486
3487	static void
3488	ehci_xfer_setup(struct usb_setup_params *parm)
3489	{
3490	struct usb_page_search page_info;
3491	struct usb_page_cache *pc;
3492	ehci_softc_t *sc;
3493	struct usb_xfer *xfer;
3494	void *last_obj;
3495	uint32_t nqtd;
3496	uint32_t nqh;
3497	uint32_t nsitd;
3498	uint32_t nitd;
3499	uint32_t n;
3500
3501	sc = EHCI_BUS2SC(parm->udev->bus);
3502	xfer = parm->curr_xfer;
3503
3504	nqtd = 0;
3505	nqh = 0;
3506	nsitd = 0;
3507	nitd = 0;
3508
3509	/*
3510	* compute maximum number of some structures
3511	*/
3512	if (parm->methods == &ehci_device_ctrl_methods) {
3513
3514	/*
3515	* The proof for the "nqtd" formula is illustrated like
3516	* this:
3517	*
3518	* +------------------------------------+
3519	* \| \|
3520	* \| \|remainder -> \|
3521	* \| +-----+---+ \|
3522	* \| \| xxx \| x \| frm 0 \|
3523	* \| +-----+---++ \|
3524	* \| \| xxx \| xx \| frm 1 \|
3525	* \| +-----+----+ \|
3526	* \| ... \|
3527	* +------------------------------------+
3528	*
3529	* "xxx" means a completely full USB transfer descriptor
3530	*
3531	* "x" and "xx" means a short USB packet
3532	*
3533	* For the remainder of an USB transfer modulo
3534	* "max_data_length" we need two USB transfer descriptors.
3535	* One to transfer the remaining data and one to finalise
3536	* with a zero length packet in case the "force_short_xfer"
3537	* flag is set. We only need two USB transfer descriptors in
3538	* the case where the transfer length of the first one is a
3539	* factor of "max_frame_size". The rest of the needed USB
3540	* transfer descriptors is given by the buffer size divided
3541	* by the maximum data payload.
3542	*/
3543	parm->hc_max_packet_size = 0x400;
3544	parm->hc_max_packet_count = 1;
3545	parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3546	xfer->flags_int.bdma_enable = 1;
3547
3548	usbd_transfer_setup_sub(parm);
3549
3550	nqh = 1;
3551	nqtd = ((2 * xfer->nframes) + 1 /* STATUS */
3552	+ (xfer->max_data_length / xfer->max_hc_frame_size));
3553
3554	} else if (parm->methods == &ehci_device_bulk_methods) {
3555
3556	parm->hc_max_packet_size = 0x400;
3557	parm->hc_max_packet_count = 1;
3558	parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3559	xfer->flags_int.bdma_enable = 1;
3560
3561	usbd_transfer_setup_sub(parm);
3562
3563	nqh = 1;
3564	nqtd = ((2 * xfer->nframes)
3565	+ (xfer->max_data_length / xfer->max_hc_frame_size));
3566
3567	} else if (parm->methods == &ehci_device_intr_methods) {
3568
3569	if (parm->speed == USB_SPEED_HIGH) {
3570	parm->hc_max_packet_size = 0x400;
3571	parm->hc_max_packet_count = 3;
3572	} else if (parm->speed == USB_SPEED_FULL) {
3573	parm->hc_max_packet_size = USB_FS_BYTES_PER_HS_UFRAME;
3574	parm->hc_max_packet_count = 1;
3575	} else {
3576	parm->hc_max_packet_size = USB_FS_BYTES_PER_HS_UFRAME / 8;
3577	parm->hc_max_packet_count = 1;
3578	}
3579
3580	parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3581	xfer->flags_int.bdma_enable = 1;
3582
3583	usbd_transfer_setup_sub(parm);
3584
3585	nqh = 1;
3586	nqtd = ((2 * xfer->nframes)
3587	+ (xfer->max_data_length / xfer->max_hc_frame_size));
3588
3589	} else if (parm->methods == &ehci_device_isoc_fs_methods) {
3590
3591	parm->hc_max_packet_size = 0x3FF;
3592	parm->hc_max_packet_count = 1;
3593	parm->hc_max_frame_size = 0x3FF;
3594	xfer->flags_int.bdma_enable = 1;
3595
3596	usbd_transfer_setup_sub(parm);
3597
3598	nsitd = xfer->nframes;
3599
3600	} else if (parm->methods == &ehci_device_isoc_hs_methods) {
3601
3602	parm->hc_max_packet_size = 0x400;
3603	parm->hc_max_packet_count = 3;
3604	parm->hc_max_frame_size = 0xC00;
3605	xfer->flags_int.bdma_enable = 1;
3606
3607	usbd_transfer_setup_sub(parm);
3608
3609	nitd = ((xfer->nframes + 7) / 8) <<
3610	usbd_xfer_get_fps_shift(xfer);
3611
3612	} else {
3613
3614	parm->hc_max_packet_size = 0x400;
3615	parm->hc_max_packet_count = 1;
3616	parm->hc_max_frame_size = 0x400;
3617
3618	usbd_transfer_setup_sub(parm);
3619	}
3620
3621	alloc_dma_set:
3622
3623	if (parm->err) {
3624	return;
3625	}
3626	/*
3627	* Allocate queue heads and transfer descriptors
3628	*/
3629	last_obj = NULL;
3630
3631	if (usbd_transfer_setup_sub_malloc(
3632	parm, &pc, sizeof(ehci_itd_t),
3633	EHCI_ITD_ALIGN, nitd)) {
3634	parm->err = USB_ERR_NOMEM;
3635	return;
3636	}
3637	if (parm->buf) {
3638	for (n = 0; n != nitd; n++) {
3639	ehci_itd_t *td;
3640
3641	usbd_get_page(pc + n, 0, &page_info);
3642
3643	td = page_info.buffer;
3644
3645	/* init TD */
3646	td->itd_self = htohc32(sc, page_info.physaddr \| EHCI_LINK_ITD);
3647	td->obj_next = last_obj;
3648	td->page_cache = pc + n;
3649
3650	last_obj = td;
3651
3652	usb_pc_cpu_flush(pc + n);
3653	}
3654	}
3655	if (usbd_transfer_setup_sub_malloc(
3656	parm, &pc, sizeof(ehci_sitd_t),
3657	EHCI_SITD_ALIGN, nsitd)) {
3658	parm->err = USB_ERR_NOMEM;
3659	return;
3660	}
3661	if (parm->buf) {
3662	for (n = 0; n != nsitd; n++) {
3663	ehci_sitd_t *td;
3664
3665	usbd_get_page(pc + n, 0, &page_info);
3666
3667	td = page_info.buffer;
3668
3669	/* init TD */
3670	td->sitd_self = htohc32(sc, page_info.physaddr \| EHCI_LINK_SITD);
3671	td->obj_next = last_obj;
3672	td->page_cache = pc + n;
3673
3674	last_obj = td;
3675
3676	usb_pc_cpu_flush(pc + n);
3677	}
3678	}
3679	if (usbd_transfer_setup_sub_malloc(
3680	parm, &pc, sizeof(ehci_qtd_t),
3681	EHCI_QTD_ALIGN, nqtd)) {
3682	parm->err = USB_ERR_NOMEM;
3683	return;
3684	}
3685	if (parm->buf) {
3686	for (n = 0; n != nqtd; n++) {
3687	ehci_qtd_t *qtd;
3688
3689	usbd_get_page(pc + n, 0, &page_info);
3690
3691	qtd = page_info.buffer;
3692
3693	/* init TD */
3694	qtd->qtd_self = htohc32(sc, page_info.physaddr);
3695	qtd->obj_next = last_obj;
3696	qtd->page_cache = pc + n;
3697
3698	last_obj = qtd;
3699
3700	usb_pc_cpu_flush(pc + n);
3701	}
3702	}
3703	xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj;
3704
3705	last_obj = NULL;
3706
3707	if (usbd_transfer_setup_sub_malloc(
3708	parm, &pc, sizeof(ehci_qh_t),
3709	EHCI_QH_ALIGN, nqh)) {
3710	parm->err = USB_ERR_NOMEM;
3711	return;
3712	}
3713	if (parm->buf) {
3714	for (n = 0; n != nqh; n++) {
3715	ehci_qh_t *qh;
3716
3717	usbd_get_page(pc + n, 0, &page_info);
3718
3719	qh = page_info.buffer;
3720
3721	/* init QH */
3722	qh->qh_self = htohc32(sc, page_info.physaddr \| EHCI_LINK_QH);
3723	qh->obj_next = last_obj;
3724	qh->page_cache = pc + n;
3725
3726	last_obj = qh;
3727
3728	usb_pc_cpu_flush(pc + n);
3729	}
3730	}
3731	xfer->qh_start[xfer->flags_int.curr_dma_set] = last_obj;
3732
3733	if (!xfer->flags_int.curr_dma_set) {
3734	xfer->flags_int.curr_dma_set = 1;
3735	goto alloc_dma_set;
3736	}
3737	}
3738
3739	static void
3740	ehci_xfer_unsetup(struct usb_xfer *xfer)
3741	{
3742	return;
3743	}
3744
3745	static void
3746	ehci_ep_init(struct usb_device udev, struct usb_endpoint_descriptor edesc,
3747	struct usb_endpoint *ep)
3748	{
3749	ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3750
3751	DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d)\n",
3752	ep, udev->address,
3753	edesc->bEndpointAddress, udev->flags.usb_mode,
3754	sc->sc_addr);
3755
3756	if (udev->device_index != sc->sc_addr) {
3757
3758	if ((udev->speed != USB_SPEED_HIGH) &&
3759	((udev->hs_hub_addr == 0) \|\|
3760	(udev->hs_port_no == 0) \|\|
3761	(udev->parent_hs_hub == NULL) \|\|
3762	(udev->parent_hs_hub->hub == NULL))) {
3763	/* We need a transaction translator */
3764	goto done;
3765	}
3766	switch (edesc->bmAttributes & UE_XFERTYPE) {
3767	case UE_CONTROL:
3768	ep->methods = &ehci_device_ctrl_methods;
3769	break;
3770	case UE_INTERRUPT:
3771	ep->methods = &ehci_device_intr_methods;
3772	break;
3773	case UE_ISOCHRONOUS:
3774	if (udev->speed == USB_SPEED_HIGH) {
3775	ep->methods = &ehci_device_isoc_hs_methods;
3776	} else if (udev->speed == USB_SPEED_FULL) {
3777	ep->methods = &ehci_device_isoc_fs_methods;
3778	}
3779	break;
3780	case UE_BULK:
3781	ep->methods = &ehci_device_bulk_methods;
3782	break;
3783	default:
3784	/* do nothing */
3785	break;
3786	}
3787	}
3788	done:
3789	return;
3790	}
3791
3792	static void
3793	ehci_get_dma_delay(struct usb_device udev, uint32_t pus)
3794	{
3795	/*
3796	* Wait until the hardware has finished any possible use of
3797	* the transfer descriptor(s) and QH
3798	*/
3799	pus = (1125); / microseconds */
3800	}
3801
3802	static void
3803	ehci_device_resume(struct usb_device *udev)
3804	{
3805	ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3806	struct usb_xfer *xfer;
3807	const struct usb_pipe_methods *methods;
3808
3809	DPRINTF("\n");
3810
3811	USB_BUS_LOCK(udev->bus);
3812
3813	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3814
3815	if (xfer->xroot->udev == udev) {
3816
3817	methods = xfer->endpoint->methods;
3818
3819	if ((methods == &ehci_device_bulk_methods) \|\|
3820	(methods == &ehci_device_ctrl_methods)) {
3821	EHCI_APPEND_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3822	sc->sc_async_p_last);
3823	}
3824	if (methods == &ehci_device_intr_methods) {
3825	EHCI_APPEND_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3826	sc->sc_intr_p_last[xfer->qh_pos]);
3827	}
3828	}
3829	}
3830
3831	USB_BUS_UNLOCK(udev->bus);
3832
3833	return;
3834	}
3835
3836	static void
3837	ehci_device_suspend(struct usb_device *udev)
3838	{
3839	ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3840	struct usb_xfer *xfer;
3841	const struct usb_pipe_methods *methods;
3842
3843	DPRINTF("\n");
3844
3845	USB_BUS_LOCK(udev->bus);
3846
3847	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3848
3849	if (xfer->xroot->udev == udev) {
3850
3851	methods = xfer->endpoint->methods;
3852
3853	if ((methods == &ehci_device_bulk_methods) \|\|
3854	(methods == &ehci_device_ctrl_methods)) {
3855	EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3856	sc->sc_async_p_last);
3857	}
3858	if (methods == &ehci_device_intr_methods) {
3859	EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3860	sc->sc_intr_p_last[xfer->qh_pos]);
3861	}
3862	}
3863	}
3864
3865	USB_BUS_UNLOCK(udev->bus);
3866	}
3867
3868	static void
3869	ehci_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
3870	{
3871	struct ehci_softc *sc = EHCI_BUS2SC(bus);
3872
3873	switch (state) {
3874	case USB_HW_POWER_SUSPEND:
3875	case USB_HW_POWER_SHUTDOWN:
3876	ehci_suspend(sc);
3877	break;
3878	case USB_HW_POWER_RESUME:
3879	ehci_resume(sc);
3880	break;
3881	default:
3882	break;
3883	}
3884	}
3885
3886	static void
3887	ehci_set_hw_power(struct usb_bus *bus)
3888	{
3889	ehci_softc_t *sc = EHCI_BUS2SC(bus);
3890	uint32_t temp;
3891	uint32_t flags;
3892
3893	DPRINTF("\n");
3894
3895	USB_BUS_LOCK(bus);
3896
3897	flags = bus->hw_power_state;
3898
3899	temp = EOREAD4(sc, EHCI_USBCMD);
3900
3901	temp &= ~(EHCI_CMD_ASE \| EHCI_CMD_PSE);
3902
3903	if (flags & (USB_HW_POWER_CONTROL \|
3904	USB_HW_POWER_BULK)) {
3905	DPRINTF("Async is active\n");
3906	temp \|= EHCI_CMD_ASE;
3907	}
3908	if (flags & (USB_HW_POWER_INTERRUPT \|
3909	USB_HW_POWER_ISOC)) {
3910	DPRINTF("Periodic is active\n");
3911	temp \|= EHCI_CMD_PSE;
3912	}
3913	EOWRITE4(sc, EHCI_USBCMD, temp);
3914
3915	USB_BUS_UNLOCK(bus);
3916
3917	return;
3918	}
3919
3920	static void
3921	ehci_start_dma_delay_second(struct usb_xfer *xfer)
3922	{
3923	struct ehci_softc *sc = EHCI_BUS2SC(xfer->xroot->bus);
3924
3925	DPRINTF("\n");
3926
3927	/* trigger doorbell */
3928	ehci_doorbell_async(sc);
3929
3930	/* give the doorbell 4ms */
3931	usbd_transfer_timeout_ms(xfer,
3932	(void ()(void ))&usb_dma_delay_done_cb, 4);
3933	}
3934
3935	/*
3936	* Ring the doorbell twice before freeing any DMA descriptors. Some host
3937	* controllers apparently cache the QH descriptors and need a message
3938	* that the cache needs to be discarded.
3939	*/
3940	static void
3941	ehci_start_dma_delay(struct usb_xfer *xfer)
3942	{
3943	struct ehci_softc *sc = EHCI_BUS2SC(xfer->xroot->bus);
3944
3945	DPRINTF("\n");
3946
3947	/* trigger doorbell */
3948	ehci_doorbell_async(sc);
3949
3950	/* give the doorbell 4ms */
3951	usbd_transfer_timeout_ms(xfer,
3952	(void ()(void ))&ehci_start_dma_delay_second, 4);
3953	}
3954
3955	static const struct usb_bus_methods ehci_bus_methods =
3956	{
3957	.endpoint_init = ehci_ep_init,
3958	.xfer_setup = ehci_xfer_setup,
3959	.xfer_unsetup = ehci_xfer_unsetup,
3960	.get_dma_delay = ehci_get_dma_delay,
3961	.device_resume = ehci_device_resume,
3962	.device_suspend = ehci_device_suspend,
3963	.set_hw_power = ehci_set_hw_power,
3964	.set_hw_power_sleep = ehci_set_hw_power_sleep,
3965	.roothub_exec = ehci_roothub_exec,
3966	.xfer_poll = ehci_do_poll,
3967	.start_dma_delay = ehci_start_dma_delay,
3968	};

Note: See TracBrowser for help on using the repository browser.

Download in other formats: