Context Navigation

usb_transfer.c @ 8e65e1b

55-freebsd-126-freebsd-12freebsd-9.3

Last change on this file since 8e65e1b was 8e65e1b, checked in by Sebastian Huber <sebastian.huber@…>, on 08/23/16 at 13:51:45

usb: Update to FreeBSD trunk 2016-08-23

FreeBSD trunk, 2016-08-23, 9fe7c416e6abb28b1398fd3e5687099846800cfd.

Property mode set to 100644

File size: 88.9 KB

Line
1	#include <machine/rtems-bsd-kernel-space.h>
2
3	/* $FreeBSD$ */
4	/*-
5	* Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
6	*
7	* Redistribution and use in source and binary forms, with or without
8	* modification, are permitted provided that the following conditions
9	* are met:
10	* 1. Redistributions of source code must retain the above copyright
11	* notice, this list of conditions and the following disclaimer.
12	* 2. Redistributions in binary form must reproduce the above copyright
13	* notice, this list of conditions and the following disclaimer in the
14	* documentation and/or other materials provided with the distribution.
15	*
16	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26	* SUCH DAMAGE.
27	*/
28
29	#ifdef USB_GLOBAL_INCLUDE_FILE
30	#include USB_GLOBAL_INCLUDE_FILE
31	#else
32	#include <sys/stdint.h>
33	#include <sys/stddef.h>
34	#include <rtems/bsd/sys/param.h>
35	#include <sys/queue.h>
36	#include <sys/types.h>
37	#include <sys/systm.h>
38	#include <sys/kernel.h>
39	#include <sys/bus.h>
40	#include <sys/module.h>
41	#include <rtems/bsd/sys/lock.h>
42	#include <sys/mutex.h>
43	#include <sys/condvar.h>
44	#include <sys/sysctl.h>
45	#include <sys/sx.h>
46	#include <rtems/bsd/sys/unistd.h>
47	#include <sys/callout.h>
48	#include <sys/malloc.h>
49	#include <sys/priv.h>
50	#include <sys/proc.h>
51
52	#include <dev/usb/usb.h>
53	#include <dev/usb/usbdi.h>
54	#include <dev/usb/usbdi_util.h>
55
56	#define USB_DEBUG_VAR usb_debug
57
58	#include <dev/usb/usb_core.h>
59	#include <dev/usb/usb_busdma.h>
60	#include <dev/usb/usb_process.h>
61	#include <dev/usb/usb_transfer.h>
62	#include <dev/usb/usb_device.h>
63	#include <dev/usb/usb_debug.h>
64	#include <dev/usb/usb_util.h>
65
66	#include <dev/usb/usb_controller.h>
67	#include <dev/usb/usb_bus.h>
68	#include <dev/usb/usb_pf.h>
69	#endif /* USB_GLOBAL_INCLUDE_FILE */
70	#ifdef __rtems__
71	#include <machine/rtems-bsd-cache.h>
72	#endif /* __rtems__ */
73
74	struct usb_std_packet_size {
75	struct {
76	uint16_t min; /* inclusive */
77	uint16_t max; /* inclusive */
78	} range;
79
80	uint16_t fixed[4];
81	};
82
83	static usb_callback_t usb_request_callback;
84
85	static const struct usb_config usb_control_ep_cfg[USB_CTRL_XFER_MAX] = {
86
87	/* This transfer is used for generic control endpoint transfers */
88
89	[0] = {
90	.type = UE_CONTROL,
91	.endpoint = 0x00, /* Control endpoint */
92	.direction = UE_DIR_ANY,
93	.bufsize = USB_EP0_BUFSIZE, /* bytes */
94	.flags = {.proxy_buffer = 1,},
95	.callback = &usb_request_callback,
96	.usb_mode = USB_MODE_DUAL, /* both modes */
97	},
98
99	/* This transfer is used for generic clear stall only */
100
101	[1] = {
102	.type = UE_CONTROL,
103	.endpoint = 0x00, /* Control pipe */
104	.direction = UE_DIR_ANY,
105	.bufsize = sizeof(struct usb_device_request),
106	.callback = &usb_do_clear_stall_callback,
107	.timeout = 1000, /* 1 second */
108	.interval = 50, /* 50ms */
109	.usb_mode = USB_MODE_HOST,
110	},
111	};
112
113	/* function prototypes */
114
115	static void usbd_update_max_frame_size(struct usb_xfer *);
116	static void usbd_transfer_unsetup_sub(struct usb_xfer_root *, uint8_t);
117	static void usbd_control_transfer_init(struct usb_xfer *);
118	static int usbd_setup_ctrl_transfer(struct usb_xfer *);
119	static void usb_callback_proc(struct usb_proc_msg *);
120	static void usbd_callback_ss_done_defer(struct usb_xfer *);
121	static void usbd_callback_wrapper(struct usb_xfer_queue *);
122	static void usbd_transfer_start_cb(void *);
123	static uint8_t usbd_callback_wrapper_sub(struct usb_xfer *);
124	static void usbd_get_std_packet_size(struct usb_std_packet_size *ptr,
125	uint8_t type, enum usb_dev_speed speed);
126
127	/------------------------------------------------------------------------
128	* usb_request_callback
129	------------------------------------------------------------------------/
130	static void
131	usb_request_callback(struct usb_xfer *xfer, usb_error_t error)
132	{
133	if (xfer->flags_int.usb_mode == USB_MODE_DEVICE)
134	usb_handle_request_callback(xfer, error);
135	else
136	usbd_do_request_callback(xfer, error);
137	}
138
139	/------------------------------------------------------------------------
140	* usbd_update_max_frame_size
141	*
142	* This function updates the maximum frame size, hence high speed USB
143	* can transfer multiple consecutive packets.
144	------------------------------------------------------------------------/
145	static void
146	usbd_update_max_frame_size(struct usb_xfer *xfer)
147	{
148	/* compute maximum frame size */
149	/* this computation should not overflow 16-bit */
150	/* max = 15 * 1024 */
151
152	xfer->max_frame_size = xfer->max_packet_size * xfer->max_packet_count;
153	}
154
155	/------------------------------------------------------------------------
156	* usbd_get_dma_delay
157	*
158	* The following function is called when we need to
159	* synchronize with DMA hardware.
160	*
161	* Returns:
162	* 0: no DMA delay required
163	* Else: milliseconds of DMA delay
164	------------------------------------------------------------------------/
165	usb_timeout_t
166	usbd_get_dma_delay(struct usb_device *udev)
167	{
168	const struct usb_bus_methods *mtod;
169	uint32_t temp;
170
171	mtod = udev->bus->methods;
172	temp = 0;
173
174	if (mtod->get_dma_delay) {
175	(mtod->get_dma_delay) (udev, &temp);
176	/*
177	* Round up and convert to milliseconds. Note that we use
178	* 1024 milliseconds per second. to save a division.
179	*/
180	temp += 0x3FF;
181	temp /= 0x400;
182	}
183	return (temp);
184	}
185
186	/------------------------------------------------------------------------
187	* usbd_transfer_setup_sub_malloc
188	*
189	* This function will allocate one or more DMA'able memory chunks
190	* according to "size", "align" and "count" arguments. "ppc" is
191	* pointed to a linear array of USB page caches afterwards.
192	*
193	* If the "align" argument is equal to "1" a non-contiguous allocation
194	* can happen. Else if the "align" argument is greater than "1", the
195	* allocation will always be contiguous in memory.
196	*
197	* Returns:
198	* 0: Success
199	* Else: Failure
200	------------------------------------------------------------------------/
201	#if USB_HAVE_BUSDMA
202	uint8_t
203	usbd_transfer_setup_sub_malloc(struct usb_setup_params *parm,
204	struct usb_page_cache **ppc, usb_size_t size, usb_size_t align,
205	usb_size_t count)
206	{
207	struct usb_page_cache *pc;
208	struct usb_page *pg;
209	void *buf;
210	usb_size_t n_dma_pc;
211	usb_size_t n_dma_pg;
212	usb_size_t n_obj;
213	usb_size_t x;
214	usb_size_t y;
215	usb_size_t r;
216	usb_size_t z;
217
218	USB_ASSERT(align > 0, ("Invalid alignment, 0x%08x\n",
219	align));
220	USB_ASSERT(size > 0, ("Invalid size = 0\n"));
221
222	if (count == 0) {
223	return (0); /* nothing to allocate */
224	}
225	#ifdef __rtems__
226	#ifdef CPU_DATA_CACHE_ALIGNMENT
227	if (align < CPU_DATA_CACHE_ALIGNMENT) {
228	align = CPU_DATA_CACHE_ALIGNMENT;
229	}
230	#endif /* CPU_DATA_CACHE_ALIGNMENT */
231	#endif /* __rtems__ */
232	/*
233	* Make sure that the size is aligned properly.
234	*/
235	size = -((-size) & (-align));
236
237	/*
238	* Try multi-allocation chunks to reduce the number of DMA
239	* allocations, hence DMA allocations are slow.
240	*/
241	if (align == 1) {
242	/* special case - non-cached multi page DMA memory */
243	n_dma_pc = count;
244	n_dma_pg = (2 + (size / USB_PAGE_SIZE));
245	n_obj = 1;
246	} else if (size >= USB_PAGE_SIZE) {
247	n_dma_pc = count;
248	n_dma_pg = 1;
249	n_obj = 1;
250	} else {
251	/* compute number of objects per page */
252	#ifdef USB_DMA_SINGLE_ALLOC
253	n_obj = 1;
254	#else
255	n_obj = (USB_PAGE_SIZE / size);
256	#endif
257	/*
258	* Compute number of DMA chunks, rounded up
259	* to nearest one:
260	*/
261	n_dma_pc = howmany(count, n_obj);
262	n_dma_pg = 1;
263	}
264
265	/*
266	* DMA memory is allocated once, but mapped twice. That's why
267	* there is one list for auto-free and another list for
268	* non-auto-free which only holds the mapping and not the
269	* allocation.
270	*/
271	if (parm->buf == NULL) {
272	/* reserve memory (auto-free) */
273	parm->dma_page_ptr += n_dma_pc * n_dma_pg;
274	parm->dma_page_cache_ptr += n_dma_pc;
275
276	/* reserve memory (no-auto-free) */
277	parm->dma_page_ptr += count * n_dma_pg;
278	parm->xfer_page_cache_ptr += count;
279	return (0);
280	}
281	for (x = 0; x != n_dma_pc; x++) {
282	/* need to initialize the page cache */
283	parm->dma_page_cache_ptr[x].tag_parent =
284	&parm->curr_xfer->xroot->dma_parent_tag;
285	}
286	for (x = 0; x != count; x++) {
287	/* need to initialize the page cache */
288	parm->xfer_page_cache_ptr[x].tag_parent =
289	&parm->curr_xfer->xroot->dma_parent_tag;
290	}
291
292	if (ppc != NULL) {
293	if (n_obj != 1)
294	*ppc = parm->xfer_page_cache_ptr;
295	else
296	*ppc = parm->dma_page_cache_ptr;
297	}
298	r = count; /* set remainder count */
299	z = n_obj * size; /* set allocation size */
300	pc = parm->xfer_page_cache_ptr;
301	pg = parm->dma_page_ptr;
302
303	if (n_obj == 1) {
304	/*
305	* Avoid mapping memory twice if only a single object
306	* should be allocated per page cache:
307	*/
308	for (x = 0; x != n_dma_pc; x++) {
309	if (usb_pc_alloc_mem(parm->dma_page_cache_ptr,
310	pg, z, align)) {
311	return (1); /* failure */
312	}
313	/* Make room for one DMA page cache and "n_dma_pg" pages */
314	parm->dma_page_cache_ptr++;
315	pg += n_dma_pg;
316	}
317	} else {
318	for (x = 0; x != n_dma_pc; x++) {
319
320	if (r < n_obj) {
321	/* compute last remainder */
322	z = r * size;
323	n_obj = r;
324	}
325	if (usb_pc_alloc_mem(parm->dma_page_cache_ptr,
326	pg, z, align)) {
327	return (1); /* failure */
328	}
329	/* Set beginning of current buffer */
330	buf = parm->dma_page_cache_ptr->buffer;
331	/* Make room for one DMA page cache and "n_dma_pg" pages */
332	parm->dma_page_cache_ptr++;
333	pg += n_dma_pg;
334
335	for (y = 0; (y != n_obj); y++, r--, pc++, pg += n_dma_pg) {
336
337	/* Load sub-chunk into DMA */
338	if (usb_pc_dmamap_create(pc, size)) {
339	return (1); /* failure */
340	}
341	pc->buffer = USB_ADD_BYTES(buf, y * size);
342	pc->page_start = pg;
343
344	mtx_lock(pc->tag_parent->mtx);
345	if (usb_pc_load_mem(pc, size, 1 /* synchronous */ )) {
346	mtx_unlock(pc->tag_parent->mtx);
347	return (1); /* failure */
348	}
349	mtx_unlock(pc->tag_parent->mtx);
350	}
351	}
352	}
353
354	parm->xfer_page_cache_ptr = pc;
355	parm->dma_page_ptr = pg;
356	return (0);
357	}
358	#endif
359
360	/------------------------------------------------------------------------
361	* usbd_transfer_setup_sub - transfer setup subroutine
362	*
363	* This function must be called from the "xfer_setup" callback of the
364	* USB Host or Device controller driver when setting up an USB
365	* transfer. This function will setup correct packet sizes, buffer
366	* sizes, flags and more, that are stored in the "usb_xfer"
367	* structure.
368	------------------------------------------------------------------------/
369	void
370	usbd_transfer_setup_sub(struct usb_setup_params *parm)
371	{
372	enum {
373	REQ_SIZE = 8,
374	MIN_PKT = 8,
375	};
376	struct usb_xfer *xfer = parm->curr_xfer;
377	const struct usb_config *setup = parm->curr_setup;
378	struct usb_endpoint_ss_comp_descriptor *ecomp;
379	struct usb_endpoint_descriptor *edesc;
380	struct usb_std_packet_size std_size;
381	usb_frcount_t n_frlengths;
382	usb_frcount_t n_frbuffers;
383	usb_frcount_t x;
384	uint16_t maxp_old;
385	uint8_t type;
386	uint8_t zmps;
387
388	/*
389	* Sanity check. The following parameters must be initialized before
390	* calling this function.
391	*/
392	if ((parm->hc_max_packet_size == 0) \|\|
393	(parm->hc_max_packet_count == 0) \|\|
394	(parm->hc_max_frame_size == 0)) {
395	parm->err = USB_ERR_INVAL;
396	goto done;
397	}
398	edesc = xfer->endpoint->edesc;
399	ecomp = xfer->endpoint->ecomp;
400
401	type = (edesc->bmAttributes & UE_XFERTYPE);
402
403	xfer->flags = setup->flags;
404	xfer->nframes = setup->frames;
405	xfer->timeout = setup->timeout;
406	xfer->callback = setup->callback;
407	xfer->interval = setup->interval;
408	xfer->endpointno = edesc->bEndpointAddress;
409	xfer->max_packet_size = UGETW(edesc->wMaxPacketSize);
410	xfer->max_packet_count = 1;
411	/* make a shadow copy: */
412	xfer->flags_int.usb_mode = parm->udev->flags.usb_mode;
413
414	parm->bufsize = setup->bufsize;
415
416	switch (parm->speed) {
417	case USB_SPEED_HIGH:
418	switch (type) {
419	case UE_ISOCHRONOUS:
420	case UE_INTERRUPT:
421	xfer->max_packet_count +=
422	(xfer->max_packet_size >> 11) & 3;
423
424	/* check for invalid max packet count */
425	if (xfer->max_packet_count > 3)
426	xfer->max_packet_count = 3;
427	break;
428	default:
429	break;
430	}
431	xfer->max_packet_size &= 0x7FF;
432	break;
433	case USB_SPEED_SUPER:
434	xfer->max_packet_count += (xfer->max_packet_size >> 11) & 3;
435
436	if (ecomp != NULL)
437	xfer->max_packet_count += ecomp->bMaxBurst;
438
439	if ((xfer->max_packet_count == 0) \|\|
440	(xfer->max_packet_count > 16))
441	xfer->max_packet_count = 16;
442
443	switch (type) {
444	case UE_CONTROL:
445	xfer->max_packet_count = 1;
446	break;
447	case UE_ISOCHRONOUS:
448	if (ecomp != NULL) {
449	uint8_t mult;
450
451	mult = UE_GET_SS_ISO_MULT(
452	ecomp->bmAttributes) + 1;
453	if (mult > 3)
454	mult = 3;
455
456	xfer->max_packet_count *= mult;
457	}
458	break;
459	default:
460	break;
461	}
462	xfer->max_packet_size &= 0x7FF;
463	break;
464	default:
465	break;
466	}
467	/* range check "max_packet_count" */
468
469	if (xfer->max_packet_count > parm->hc_max_packet_count) {
470	xfer->max_packet_count = parm->hc_max_packet_count;
471	}
472
473	/* store max packet size value before filtering */
474
475	maxp_old = xfer->max_packet_size;
476
477	/* filter "wMaxPacketSize" according to HC capabilities */
478
479	if ((xfer->max_packet_size > parm->hc_max_packet_size) \|\|
480	(xfer->max_packet_size == 0)) {
481	xfer->max_packet_size = parm->hc_max_packet_size;
482	}
483	/* filter "wMaxPacketSize" according to standard sizes */
484
485	usbd_get_std_packet_size(&std_size, type, parm->speed);
486
487	if (std_size.range.min \|\| std_size.range.max) {
488
489	if (xfer->max_packet_size < std_size.range.min) {
490	xfer->max_packet_size = std_size.range.min;
491	}
492	if (xfer->max_packet_size > std_size.range.max) {
493	xfer->max_packet_size = std_size.range.max;
494	}
495	} else {
496
497	if (xfer->max_packet_size >= std_size.fixed[3]) {
498	xfer->max_packet_size = std_size.fixed[3];
499	} else if (xfer->max_packet_size >= std_size.fixed[2]) {
500	xfer->max_packet_size = std_size.fixed[2];
501	} else if (xfer->max_packet_size >= std_size.fixed[1]) {
502	xfer->max_packet_size = std_size.fixed[1];
503	} else {
504	/* only one possibility left */
505	xfer->max_packet_size = std_size.fixed[0];
506	}
507	}
508
509	/*
510	* Check if the max packet size was outside its allowed range
511	* and clamped to a valid value:
512	*/
513	if (maxp_old != xfer->max_packet_size)
514	xfer->flags_int.maxp_was_clamped = 1;
515
516	/* compute "max_frame_size" */
517
518	usbd_update_max_frame_size(xfer);
519
520	/* check interrupt interval and transfer pre-delay */
521
522	if (type == UE_ISOCHRONOUS) {
523
524	uint16_t frame_limit;
525
526	xfer->interval = 0; /* not used, must be zero */
527	xfer->flags_int.isochronous_xfr = 1; /* set flag */
528
529	if (xfer->timeout == 0) {
530	/*
531	* set a default timeout in
532	* case something goes wrong!
533	*/
534	xfer->timeout = 1000 / 4;
535	}
536	switch (parm->speed) {
537	case USB_SPEED_LOW:
538	case USB_SPEED_FULL:
539	frame_limit = USB_MAX_FS_ISOC_FRAMES_PER_XFER;
540	xfer->fps_shift = 0;
541	break;
542	default:
543	frame_limit = USB_MAX_HS_ISOC_FRAMES_PER_XFER;
544	xfer->fps_shift = edesc->bInterval;
545	if (xfer->fps_shift > 0)
546	xfer->fps_shift--;
547	if (xfer->fps_shift > 3)
548	xfer->fps_shift = 3;
549	if (xfer->flags.pre_scale_frames != 0)
550	xfer->nframes <<= (3 - xfer->fps_shift);
551	break;
552	}
553
554	if (xfer->nframes > frame_limit) {
555	/*
556	* this is not going to work
557	* cross hardware
558	*/
559	parm->err = USB_ERR_INVAL;
560	goto done;
561	}
562	if (xfer->nframes == 0) {
563	/*
564	* this is not a valid value
565	*/
566	parm->err = USB_ERR_ZERO_NFRAMES;
567	goto done;
568	}
569	} else {
570
571	/*
572	* If a value is specified use that else check the
573	* endpoint descriptor!
574	*/
575	if (type == UE_INTERRUPT) {
576
577	uint32_t temp;
578
579	if (xfer->interval == 0) {
580
581	xfer->interval = edesc->bInterval;
582
583	switch (parm->speed) {
584	case USB_SPEED_LOW:
585	case USB_SPEED_FULL:
586	break;
587	default:
588	/* 125us -> 1ms */
589	if (xfer->interval < 4)
590	xfer->interval = 1;
591	else if (xfer->interval > 16)
592	xfer->interval = (1 << (16 - 4));
593	else
594	xfer->interval =
595	(1 << (xfer->interval - 4));
596	break;
597	}
598	}
599
600	if (xfer->interval == 0) {
601	/*
602	* One millisecond is the smallest
603	* interval we support:
604	*/
605	xfer->interval = 1;
606	}
607
608	xfer->fps_shift = 0;
609	temp = 1;
610
611	while ((temp != 0) && (temp < xfer->interval)) {
612	xfer->fps_shift++;
613	temp *= 2;
614	}
615
616	switch (parm->speed) {
617	case USB_SPEED_LOW:
618	case USB_SPEED_FULL:
619	break;
620	default:
621	xfer->fps_shift += 3;
622	break;
623	}
624	}
625	}
626
627	/*
628	* NOTE: we do not allow "max_packet_size" or "max_frame_size"
629	* to be equal to zero when setting up USB transfers, hence
630	* this leads to a lot of extra code in the USB kernel.
631	*/
632
633	if ((xfer->max_frame_size == 0) \|\|
634	(xfer->max_packet_size == 0)) {
635
636	zmps = 1;
637
638	if ((parm->bufsize <= MIN_PKT) &&
639	(type != UE_CONTROL) &&
640	(type != UE_BULK)) {
641
642	/* workaround */
643	xfer->max_packet_size = MIN_PKT;
644	xfer->max_packet_count = 1;
645	parm->bufsize = 0; /* automatic setup length */
646	usbd_update_max_frame_size(xfer);
647
648	} else {
649	parm->err = USB_ERR_ZERO_MAXP;
650	goto done;
651	}
652
653	} else {
654	zmps = 0;
655	}
656
657	/*
658	* check if we should setup a default
659	* length:
660	*/
661
662	if (parm->bufsize == 0) {
663
664	parm->bufsize = xfer->max_frame_size;
665
666	if (type == UE_ISOCHRONOUS) {
667	parm->bufsize *= xfer->nframes;
668	}
669	}
670	/*
671	* check if we are about to setup a proxy
672	* type of buffer:
673	*/
674
675	if (xfer->flags.proxy_buffer) {
676
677	/* round bufsize up */
678
679	parm->bufsize += (xfer->max_frame_size - 1);
680
681	if (parm->bufsize < xfer->max_frame_size) {
682	/* length wrapped around */
683	parm->err = USB_ERR_INVAL;
684	goto done;
685	}
686	/* subtract remainder */
687
688	parm->bufsize -= (parm->bufsize % xfer->max_frame_size);
689
690	/* add length of USB device request structure, if any */
691
692	if (type == UE_CONTROL) {
693	parm->bufsize += REQ_SIZE; /* SETUP message */
694	}
695	}
696	xfer->max_data_length = parm->bufsize;
697
698	/* Setup "n_frlengths" and "n_frbuffers" */
699
700	if (type == UE_ISOCHRONOUS) {
701	n_frlengths = xfer->nframes;
702	n_frbuffers = 1;
703	} else {
704
705	if (type == UE_CONTROL) {
706	xfer->flags_int.control_xfr = 1;
707	if (xfer->nframes == 0) {
708	if (parm->bufsize <= REQ_SIZE) {
709	/*
710	* there will never be any data
711	* stage
712	*/
713	xfer->nframes = 1;
714	} else {
715	xfer->nframes = 2;
716	}
717	}
718	} else {
719	if (xfer->nframes == 0) {
720	xfer->nframes = 1;
721	}
722	}
723
724	n_frlengths = xfer->nframes;
725	n_frbuffers = xfer->nframes;
726	}
727
728	/*
729	* check if we have room for the
730	* USB device request structure:
731	*/
732
733	if (type == UE_CONTROL) {
734
735	if (xfer->max_data_length < REQ_SIZE) {
736	/* length wrapped around or too small bufsize */
737	parm->err = USB_ERR_INVAL;
738	goto done;
739	}
740	xfer->max_data_length -= REQ_SIZE;
741	}
742	/*
743	* Setup "frlengths" and shadow "frlengths" for keeping the
744	* initial frame lengths when a USB transfer is complete. This
745	* information is useful when computing isochronous offsets.
746	*/
747	xfer->frlengths = parm->xfer_length_ptr;
748	parm->xfer_length_ptr += 2 * n_frlengths;
749
750	/* setup "frbuffers" */
751	xfer->frbuffers = parm->xfer_page_cache_ptr;
752	parm->xfer_page_cache_ptr += n_frbuffers;
753
754	/* initialize max frame count */
755	xfer->max_frame_count = xfer->nframes;
756
757	/*
758	* check if we need to setup
759	* a local buffer:
760	*/
761
762	if (!xfer->flags.ext_buffer) {
763	#if USB_HAVE_BUSDMA
764	struct usb_page_search page_info;
765	struct usb_page_cache *pc;
766
767	if (usbd_transfer_setup_sub_malloc(parm,
768	&pc, parm->bufsize, 1, 1)) {
769	parm->err = USB_ERR_NOMEM;
770	} else if (parm->buf != NULL) {
771
772	usbd_get_page(pc, 0, &page_info);
773
774	xfer->local_buffer = page_info.buffer;
775
776	usbd_xfer_set_frame_offset(xfer, 0, 0);
777
778	if ((type == UE_CONTROL) && (n_frbuffers > 1)) {
779	usbd_xfer_set_frame_offset(xfer, REQ_SIZE, 1);
780	}
781	}
782	#else
783	/* align data */
784	#ifdef __rtems__
785	#ifdef CPU_DATA_CACHE_ALIGNMENT
786	parm->size[0] += CPU_DATA_CACHE_ALIGNMENT;
787	#endif /* CPU_DATA_CACHE_ALIGNMENT */
788	#else /* __rtems__ */
789	parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
790	#endif /* __rtems__ */
791
792	if (parm->buf != NULL) {
793	xfer->local_buffer =
794	USB_ADD_BYTES(parm->buf, parm->size[0]);
795	#ifdef __rtems__
796	#ifdef CPU_DATA_CACHE_ALIGNMENT
797	xfer->local_buffer = (char *) xfer->local_buffer
798	+ ((-(uintptr_t) xfer->local_buffer)
799	& (CPU_DATA_CACHE_ALIGNMENT - 1));
800	#endif /* CPU_DATA_CACHE_ALIGNMENT */
801	#endif /* __rtems__ */
802
803	usbd_xfer_set_frame_offset(xfer, 0, 0);
804
805	if ((type == UE_CONTROL) && (n_frbuffers > 1)) {
806	usbd_xfer_set_frame_offset(xfer, REQ_SIZE, 1);
807	}
808	}
809	parm->size[0] += parm->bufsize;
810
811	/* align data again */
812	#ifdef __rtems__
813	#ifdef CPU_DATA_CACHE_ALIGNMENT
814	parm->size[0] += CPU_DATA_CACHE_ALIGNMENT;
815	#endif /* CPU_DATA_CACHE_ALIGNMENT */
816	#endif /* __rtems__ */
817	parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
818	#endif
819	}
820	/*
821	* Compute maximum buffer size
822	*/
823
824	if (parm->bufsize_max < parm->bufsize) {
825	parm->bufsize_max = parm->bufsize;
826	}
827	#if USB_HAVE_BUSDMA
828	if (xfer->flags_int.bdma_enable) {
829	/*
830	* Setup "dma_page_ptr".
831	*
832	* Proof for formula below:
833	*
834	* Assume there are three USB frames having length "a", "b" and
835	* "c". These USB frames will at maximum need "z"
836	* "usb_page" structures. "z" is given by:
837	*
838	* z = ((a / USB_PAGE_SIZE) + 2) + ((b / USB_PAGE_SIZE) + 2) +
839	* ((c / USB_PAGE_SIZE) + 2);
840	*
841	* Constraining "a", "b" and "c" like this:
842	*
843	* (a + b + c) <= parm->bufsize
844	*
845	* We know that:
846	*
847	* z <= ((parm->bufsize / USB_PAGE_SIZE) + (3*2));
848	*
849	* Here is the general formula:
850	*/
851	xfer->dma_page_ptr = parm->dma_page_ptr;
852	parm->dma_page_ptr += (2 * n_frbuffers);
853	parm->dma_page_ptr += (parm->bufsize / USB_PAGE_SIZE);
854	}
855	#endif
856	if (zmps) {
857	/* correct maximum data length */
858	xfer->max_data_length = 0;
859	}
860	/* subtract USB frame remainder from "hc_max_frame_size" */
861
862	xfer->max_hc_frame_size =
863	(parm->hc_max_frame_size -
864	(parm->hc_max_frame_size % xfer->max_frame_size));
865
866	if (xfer->max_hc_frame_size == 0) {
867	parm->err = USB_ERR_INVAL;
868	goto done;
869	}
870
871	/* initialize frame buffers */
872
873	if (parm->buf) {
874	for (x = 0; x != n_frbuffers; x++) {
875	xfer->frbuffers[x].tag_parent =
876	&xfer->xroot->dma_parent_tag;
877	#if USB_HAVE_BUSDMA
878	if (xfer->flags_int.bdma_enable &&
879	(parm->bufsize_max > 0)) {
880
881	if (usb_pc_dmamap_create(
882	xfer->frbuffers + x,
883	parm->bufsize_max)) {
884	parm->err = USB_ERR_NOMEM;
885	goto done;
886	}
887	}
888	#endif
889	}
890	}
891	done:
892	if (parm->err) {
893	/*
894	* Set some dummy values so that we avoid division by zero:
895	*/
896	xfer->max_hc_frame_size = 1;
897	xfer->max_frame_size = 1;
898	xfer->max_packet_size = 1;
899	xfer->max_data_length = 0;
900	xfer->nframes = 0;
901	xfer->max_frame_count = 0;
902	}
903	}
904
905	static uint8_t
906	usbd_transfer_setup_has_bulk(const struct usb_config *setup_start,
907	uint16_t n_setup)
908	{
909	while (n_setup--) {
910	uint8_t type = setup_start[n_setup].type;
911	if (type == UE_BULK \|\| type == UE_BULK_INTR \|\|
912	type == UE_TYPE_ANY)
913	return (1);
914	}
915	return (0);
916	}
917
918	/------------------------------------------------------------------------
919	* usbd_transfer_setup - setup an array of USB transfers
920	*
921	* NOTE: You must always call "usbd_transfer_unsetup" after calling
922	* "usbd_transfer_setup" if success was returned.
923	*
924	* The idea is that the USB device driver should pre-allocate all its
925	* transfers by one call to this function.
926	*
927	* Return values:
928	* 0: Success
929	* Else: Failure
930	------------------------------------------------------------------------/
931	usb_error_t
932	usbd_transfer_setup(struct usb_device *udev,
933	const uint8_t ifaces, struct usb_xfer *ppxfer,
934	const struct usb_config *setup_start, uint16_t n_setup,
935	void priv_sc, struct mtx xfer_mtx)
936	{
937	const struct usb_config *setup_end = setup_start + n_setup;
938	const struct usb_config *setup;
939	struct usb_setup_params *parm;
940	struct usb_endpoint *ep;
941	struct usb_xfer_root *info;
942	struct usb_xfer *xfer;
943	void *buf = NULL;
944	usb_error_t error = 0;
945	uint16_t n;
946	uint16_t refcount;
947	uint8_t do_unlock;
948
949	WITNESS_WARN(WARN_GIANTOK \| WARN_SLEEPOK, NULL,
950	"usbd_transfer_setup can sleep!");
951
952	/* do some checking first */
953
954	if (n_setup == 0) {
955	DPRINTFN(6, "setup array has zero length!\n");
956	return (USB_ERR_INVAL);
957	}
958	if (ifaces == NULL) {
959	DPRINTFN(6, "ifaces array is NULL!\n");
960	return (USB_ERR_INVAL);
961	}
962	if (xfer_mtx == NULL) {
963	DPRINTFN(6, "using global lock\n");
964	xfer_mtx = &Giant;
965	}
966
967	/* more sanity checks */
968
969	for (setup = setup_start, n = 0;
970	setup != setup_end; setup++, n++) {
971	if (setup->bufsize == (usb_frlength_t)-1) {
972	error = USB_ERR_BAD_BUFSIZE;
973	DPRINTF("invalid bufsize\n");
974	}
975	if (setup->callback == NULL) {
976	error = USB_ERR_NO_CALLBACK;
977	DPRINTF("no callback\n");
978	}
979	ppxfer[n] = NULL;
980	}
981
982	if (error)
983	return (error);
984
985	/* Protect scratch area */
986	do_unlock = usbd_enum_lock(udev);
987
988	refcount = 0;
989	info = NULL;
990
991	parm = &udev->scratch.xfer_setup[0].parm;
992	memset(parm, 0, sizeof(*parm));
993
994	parm->udev = udev;
995	parm->speed = usbd_get_speed(udev);
996	parm->hc_max_packet_count = 1;
997
998	if (parm->speed >= USB_SPEED_MAX) {
999	parm->err = USB_ERR_INVAL;
1000	goto done;
1001	}
1002	/* setup all transfers */
1003
1004	while (1) {
1005
1006	if (buf) {
1007	/*
1008	* Initialize the "usb_xfer_root" structure,
1009	* which is common for all our USB transfers.
1010	*/
1011	info = USB_ADD_BYTES(buf, 0);
1012
1013	info->memory_base = buf;
1014	info->memory_size = parm->size[0];
1015
1016	#if USB_HAVE_BUSDMA
1017	info->dma_page_cache_start = USB_ADD_BYTES(buf, parm->size[4]);
1018	info->dma_page_cache_end = USB_ADD_BYTES(buf, parm->size[5]);
1019	#endif
1020	info->xfer_page_cache_start = USB_ADD_BYTES(buf, parm->size[5]);
1021	info->xfer_page_cache_end = USB_ADD_BYTES(buf, parm->size[2]);
1022
1023	cv_init(&info->cv_drain, "WDRAIN");
1024
1025	info->xfer_mtx = xfer_mtx;
1026	#if USB_HAVE_BUSDMA
1027	usb_dma_tag_setup(&info->dma_parent_tag,
1028	parm->dma_tag_p, udev->bus->dma_parent_tag[0].tag,
1029	xfer_mtx, &usb_bdma_done_event, udev->bus->dma_bits,
1030	parm->dma_tag_max);
1031	#endif
1032
1033	info->bus = udev->bus;
1034	info->udev = udev;
1035
1036	TAILQ_INIT(&info->done_q.head);
1037	info->done_q.command = &usbd_callback_wrapper;
1038	#if USB_HAVE_BUSDMA
1039	TAILQ_INIT(&info->dma_q.head);
1040	info->dma_q.command = &usb_bdma_work_loop;
1041	#endif
1042	info->done_m[0].hdr.pm_callback = &usb_callback_proc;
1043	info->done_m[0].xroot = info;
1044	info->done_m[1].hdr.pm_callback = &usb_callback_proc;
1045	info->done_m[1].xroot = info;
1046
1047	/*
1048	* In device side mode control endpoint
1049	* requests need to run from a separate
1050	* context, else there is a chance of
1051	* deadlock!
1052	*/
1053	if (setup_start == usb_control_ep_cfg)
1054	info->done_p =
1055	USB_BUS_CONTROL_XFER_PROC(udev->bus);
1056	else if (xfer_mtx == &Giant)
1057	info->done_p =
1058	USB_BUS_GIANT_PROC(udev->bus);
1059	else if (usbd_transfer_setup_has_bulk(setup_start, n_setup))
1060	info->done_p =
1061	USB_BUS_NON_GIANT_BULK_PROC(udev->bus);
1062	else
1063	info->done_p =
1064	USB_BUS_NON_GIANT_ISOC_PROC(udev->bus);
1065	}
1066	/* reset sizes */
1067
1068	parm->size[0] = 0;
1069	parm->buf = buf;
1070	parm->size[0] += sizeof(info[0]);
1071
1072	for (setup = setup_start, n = 0;
1073	setup != setup_end; setup++, n++) {
1074
1075	/* skip USB transfers without callbacks: */
1076	if (setup->callback == NULL) {
1077	continue;
1078	}
1079	/* see if there is a matching endpoint */
1080	ep = usbd_get_endpoint(udev,
1081	ifaces[setup->if_index], setup);
1082
1083	/*
1084	* Check that the USB PIPE is valid and that
1085	* the endpoint mode is proper.
1086	*
1087	* Make sure we don't allocate a streams
1088	* transfer when such a combination is not
1089	* valid.
1090	*/
1091	if ((ep == NULL) \|\| (ep->methods == NULL) \|\|
1092	((ep->ep_mode != USB_EP_MODE_STREAMS) &&
1093	(ep->ep_mode != USB_EP_MODE_DEFAULT)) \|\|
1094	(setup->stream_id != 0 &&
1095	(setup->stream_id >= USB_MAX_EP_STREAMS \|\|
1096	(ep->ep_mode != USB_EP_MODE_STREAMS)))) {
1097	if (setup->flags.no_pipe_ok)
1098	continue;
1099	if ((setup->usb_mode != USB_MODE_DUAL) &&
1100	(setup->usb_mode != udev->flags.usb_mode))
1101	continue;
1102	parm->err = USB_ERR_NO_PIPE;
1103	goto done;
1104	}
1105
1106	/* align data properly */
1107	parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1108
1109	/* store current setup pointer */
1110	parm->curr_setup = setup;
1111
1112	if (buf) {
1113	/*
1114	* Common initialization of the
1115	* "usb_xfer" structure.
1116	*/
1117	xfer = USB_ADD_BYTES(buf, parm->size[0]);
1118	xfer->address = udev->address;
1119	xfer->priv_sc = priv_sc;
1120	xfer->xroot = info;
1121
1122	usb_callout_init_mtx(&xfer->timeout_handle,
1123	&udev->bus->bus_mtx, 0);
1124	} else {
1125	/*
1126	* Setup a dummy xfer, hence we are
1127	* writing to the "usb_xfer"
1128	* structure pointed to by "xfer"
1129	* before we have allocated any
1130	* memory:
1131	*/
1132	xfer = &udev->scratch.xfer_setup[0].dummy;
1133	memset(xfer, 0, sizeof(*xfer));
1134	refcount++;
1135	}
1136
1137	/* set transfer endpoint pointer */
1138	xfer->endpoint = ep;
1139
1140	/* set transfer stream ID */
1141	xfer->stream_id = setup->stream_id;
1142
1143	parm->size[0] += sizeof(xfer[0]);
1144	parm->methods = xfer->endpoint->methods;
1145	parm->curr_xfer = xfer;
1146
1147	/*
1148	* Call the Host or Device controller transfer
1149	* setup routine:
1150	*/
1151	(udev->bus->methods->xfer_setup) (parm);
1152
1153	/* check for error */
1154	if (parm->err)
1155	goto done;
1156
1157	if (buf) {
1158	/*
1159	* Increment the endpoint refcount. This
1160	* basically prevents setting a new
1161	* configuration and alternate setting
1162	* when USB transfers are in use on
1163	* the given interface. Search the USB
1164	* code for "endpoint->refcount_alloc" if you
1165	* want more information.
1166	*/
1167	USB_BUS_LOCK(info->bus);
1168	if (xfer->endpoint->refcount_alloc >= USB_EP_REF_MAX)
1169	parm->err = USB_ERR_INVAL;
1170
1171	xfer->endpoint->refcount_alloc++;
1172
1173	if (xfer->endpoint->refcount_alloc == 0)
1174	panic("usbd_transfer_setup(): Refcount wrapped to zero\n");
1175	USB_BUS_UNLOCK(info->bus);
1176
1177	/*
1178	* Whenever we set ppxfer[] then we
1179	* also need to increment the
1180	* "setup_refcount":
1181	*/
1182	info->setup_refcount++;
1183
1184	/*
1185	* Transfer is successfully setup and
1186	* can be used:
1187	*/
1188	ppxfer[n] = xfer;
1189	}
1190
1191	/* check for error */
1192	if (parm->err)
1193	goto done;
1194	}
1195
1196	if (buf != NULL \|\| parm->err != 0)
1197	goto done;
1198
1199	/* if no transfers, nothing to do */
1200	if (refcount == 0)
1201	goto done;
1202
1203	/* align data properly */
1204	parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1205
1206	/* store offset temporarily */
1207	parm->size[1] = parm->size[0];
1208
1209	/*
1210	* The number of DMA tags required depends on
1211	* the number of endpoints. The current estimate
1212	* for maximum number of DMA tags per endpoint
1213	* is three:
1214	* 1) for loading memory
1215	* 2) for allocating memory
1216	* 3) for fixing memory [UHCI]
1217	*/
1218	parm->dma_tag_max += 3 * MIN(n_setup, USB_EP_MAX);
1219
1220	/*
1221	* DMA tags for QH, TD, Data and more.
1222	*/
1223	parm->dma_tag_max += 8;
1224
1225	parm->dma_tag_p += parm->dma_tag_max;
1226
1227	parm->size[0] += ((uint8_t *)parm->dma_tag_p) -
1228	((uint8_t *)0);
1229
1230	/* align data properly */
1231	parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1232
1233	/* store offset temporarily */
1234	parm->size[3] = parm->size[0];
1235
1236	parm->size[0] += ((uint8_t *)parm->dma_page_ptr) -
1237	((uint8_t *)0);
1238
1239	/* align data properly */
1240	parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1241
1242	/* store offset temporarily */
1243	parm->size[4] = parm->size[0];
1244
1245	parm->size[0] += ((uint8_t *)parm->dma_page_cache_ptr) -
1246	((uint8_t *)0);
1247
1248	/* store end offset temporarily */
1249	parm->size[5] = parm->size[0];
1250
1251	parm->size[0] += ((uint8_t *)parm->xfer_page_cache_ptr) -
1252	((uint8_t *)0);
1253
1254	/* store end offset temporarily */
1255
1256	parm->size[2] = parm->size[0];
1257
1258	/* align data properly */
1259	parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1260
1261	parm->size[6] = parm->size[0];
1262
1263	parm->size[0] += ((uint8_t *)parm->xfer_length_ptr) -
1264	((uint8_t *)0);
1265
1266	/* align data properly */
1267	parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1268
1269	/* allocate zeroed memory */
1270	buf = malloc(parm->size[0], M_USB, M_WAITOK \| M_ZERO);
1271
1272	if (buf == NULL) {
1273	parm->err = USB_ERR_NOMEM;
1274	DPRINTFN(0, "cannot allocate memory block for "
1275	"configuration (%d bytes)\n",
1276	parm->size[0]);
1277	goto done;
1278	}
1279	parm->dma_tag_p = USB_ADD_BYTES(buf, parm->size[1]);
1280	parm->dma_page_ptr = USB_ADD_BYTES(buf, parm->size[3]);
1281	parm->dma_page_cache_ptr = USB_ADD_BYTES(buf, parm->size[4]);
1282	parm->xfer_page_cache_ptr = USB_ADD_BYTES(buf, parm->size[5]);
1283	parm->xfer_length_ptr = USB_ADD_BYTES(buf, parm->size[6]);
1284	}
1285
1286	done:
1287	if (buf) {
1288	if (info->setup_refcount == 0) {
1289	/*
1290	* "usbd_transfer_unsetup_sub" will unlock
1291	* the bus mutex before returning !
1292	*/
1293	USB_BUS_LOCK(info->bus);
1294
1295	/* something went wrong */
1296	usbd_transfer_unsetup_sub(info, 0);
1297	}
1298	}
1299
1300	/* check if any errors happened */
1301	if (parm->err)
1302	usbd_transfer_unsetup(ppxfer, n_setup);
1303
1304	error = parm->err;
1305
1306	if (do_unlock)
1307	usbd_enum_unlock(udev);
1308
1309	return (error);
1310	}
1311
1312	/------------------------------------------------------------------------
1313	* usbd_transfer_unsetup_sub - factored out code
1314	------------------------------------------------------------------------/
1315	static void
1316	usbd_transfer_unsetup_sub(struct usb_xfer_root *info, uint8_t needs_delay)
1317	{
1318	#if USB_HAVE_BUSDMA
1319	struct usb_page_cache *pc;
1320	#endif
1321
1322	USB_BUS_LOCK_ASSERT(info->bus, MA_OWNED);
1323
1324	/* wait for any outstanding DMA operations */
1325
1326	if (needs_delay) {
1327	usb_timeout_t temp;
1328	temp = usbd_get_dma_delay(info->udev);
1329	if (temp != 0) {
1330	usb_pause_mtx(&info->bus->bus_mtx,
1331	USB_MS_TO_TICKS(temp));
1332	}
1333	}
1334
1335	/* make sure that our done messages are not queued anywhere */
1336	usb_proc_mwait(info->done_p, &info->done_m[0], &info->done_m[1]);
1337
1338	USB_BUS_UNLOCK(info->bus);
1339
1340	#if USB_HAVE_BUSDMA
1341	/* free DMA'able memory, if any */
1342	pc = info->dma_page_cache_start;
1343	while (pc != info->dma_page_cache_end) {
1344	usb_pc_free_mem(pc);
1345	pc++;
1346	}
1347
1348	/* free DMA maps in all "xfer->frbuffers" */
1349	pc = info->xfer_page_cache_start;
1350	while (pc != info->xfer_page_cache_end) {
1351	usb_pc_dmamap_destroy(pc);
1352	pc++;
1353	}
1354
1355	/* free all DMA tags */
1356	usb_dma_tag_unsetup(&info->dma_parent_tag);
1357	#endif
1358
1359	cv_destroy(&info->cv_drain);
1360
1361	/*
1362	* free the "memory_base" last, hence the "info" structure is
1363	* contained within the "memory_base"!
1364	*/
1365	free(info->memory_base, M_USB);
1366	}
1367
1368	/------------------------------------------------------------------------
1369	* usbd_transfer_unsetup - unsetup/free an array of USB transfers
1370	*
1371	* NOTE: All USB transfers in progress will get called back passing
1372	* the error code "USB_ERR_CANCELLED" before this function
1373	* returns.
1374	------------------------------------------------------------------------/
1375	void
1376	usbd_transfer_unsetup(struct usb_xfer **pxfer, uint16_t n_setup)
1377	{
1378	struct usb_xfer *xfer;
1379	struct usb_xfer_root *info;
1380	uint8_t needs_delay = 0;
1381
1382	WITNESS_WARN(WARN_GIANTOK \| WARN_SLEEPOK, NULL,
1383	"usbd_transfer_unsetup can sleep!");
1384
1385	while (n_setup--) {
1386	xfer = pxfer[n_setup];
1387
1388	if (xfer == NULL)
1389	continue;
1390
1391	info = xfer->xroot;
1392
1393	USB_XFER_LOCK(xfer);
1394	USB_BUS_LOCK(info->bus);
1395
1396	/*
1397	* HINT: when you start/stop a transfer, it might be a
1398	* good idea to directly use the "pxfer[]" structure:
1399	*
1400	* usbd_transfer_start(sc->pxfer[0]);
1401	* usbd_transfer_stop(sc->pxfer[0]);
1402	*
1403	* That way, if your code has many parts that will not
1404	* stop running under the same lock, in other words
1405	* "xfer_mtx", the usbd_transfer_start and
1406	* usbd_transfer_stop functions will simply return
1407	* when they detect a NULL pointer argument.
1408	*
1409	* To avoid any races we clear the "pxfer[]" pointer
1410	* while holding the private mutex of the driver:
1411	*/
1412	pxfer[n_setup] = NULL;
1413
1414	USB_BUS_UNLOCK(info->bus);
1415	USB_XFER_UNLOCK(xfer);
1416
1417	usbd_transfer_drain(xfer);
1418
1419	#if USB_HAVE_BUSDMA
1420	if (xfer->flags_int.bdma_enable)
1421	needs_delay = 1;
1422	#endif
1423	/*
1424	* NOTE: default endpoint does not have an
1425	* interface, even if endpoint->iface_index == 0
1426	*/
1427	USB_BUS_LOCK(info->bus);
1428	xfer->endpoint->refcount_alloc--;
1429	USB_BUS_UNLOCK(info->bus);
1430
1431	usb_callout_drain(&xfer->timeout_handle);
1432
1433	USB_BUS_LOCK(info->bus);
1434
1435	USB_ASSERT(info->setup_refcount != 0, ("Invalid setup "
1436	"reference count\n"));
1437
1438	info->setup_refcount--;
1439
1440	if (info->setup_refcount == 0) {
1441	usbd_transfer_unsetup_sub(info,
1442	needs_delay);
1443	} else {
1444	USB_BUS_UNLOCK(info->bus);
1445	}
1446	}
1447	}
1448
1449	/------------------------------------------------------------------------
1450	* usbd_control_transfer_init - factored out code
1451	*
1452	* In USB Device Mode we have to wait for the SETUP packet which
1453	* containst the "struct usb_device_request" structure, before we can
1454	* transfer any data. In USB Host Mode we already have the SETUP
1455	* packet at the moment the USB transfer is started. This leads us to
1456	* having to setup the USB transfer at two different places in
1457	* time. This function just contains factored out control transfer
1458	* initialisation code, so that we don't duplicate the code.
1459	------------------------------------------------------------------------/
1460	static void
1461	usbd_control_transfer_init(struct usb_xfer *xfer)
1462	{
1463	struct usb_device_request req;
1464
1465	/* copy out the USB request header */
1466
1467	usbd_copy_out(xfer->frbuffers, 0, &req, sizeof(req));
1468
1469	/* setup remainder */
1470
1471	xfer->flags_int.control_rem = UGETW(req.wLength);
1472
1473	/* copy direction to endpoint variable */
1474
1475	xfer->endpointno &= ~(UE_DIR_IN \| UE_DIR_OUT);
1476	xfer->endpointno \|=
1477	(req.bmRequestType & UT_READ) ? UE_DIR_IN : UE_DIR_OUT;
1478	}
1479
1480	/------------------------------------------------------------------------
1481	* usbd_control_transfer_did_data
1482	*
1483	* This function returns non-zero if a control endpoint has
1484	* transferred the first DATA packet after the SETUP packet.
1485	* Else it returns zero.
1486	------------------------------------------------------------------------/
1487	static uint8_t
1488	usbd_control_transfer_did_data(struct usb_xfer *xfer)
1489	{
1490	struct usb_device_request req;
1491
1492	/* SETUP packet is not yet sent */
1493	if (xfer->flags_int.control_hdr != 0)
1494	return (0);
1495
1496	/* copy out the USB request header */
1497	usbd_copy_out(xfer->frbuffers, 0, &req, sizeof(req));
1498
1499	/* compare remainder to the initial value */
1500	return (xfer->flags_int.control_rem != UGETW(req.wLength));
1501	}
1502
1503	/------------------------------------------------------------------------
1504	* usbd_setup_ctrl_transfer
1505	*
1506	* This function handles initialisation of control transfers. Control
1507	* transfers are special in that regard that they can both transmit
1508	* and receive data.
1509	*
1510	* Return values:
1511	* 0: Success
1512	* Else: Failure
1513	------------------------------------------------------------------------/
1514	static int
1515	usbd_setup_ctrl_transfer(struct usb_xfer *xfer)
1516	{
1517	usb_frlength_t len;
1518
1519	/* Check for control endpoint stall */
1520	if (xfer->flags.stall_pipe && xfer->flags_int.control_act) {
1521	/* the control transfer is no longer active */
1522	xfer->flags_int.control_stall = 1;
1523	xfer->flags_int.control_act = 0;
1524	} else {
1525	/* don't stall control transfer by default */
1526	xfer->flags_int.control_stall = 0;
1527	}
1528
1529	/* Check for invalid number of frames */
1530	if (xfer->nframes > 2) {
1531	/*
1532	* If you need to split a control transfer, you
1533	* have to do one part at a time. Only with
1534	* non-control transfers you can do multiple
1535	* parts a time.
1536	*/
1537	DPRINTFN(0, "Too many frames: %u\n",
1538	(unsigned int)xfer->nframes);
1539	goto error;
1540	}
1541
1542	/*
1543	* Check if there is a control
1544	* transfer in progress:
1545	*/
1546	if (xfer->flags_int.control_act) {
1547
1548	if (xfer->flags_int.control_hdr) {
1549
1550	/* clear send header flag */
1551
1552	xfer->flags_int.control_hdr = 0;
1553
1554	/* setup control transfer */
1555	if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
1556	usbd_control_transfer_init(xfer);
1557	}
1558	}
1559	/* get data length */
1560
1561	len = xfer->sumlen;
1562
1563	} else {
1564
1565	/* the size of the SETUP structure is hardcoded ! */
1566
1567	if (xfer->frlengths[0] != sizeof(struct usb_device_request)) {
1568	DPRINTFN(0, "Wrong framelength %u != %zu\n",
1569	xfer->frlengths[0], sizeof(struct
1570	usb_device_request));
1571	goto error;
1572	}
1573	/* check USB mode */
1574	if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
1575
1576	/* check number of frames */
1577	if (xfer->nframes != 1) {
1578	/*
1579	* We need to receive the setup
1580	* message first so that we know the
1581	* data direction!
1582	*/
1583	DPRINTF("Misconfigured transfer\n");
1584	goto error;
1585	}
1586	/*
1587	* Set a dummy "control_rem" value. This
1588	* variable will be overwritten later by a
1589	* call to "usbd_control_transfer_init()" !
1590	*/
1591	xfer->flags_int.control_rem = 0xFFFF;
1592	} else {
1593
1594	/* setup "endpoint" and "control_rem" */
1595
1596	usbd_control_transfer_init(xfer);
1597	}
1598
1599	/* set transfer-header flag */
1600
1601	xfer->flags_int.control_hdr = 1;
1602
1603	/* get data length */
1604
1605	len = (xfer->sumlen - sizeof(struct usb_device_request));
1606	}
1607
1608	/* update did data flag */
1609
1610	xfer->flags_int.control_did_data =
1611	usbd_control_transfer_did_data(xfer);
1612
1613	/* check if there is a length mismatch */
1614
1615	if (len > xfer->flags_int.control_rem) {
1616	DPRINTFN(0, "Length (%d) greater than "
1617	"remaining length (%d)\n", len,
1618	xfer->flags_int.control_rem);
1619	goto error;
1620	}
1621	/* check if we are doing a short transfer */
1622
1623	if (xfer->flags.force_short_xfer) {
1624	xfer->flags_int.control_rem = 0;
1625	} else {
1626	if ((len != xfer->max_data_length) &&
1627	(len != xfer->flags_int.control_rem) &&
1628	(xfer->nframes != 1)) {
1629	DPRINTFN(0, "Short control transfer without "
1630	"force_short_xfer set\n");
1631	goto error;
1632	}
1633	xfer->flags_int.control_rem -= len;
1634	}
1635
1636	/* the status part is executed when "control_act" is 0 */
1637
1638	if ((xfer->flags_int.control_rem > 0) \|\|
1639	(xfer->flags.manual_status)) {
1640	/* don't execute the STATUS stage yet */
1641	xfer->flags_int.control_act = 1;
1642
1643	/* sanity check */
1644	if ((!xfer->flags_int.control_hdr) &&
1645	(xfer->nframes == 1)) {
1646	/*
1647	* This is not a valid operation!
1648	*/
1649	DPRINTFN(0, "Invalid parameter "
1650	"combination\n");
1651	goto error;
1652	}
1653	} else {
1654	/* time to execute the STATUS stage */
1655	xfer->flags_int.control_act = 0;
1656	}
1657	return (0); /* success */
1658
1659	error:
1660	return (1); /* failure */
1661	}
1662
1663	/------------------------------------------------------------------------
1664	* usbd_transfer_submit - start USB hardware for the given transfer
1665	*
1666	* This function should only be called from the USB callback.
1667	------------------------------------------------------------------------/
1668	void
1669	usbd_transfer_submit(struct usb_xfer *xfer)
1670	{
1671	struct usb_xfer_root *info;
1672	struct usb_bus *bus;
1673	usb_frcount_t x;
1674
1675	info = xfer->xroot;
1676	bus = info->bus;
1677
1678	DPRINTF("xfer=%p, endpoint=%p, nframes=%d, dir=%s\n",
1679	xfer, xfer->endpoint, xfer->nframes, USB_GET_DATA_ISREAD(xfer) ?
1680	"read" : "write");
1681
1682	#ifdef USB_DEBUG
1683	if (USB_DEBUG_VAR > 0) {
1684	USB_BUS_LOCK(bus);
1685
1686	usb_dump_endpoint(xfer->endpoint);
1687
1688	USB_BUS_UNLOCK(bus);
1689	}
1690	#endif
1691
1692	USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
1693	USB_BUS_LOCK_ASSERT(bus, MA_NOTOWNED);
1694
1695	/* Only open the USB transfer once! */
1696	if (!xfer->flags_int.open) {
1697	xfer->flags_int.open = 1;
1698
1699	DPRINTF("open\n");
1700
1701	USB_BUS_LOCK(bus);
1702	(xfer->endpoint->methods->open) (xfer);
1703	USB_BUS_UNLOCK(bus);
1704	}
1705	/* set "transferring" flag */
1706	xfer->flags_int.transferring = 1;
1707
1708	#if USB_HAVE_POWERD
1709	/* increment power reference */
1710	usbd_transfer_power_ref(xfer, 1);
1711	#endif
1712	/*
1713	* Check if the transfer is waiting on a queue, most
1714	* frequently the "done_q":
1715	*/
1716	if (xfer->wait_queue) {
1717	USB_BUS_LOCK(bus);
1718	usbd_transfer_dequeue(xfer);
1719	USB_BUS_UNLOCK(bus);
1720	}
1721	/* clear "did_dma_delay" flag */
1722	xfer->flags_int.did_dma_delay = 0;
1723
1724	/* clear "did_close" flag */
1725	xfer->flags_int.did_close = 0;
1726
1727	#if USB_HAVE_BUSDMA
1728	/* clear "bdma_setup" flag */
1729	xfer->flags_int.bdma_setup = 0;
1730	#endif
1731	/* by default we cannot cancel any USB transfer immediately */
1732	xfer->flags_int.can_cancel_immed = 0;
1733
1734	/* clear lengths and frame counts by default */
1735	xfer->sumlen = 0;
1736	xfer->actlen = 0;
1737	xfer->aframes = 0;
1738
1739	/* clear any previous errors */
1740	xfer->error = 0;
1741
1742	/* Check if the device is still alive */
1743	if (info->udev->state < USB_STATE_POWERED) {
1744	USB_BUS_LOCK(bus);
1745	/*
1746	* Must return cancelled error code else
1747	* device drivers can hang.
1748	*/
1749	usbd_transfer_done(xfer, USB_ERR_CANCELLED);
1750	USB_BUS_UNLOCK(bus);
1751	return;
1752	}
1753
1754	/* sanity check */
1755	if (xfer->nframes == 0) {
1756	if (xfer->flags.stall_pipe) {
1757	/*
1758	* Special case - want to stall without transferring
1759	* any data:
1760	*/
1761	DPRINTF("xfer=%p nframes=0: stall "
1762	"or clear stall!\n", xfer);
1763	USB_BUS_LOCK(bus);
1764	xfer->flags_int.can_cancel_immed = 1;
1765	/* start the transfer */
1766	usb_command_wrapper(&xfer->endpoint->
1767	endpoint_q[xfer->stream_id], xfer);
1768	USB_BUS_UNLOCK(bus);
1769	return;
1770	}
1771	USB_BUS_LOCK(bus);
1772	usbd_transfer_done(xfer, USB_ERR_INVAL);
1773	USB_BUS_UNLOCK(bus);
1774	return;
1775	}
1776	/* compute some variables */
1777
1778	for (x = 0; x != xfer->nframes; x++) {
1779	/* make a copy of the frlenghts[] */
1780	xfer->frlengths[x + xfer->max_frame_count] = xfer->frlengths[x];
1781	/* compute total transfer length */
1782	xfer->sumlen += xfer->frlengths[x];
1783	if (xfer->sumlen < xfer->frlengths[x]) {
1784	/* length wrapped around */
1785	USB_BUS_LOCK(bus);
1786	usbd_transfer_done(xfer, USB_ERR_INVAL);
1787	USB_BUS_UNLOCK(bus);
1788	return;
1789	}
1790	}
1791
1792	/* clear some internal flags */
1793
1794	xfer->flags_int.short_xfer_ok = 0;
1795	xfer->flags_int.short_frames_ok = 0;
1796
1797	/* check if this is a control transfer */
1798
1799	if (xfer->flags_int.control_xfr) {
1800
1801	if (usbd_setup_ctrl_transfer(xfer)) {
1802	USB_BUS_LOCK(bus);
1803	usbd_transfer_done(xfer, USB_ERR_STALLED);
1804	USB_BUS_UNLOCK(bus);
1805	return;
1806	}
1807	}
1808	/*
1809	* Setup filtered version of some transfer flags,
1810	* in case of data read direction
1811	*/
1812	if (USB_GET_DATA_ISREAD(xfer)) {
1813
1814	if (xfer->flags.short_frames_ok) {
1815	xfer->flags_int.short_xfer_ok = 1;
1816	xfer->flags_int.short_frames_ok = 1;
1817	} else if (xfer->flags.short_xfer_ok) {
1818	xfer->flags_int.short_xfer_ok = 1;
1819
1820	/* check for control transfer */
1821	if (xfer->flags_int.control_xfr) {
1822	/*
1823	* 1) Control transfers do not support
1824	* reception of multiple short USB
1825	* frames in host mode and device side
1826	* mode, with exception of:
1827	*
1828	* 2) Due to sometimes buggy device
1829	* side firmware we need to do a
1830	* STATUS stage in case of short
1831	* control transfers in USB host mode.
1832	* The STATUS stage then becomes the
1833	* "alt_next" to the DATA stage.
1834	*/
1835	xfer->flags_int.short_frames_ok = 1;
1836	}
1837	}
1838	}
1839	/*
1840	* Check if BUS-DMA support is enabled and try to load virtual
1841	* buffers into DMA, if any:
1842	*/
1843	#if USB_HAVE_BUSDMA
1844	if (xfer->flags_int.bdma_enable) {
1845	/* insert the USB transfer last in the BUS-DMA queue */
1846	usb_command_wrapper(&xfer->xroot->dma_q, xfer);
1847	return;
1848	}
1849	#endif
1850	/*
1851	* Enter the USB transfer into the Host Controller or
1852	* Device Controller schedule:
1853	*/
1854	usbd_pipe_enter(xfer);
1855	}
1856
1857	/------------------------------------------------------------------------
1858	* usbd_pipe_enter - factored out code
1859	------------------------------------------------------------------------/
1860	void
1861	usbd_pipe_enter(struct usb_xfer *xfer)
1862	{
1863	struct usb_endpoint *ep;
1864
1865	USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
1866
1867	USB_BUS_LOCK(xfer->xroot->bus);
1868
1869	ep = xfer->endpoint;
1870
1871	DPRINTF("enter\n");
1872
1873	/* the transfer can now be cancelled */
1874	xfer->flags_int.can_cancel_immed = 1;
1875
1876	/* enter the transfer */
1877	(ep->methods->enter) (xfer);
1878
1879	/* check for transfer error */
1880	if (xfer->error) {
1881	/* some error has happened */
1882	usbd_transfer_done(xfer, 0);
1883	USB_BUS_UNLOCK(xfer->xroot->bus);
1884	return;
1885	}
1886
1887	/* start the transfer */
1888	usb_command_wrapper(&ep->endpoint_q[xfer->stream_id], xfer);
1889	USB_BUS_UNLOCK(xfer->xroot->bus);
1890	}
1891
1892	/------------------------------------------------------------------------
1893	* usbd_transfer_start - start an USB transfer
1894	*
1895	* NOTE: Calling this function more than one time will only
1896	* result in a single transfer start, until the USB transfer
1897	* completes.
1898	------------------------------------------------------------------------/
1899	void
1900	usbd_transfer_start(struct usb_xfer *xfer)
1901	{
1902	if (xfer == NULL) {
1903	/* transfer is gone */
1904	return;
1905	}
1906	USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
1907
1908	/* mark the USB transfer started */
1909
1910	if (!xfer->flags_int.started) {
1911	/* lock the BUS lock to avoid races updating flags_int */
1912	USB_BUS_LOCK(xfer->xroot->bus);
1913	xfer->flags_int.started = 1;
1914	USB_BUS_UNLOCK(xfer->xroot->bus);
1915	}
1916	/* check if the USB transfer callback is already transferring */
1917
1918	if (xfer->flags_int.transferring) {
1919	return;
1920	}
1921	USB_BUS_LOCK(xfer->xroot->bus);
1922	/* call the USB transfer callback */
1923	usbd_callback_ss_done_defer(xfer);
1924	USB_BUS_UNLOCK(xfer->xroot->bus);
1925	}
1926
1927	/------------------------------------------------------------------------
1928	* usbd_transfer_stop - stop an USB transfer
1929	*
1930	* NOTE: Calling this function more than one time will only
1931	* result in a single transfer stop.
1932	* NOTE: When this function returns it is not safe to free nor
1933	* reuse any DMA buffers. See "usbd_transfer_drain()".
1934	------------------------------------------------------------------------/
1935	void
1936	usbd_transfer_stop(struct usb_xfer *xfer)
1937	{
1938	struct usb_endpoint *ep;
1939
1940	if (xfer == NULL) {
1941	/* transfer is gone */
1942	return;
1943	}
1944	USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
1945
1946	/* check if the USB transfer was ever opened */
1947
1948	if (!xfer->flags_int.open) {
1949	if (xfer->flags_int.started) {
1950	/* nothing to do except clearing the "started" flag */
1951	/* lock the BUS lock to avoid races updating flags_int */
1952	USB_BUS_LOCK(xfer->xroot->bus);
1953	xfer->flags_int.started = 0;
1954	USB_BUS_UNLOCK(xfer->xroot->bus);
1955	}
1956	return;
1957	}
1958	/* try to stop the current USB transfer */
1959
1960	USB_BUS_LOCK(xfer->xroot->bus);
1961	/* override any previous error */
1962	xfer->error = USB_ERR_CANCELLED;
1963
1964	/*
1965	* Clear "open" and "started" when both private and USB lock
1966	* is locked so that we don't get a race updating "flags_int"
1967	*/
1968	xfer->flags_int.open = 0;
1969	xfer->flags_int.started = 0;
1970
1971	/*
1972	* Check if we can cancel the USB transfer immediately.
1973	*/
1974	if (xfer->flags_int.transferring) {
1975	if (xfer->flags_int.can_cancel_immed &&
1976	(!xfer->flags_int.did_close)) {
1977	DPRINTF("close\n");
1978	/*
1979	* The following will lead to an USB_ERR_CANCELLED
1980	* error code being passed to the USB callback.
1981	*/
1982	(xfer->endpoint->methods->close) (xfer);
1983	/* only close once */
1984	xfer->flags_int.did_close = 1;
1985	} else {
1986	/* need to wait for the next done callback */
1987	}
1988	} else {
1989	DPRINTF("close\n");
1990
1991	/* close here and now */
1992	(xfer->endpoint->methods->close) (xfer);
1993
1994	/*
1995	* Any additional DMA delay is done by
1996	* "usbd_transfer_unsetup()".
1997	*/
1998
1999	/*
2000	* Special case. Check if we need to restart a blocked
2001	* endpoint.
2002	*/
2003	ep = xfer->endpoint;
2004
2005	/*
2006	* If the current USB transfer is completing we need
2007	* to start the next one:
2008	*/
2009	if (ep->endpoint_q[xfer->stream_id].curr == xfer) {
2010	usb_command_wrapper(
2011	&ep->endpoint_q[xfer->stream_id], NULL);
2012	}
2013	}
2014
2015	USB_BUS_UNLOCK(xfer->xroot->bus);
2016	}
2017
2018	/------------------------------------------------------------------------
2019	* usbd_transfer_pending
2020	*
2021	* This function will check if an USB transfer is pending which is a
2022	* little bit complicated!
2023	* Return values:
2024	* 0: Not pending
2025	* 1: Pending: The USB transfer will receive a callback in the future.
2026	------------------------------------------------------------------------/
2027	uint8_t
2028	usbd_transfer_pending(struct usb_xfer *xfer)
2029	{
2030	struct usb_xfer_root *info;
2031	struct usb_xfer_queue *pq;
2032
2033	if (xfer == NULL) {
2034	/* transfer is gone */
2035	return (0);
2036	}
2037	USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
2038
2039	if (xfer->flags_int.transferring) {
2040	/* trivial case */
2041	return (1);
2042	}
2043	USB_BUS_LOCK(xfer->xroot->bus);
2044	if (xfer->wait_queue) {
2045	/* we are waiting on a queue somewhere */
2046	USB_BUS_UNLOCK(xfer->xroot->bus);
2047	return (1);
2048	}
2049	info = xfer->xroot;
2050	pq = &info->done_q;
2051
2052	if (pq->curr == xfer) {
2053	/* we are currently scheduled for callback */
2054	USB_BUS_UNLOCK(xfer->xroot->bus);
2055	return (1);
2056	}
2057	/* we are not pending */
2058	USB_BUS_UNLOCK(xfer->xroot->bus);
2059	return (0);
2060	}
2061
2062	/------------------------------------------------------------------------
2063	* usbd_transfer_drain
2064	*
2065	* This function will stop the USB transfer and wait for any
2066	* additional BUS-DMA and HW-DMA operations to complete. Buffers that
2067	* are loaded into DMA can safely be freed or reused after that this
2068	* function has returned.
2069	------------------------------------------------------------------------/
2070	void
2071	usbd_transfer_drain(struct usb_xfer *xfer)
2072	{
2073	WITNESS_WARN(WARN_GIANTOK \| WARN_SLEEPOK, NULL,
2074	"usbd_transfer_drain can sleep!");
2075
2076	if (xfer == NULL) {
2077	/* transfer is gone */
2078	return;
2079	}
2080	if (xfer->xroot->xfer_mtx != &Giant) {
2081	USB_XFER_LOCK_ASSERT(xfer, MA_NOTOWNED);
2082	}
2083	USB_XFER_LOCK(xfer);
2084
2085	usbd_transfer_stop(xfer);
2086
2087	while (usbd_transfer_pending(xfer) \|\|
2088	xfer->flags_int.doing_callback) {
2089
2090	/*
2091	* It is allowed that the callback can drop its
2092	* transfer mutex. In that case checking only
2093	* "usbd_transfer_pending()" is not enough to tell if
2094	* the USB transfer is fully drained. We also need to
2095	* check the internal "doing_callback" flag.
2096	*/
2097	xfer->flags_int.draining = 1;
2098
2099	/*
2100	* Wait until the current outstanding USB
2101	* transfer is complete !
2102	*/
2103	cv_wait(&xfer->xroot->cv_drain, xfer->xroot->xfer_mtx);
2104	}
2105	USB_XFER_UNLOCK(xfer);
2106	}
2107
2108	struct usb_page_cache *
2109	usbd_xfer_get_frame(struct usb_xfer *xfer, usb_frcount_t frindex)
2110	{
2111	KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2112
2113	return (&xfer->frbuffers[frindex]);
2114	}
2115
2116	void *
2117	usbd_xfer_get_frame_buffer(struct usb_xfer *xfer, usb_frcount_t frindex)
2118	{
2119	struct usb_page_search page_info;
2120
2121	KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2122
2123	usbd_get_page(&xfer->frbuffers[frindex], 0, &page_info);
2124	return (page_info.buffer);
2125	}
2126
2127	/------------------------------------------------------------------------
2128	* usbd_xfer_get_fps_shift
2129	*
2130	* The following function is only useful for isochronous transfers. It
2131	* returns how many times the frame execution rate has been shifted
2132	* down.
2133	*
2134	* Return value:
2135	* Success: 0..3
2136	* Failure: 0
2137	------------------------------------------------------------------------/
2138	uint8_t
2139	usbd_xfer_get_fps_shift(struct usb_xfer *xfer)
2140	{
2141	return (xfer->fps_shift);
2142	}
2143
2144	usb_frlength_t
2145	usbd_xfer_frame_len(struct usb_xfer *xfer, usb_frcount_t frindex)
2146	{
2147	KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2148
2149	return (xfer->frlengths[frindex]);
2150	}
2151
2152	/------------------------------------------------------------------------
2153	* usbd_xfer_set_frame_data
2154	*
2155	* This function sets the pointer of the buffer that should
2156	* loaded directly into DMA for the given USB frame. Passing "ptr"
2157	* equal to NULL while the corresponding "frlength" is greater
2158	* than zero gives undefined results!
2159	------------------------------------------------------------------------/
2160	void
2161	usbd_xfer_set_frame_data(struct usb_xfer *xfer, usb_frcount_t frindex,
2162	void *ptr, usb_frlength_t len)
2163	{
2164	KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2165
2166	/* set virtual address to load and length */
2167	xfer->frbuffers[frindex].buffer = ptr;
2168	usbd_xfer_set_frame_len(xfer, frindex, len);
2169	}
2170
2171	void
2172	usbd_xfer_frame_data(struct usb_xfer *xfer, usb_frcount_t frindex,
2173	void *ptr, int len)
2174	{
2175	KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2176
2177	if (ptr != NULL)
2178	*ptr = xfer->frbuffers[frindex].buffer;
2179	if (len != NULL)
2180	*len = xfer->frlengths[frindex];
2181	}
2182
2183	/------------------------------------------------------------------------
2184	* usbd_xfer_old_frame_length
2185	*
2186	* This function returns the framelength of the given frame at the
2187	* time the transfer was submitted. This function can be used to
2188	* compute the starting data pointer of the next isochronous frame
2189	* when an isochronous transfer has completed.
2190	------------------------------------------------------------------------/
2191	usb_frlength_t
2192	usbd_xfer_old_frame_length(struct usb_xfer *xfer, usb_frcount_t frindex)
2193	{
2194	KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2195
2196	return (xfer->frlengths[frindex + xfer->max_frame_count]);
2197	}
2198
2199	void
2200	usbd_xfer_status(struct usb_xfer xfer, int actlen, int sumlen, int aframes,
2201	int *nframes)
2202	{
2203	if (actlen != NULL)
2204	*actlen = xfer->actlen;
2205	if (sumlen != NULL)
2206	*sumlen = xfer->sumlen;
2207	if (aframes != NULL)
2208	*aframes = xfer->aframes;
2209	if (nframes != NULL)
2210	*nframes = xfer->nframes;
2211	}
2212
2213	/------------------------------------------------------------------------
2214	* usbd_xfer_set_frame_offset
2215	*
2216	* This function sets the frame data buffer offset relative to the beginning
2217	* of the USB DMA buffer allocated for this USB transfer.
2218	------------------------------------------------------------------------/
2219	void
2220	usbd_xfer_set_frame_offset(struct usb_xfer *xfer, usb_frlength_t offset,
2221	usb_frcount_t frindex)
2222	{
2223	KASSERT(!xfer->flags.ext_buffer, ("Cannot offset data frame "
2224	"when the USB buffer is external\n"));
2225	KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2226
2227	/* set virtual address to load */
2228	xfer->frbuffers[frindex].buffer =
2229	USB_ADD_BYTES(xfer->local_buffer, offset);
2230	}
2231
2232	void
2233	usbd_xfer_set_interval(struct usb_xfer *xfer, int i)
2234	{
2235	xfer->interval = i;
2236	}
2237
2238	void
2239	usbd_xfer_set_timeout(struct usb_xfer *xfer, int t)
2240	{
2241	xfer->timeout = t;
2242	}
2243
2244	void
2245	usbd_xfer_set_frames(struct usb_xfer *xfer, usb_frcount_t n)
2246	{
2247	xfer->nframes = n;
2248	}
2249
2250	usb_frcount_t
2251	usbd_xfer_max_frames(struct usb_xfer *xfer)
2252	{
2253	return (xfer->max_frame_count);
2254	}
2255
2256	usb_frlength_t
2257	usbd_xfer_max_len(struct usb_xfer *xfer)
2258	{
2259	return (xfer->max_data_length);
2260	}
2261
2262	usb_frlength_t
2263	usbd_xfer_max_framelen(struct usb_xfer *xfer)
2264	{
2265	return (xfer->max_frame_size);
2266	}
2267
2268	void
2269	usbd_xfer_set_frame_len(struct usb_xfer *xfer, usb_frcount_t frindex,
2270	usb_frlength_t len)
2271	{
2272	KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2273
2274	xfer->frlengths[frindex] = len;
2275	}
2276
2277	/------------------------------------------------------------------------
2278	* usb_callback_proc - factored out code
2279	*
2280	* This function performs USB callbacks.
2281	------------------------------------------------------------------------/
2282	static void
2283	usb_callback_proc(struct usb_proc_msg *_pm)
2284	{
2285	struct usb_done_msg pm = (void )_pm;
2286	struct usb_xfer_root *info = pm->xroot;
2287
2288	/* Change locking order */
2289	USB_BUS_UNLOCK(info->bus);
2290
2291	/*
2292	* We exploit the fact that the mutex is the same for all
2293	* callbacks that will be called from this thread:
2294	*/
2295	mtx_lock(info->xfer_mtx);
2296	USB_BUS_LOCK(info->bus);
2297
2298	/* Continue where we lost track */
2299	usb_command_wrapper(&info->done_q,
2300	info->done_q.curr);
2301
2302	mtx_unlock(info->xfer_mtx);
2303	}
2304
2305	/------------------------------------------------------------------------
2306	* usbd_callback_ss_done_defer
2307	*
2308	* This function will defer the start, stop and done callback to the
2309	* correct thread.
2310	------------------------------------------------------------------------/
2311	static void
2312	usbd_callback_ss_done_defer(struct usb_xfer *xfer)
2313	{
2314	struct usb_xfer_root *info = xfer->xroot;
2315	struct usb_xfer_queue *pq = &info->done_q;
2316
2317	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
2318
2319	if (pq->curr != xfer) {
2320	usbd_transfer_enqueue(pq, xfer);
2321	}
2322	if (!pq->recurse_1) {
2323
2324	/*
2325	* We have to postpone the callback due to the fact we
2326	* will have a Lock Order Reversal, LOR, if we try to
2327	* proceed !
2328	*/
2329	(void) usb_proc_msignal(info->done_p,
2330	&info->done_m[0], &info->done_m[1]);
2331	} else {
2332	/* clear second recurse flag */
2333	pq->recurse_2 = 0;
2334	}
2335	return;
2336
2337	}
2338
2339	/------------------------------------------------------------------------
2340	* usbd_callback_wrapper
2341	*
2342	* This is a wrapper for USB callbacks. This wrapper does some
2343	* auto-magic things like figuring out if we can call the callback
2344	* directly from the current context or if we need to wakeup the
2345	* interrupt process.
2346	------------------------------------------------------------------------/
2347	static void
2348	usbd_callback_wrapper(struct usb_xfer_queue *pq)
2349	{
2350	struct usb_xfer *xfer = pq->curr;
2351	struct usb_xfer_root *info = xfer->xroot;
2352
2353	USB_BUS_LOCK_ASSERT(info->bus, MA_OWNED);
2354	if ((pq->recurse_3 != 0 \|\| mtx_owned(info->xfer_mtx) == 0) &&
2355	SCHEDULER_STOPPED() == 0) {
2356	/*
2357	* Cases that end up here:
2358	*
2359	* 5) HW interrupt done callback or other source.
2360	* 6) HW completed transfer during callback
2361	*/
2362	DPRINTFN(3, "case 5 and 6\n");
2363
2364	/*
2365	* We have to postpone the callback due to the fact we
2366	* will have a Lock Order Reversal, LOR, if we try to
2367	* proceed!
2368	*
2369	* Postponing the callback also ensures that other USB
2370	* transfer queues get a chance.
2371	*/
2372	(void) usb_proc_msignal(info->done_p,
2373	&info->done_m[0], &info->done_m[1]);
2374	return;
2375	}
2376	/*
2377	* Cases that end up here:
2378	*
2379	* 1) We are starting a transfer
2380	* 2) We are prematurely calling back a transfer
2381	* 3) We are stopping a transfer
2382	* 4) We are doing an ordinary callback
2383	*/
2384	DPRINTFN(3, "case 1-4\n");
2385	/* get next USB transfer in the queue */
2386	info->done_q.curr = NULL;
2387
2388	/* set flag in case of drain */
2389	xfer->flags_int.doing_callback = 1;
2390
2391	USB_BUS_UNLOCK(info->bus);
2392	USB_BUS_LOCK_ASSERT(info->bus, MA_NOTOWNED);
2393
2394	/* set correct USB state for callback */
2395	if (!xfer->flags_int.transferring) {
2396	xfer->usb_state = USB_ST_SETUP;
2397	if (!xfer->flags_int.started) {
2398	/* we got stopped before we even got started */
2399	USB_BUS_LOCK(info->bus);
2400	goto done;
2401	}
2402	} else {
2403
2404	if (usbd_callback_wrapper_sub(xfer)) {
2405	/* the callback has been deferred */
2406	USB_BUS_LOCK(info->bus);
2407	goto done;
2408	}
2409	#if USB_HAVE_POWERD
2410	/* decrement power reference */
2411	usbd_transfer_power_ref(xfer, -1);
2412	#endif
2413	xfer->flags_int.transferring = 0;
2414
2415	if (xfer->error) {
2416	xfer->usb_state = USB_ST_ERROR;
2417	} else {
2418	/* set transferred state */
2419	xfer->usb_state = USB_ST_TRANSFERRED;
2420	#if USB_HAVE_BUSDMA
2421	/* sync DMA memory, if any */
2422	if (xfer->flags_int.bdma_enable &&
2423	(!xfer->flags_int.bdma_no_post_sync)) {
2424	usb_bdma_post_sync(xfer);
2425	}
2426	#endif
2427	}
2428	}
2429
2430	#if USB_HAVE_PF
2431	if (xfer->usb_state != USB_ST_SETUP) {
2432	USB_BUS_LOCK(info->bus);
2433	usbpf_xfertap(xfer, USBPF_XFERTAP_DONE);
2434	USB_BUS_UNLOCK(info->bus);
2435	}
2436	#endif
2437	/* call processing routine */
2438	(xfer->callback) (xfer, xfer->error);
2439
2440	/* pickup the USB mutex again */
2441	USB_BUS_LOCK(info->bus);
2442
2443	/*
2444	* Check if we got started after that we got cancelled, but
2445	* before we managed to do the callback.
2446	*/
2447	if ((!xfer->flags_int.open) &&
2448	(xfer->flags_int.started) &&
2449	(xfer->usb_state == USB_ST_ERROR)) {
2450	/* clear flag in case of drain */
2451	xfer->flags_int.doing_callback = 0;
2452	/* try to loop, but not recursivly */
2453	usb_command_wrapper(&info->done_q, xfer);
2454	return;
2455	}
2456
2457	done:
2458	/* clear flag in case of drain */
2459	xfer->flags_int.doing_callback = 0;
2460
2461	/*
2462	* Check if we are draining.
2463	*/
2464	if (xfer->flags_int.draining &&
2465	(!xfer->flags_int.transferring)) {
2466	/* "usbd_transfer_drain()" is waiting for end of transfer */
2467	xfer->flags_int.draining = 0;
2468	cv_broadcast(&info->cv_drain);
2469	}
2470
2471	/* do the next callback, if any */
2472	usb_command_wrapper(&info->done_q,
2473	info->done_q.curr);
2474	}
2475
2476	/------------------------------------------------------------------------
2477	* usb_dma_delay_done_cb
2478	*
2479	* This function is called when the DMA delay has been exectuded, and
2480	* will make sure that the callback is called to complete the USB
2481	* transfer. This code path is usually only used when there is an USB
2482	* error like USB_ERR_CANCELLED.
2483	------------------------------------------------------------------------/
2484	void
2485	usb_dma_delay_done_cb(struct usb_xfer *xfer)
2486	{
2487	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
2488
2489	DPRINTFN(3, "Completed %p\n", xfer);
2490
2491	/* queue callback for execution, again */
2492	usbd_transfer_done(xfer, 0);
2493	}
2494
2495	/------------------------------------------------------------------------
2496	* usbd_transfer_dequeue
2497	*
2498	* - This function is used to remove an USB transfer from a USB
2499	* transfer queue.
2500	*
2501	* - This function can be called multiple times in a row.
2502	------------------------------------------------------------------------/
2503	void
2504	usbd_transfer_dequeue(struct usb_xfer *xfer)
2505	{
2506	struct usb_xfer_queue *pq;
2507
2508	pq = xfer->wait_queue;
2509	if (pq) {
2510	TAILQ_REMOVE(&pq->head, xfer, wait_entry);
2511	xfer->wait_queue = NULL;
2512	}
2513	}
2514
2515	/------------------------------------------------------------------------
2516	* usbd_transfer_enqueue
2517	*
2518	* - This function is used to insert an USB transfer into a USB *
2519	* transfer queue.
2520	*
2521	* - This function can be called multiple times in a row.
2522	------------------------------------------------------------------------/
2523	void
2524	usbd_transfer_enqueue(struct usb_xfer_queue pq, struct usb_xfer xfer)
2525	{
2526	/*
2527	* Insert the USB transfer into the queue, if it is not
2528	* already on a USB transfer queue:
2529	*/
2530	if (xfer->wait_queue == NULL) {
2531	xfer->wait_queue = pq;
2532	TAILQ_INSERT_TAIL(&pq->head, xfer, wait_entry);
2533	}
2534	}
2535
2536	/------------------------------------------------------------------------
2537	* usbd_transfer_done
2538	*
2539	* - This function is used to remove an USB transfer from the busdma,
2540	* pipe or interrupt queue.
2541	*
2542	* - This function is used to queue the USB transfer on the done
2543	* queue.
2544	*
2545	* - This function is used to stop any USB transfer timeouts.
2546	------------------------------------------------------------------------/
2547	void
2548	usbd_transfer_done(struct usb_xfer *xfer, usb_error_t error)
2549	{
2550	struct usb_xfer_root *info = xfer->xroot;
2551
2552	USB_BUS_LOCK_ASSERT(info->bus, MA_OWNED);
2553
2554	DPRINTF("err=%s\n", usbd_errstr(error));
2555
2556	/*
2557	* If we are not transferring then just return.
2558	* This can happen during transfer cancel.
2559	*/
2560	if (!xfer->flags_int.transferring) {
2561	DPRINTF("not transferring\n");
2562	/* end of control transfer, if any */
2563	xfer->flags_int.control_act = 0;
2564	return;
2565	}
2566	/* only set transfer error, if not already set */
2567	if (xfer->error == USB_ERR_NORMAL_COMPLETION)
2568	xfer->error = error;
2569
2570	/* stop any callouts */
2571	usb_callout_stop(&xfer->timeout_handle);
2572
2573	/*
2574	* If we are waiting on a queue, just remove the USB transfer
2575	* from the queue, if any. We should have the required locks
2576	* locked to do the remove when this function is called.
2577	*/
2578	usbd_transfer_dequeue(xfer);
2579
2580	#if USB_HAVE_BUSDMA
2581	if (mtx_owned(info->xfer_mtx)) {
2582	struct usb_xfer_queue *pq;
2583
2584	/*
2585	* If the private USB lock is not locked, then we assume
2586	* that the BUS-DMA load stage has been passed:
2587	*/
2588	pq = &info->dma_q;
2589
2590	if (pq->curr == xfer) {
2591	/* start the next BUS-DMA load, if any */
2592	usb_command_wrapper(pq, NULL);
2593	}
2594	}
2595	#endif
2596	/* keep some statistics */
2597	if (xfer->error) {
2598	info->bus->stats_err.uds_requests
2599	[xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE]++;
2600	} else {
2601	info->bus->stats_ok.uds_requests
2602	[xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE]++;
2603	}
2604
2605	/* call the USB transfer callback */
2606	usbd_callback_ss_done_defer(xfer);
2607	}
2608
2609	/------------------------------------------------------------------------
2610	* usbd_transfer_start_cb
2611	*
2612	* This function is called to start the USB transfer when
2613	* "xfer->interval" is greater than zero, and and the endpoint type is
2614	* BULK or CONTROL.
2615	------------------------------------------------------------------------/
2616	static void
2617	usbd_transfer_start_cb(void *arg)
2618	{
2619	struct usb_xfer *xfer = arg;
2620	struct usb_endpoint *ep = xfer->endpoint;
2621
2622	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
2623
2624	DPRINTF("start\n");
2625
2626	#if USB_HAVE_PF
2627	usbpf_xfertap(xfer, USBPF_XFERTAP_SUBMIT);
2628	#endif
2629
2630	/* the transfer can now be cancelled */
2631	xfer->flags_int.can_cancel_immed = 1;
2632
2633	/* start USB transfer, if no error */
2634	if (xfer->error == 0)
2635	(ep->methods->start) (xfer);
2636
2637	/* check for transfer error */
2638	if (xfer->error) {
2639	/* some error has happened */
2640	usbd_transfer_done(xfer, 0);
2641	}
2642	}
2643
2644	/------------------------------------------------------------------------
2645	* usbd_xfer_set_stall
2646	*
2647	* This function is used to set the stall flag outside the
2648	* callback. This function is NULL safe.
2649	------------------------------------------------------------------------/
2650	void
2651	usbd_xfer_set_stall(struct usb_xfer *xfer)
2652	{
2653	if (xfer == NULL) {
2654	/* tearing down */
2655	return;
2656	}
2657	USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
2658
2659	/* avoid any races by locking the USB mutex */
2660	USB_BUS_LOCK(xfer->xroot->bus);
2661	xfer->flags.stall_pipe = 1;
2662	USB_BUS_UNLOCK(xfer->xroot->bus);
2663	}
2664
2665	int
2666	usbd_xfer_is_stalled(struct usb_xfer *xfer)
2667	{
2668	return (xfer->endpoint->is_stalled);
2669	}
2670
2671	/------------------------------------------------------------------------
2672	* usbd_transfer_clear_stall
2673	*
2674	* This function is used to clear the stall flag outside the
2675	* callback. This function is NULL safe.
2676	------------------------------------------------------------------------/
2677	void
2678	usbd_transfer_clear_stall(struct usb_xfer *xfer)
2679	{
2680	if (xfer == NULL) {
2681	/* tearing down */
2682	return;
2683	}
2684	USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
2685
2686	/* avoid any races by locking the USB mutex */
2687	USB_BUS_LOCK(xfer->xroot->bus);
2688
2689	xfer->flags.stall_pipe = 0;
2690
2691	USB_BUS_UNLOCK(xfer->xroot->bus);
2692	}
2693
2694	/------------------------------------------------------------------------
2695	* usbd_pipe_start
2696	*
2697	* This function is used to add an USB transfer to the pipe transfer list.
2698	------------------------------------------------------------------------/
2699	void
2700	usbd_pipe_start(struct usb_xfer_queue *pq)
2701	{
2702	struct usb_endpoint *ep;
2703	struct usb_xfer *xfer;
2704	uint8_t type;
2705
2706	xfer = pq->curr;
2707	ep = xfer->endpoint;
2708
2709	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
2710
2711	/*
2712	* If the endpoint is already stalled we do nothing !
2713	*/
2714	if (ep->is_stalled) {
2715	return;
2716	}
2717	/*
2718	* Check if we are supposed to stall the endpoint:
2719	*/
2720	if (xfer->flags.stall_pipe) {
2721	struct usb_device *udev;
2722	struct usb_xfer_root *info;
2723
2724	/* clear stall command */
2725	xfer->flags.stall_pipe = 0;
2726
2727	/* get pointer to USB device */
2728	info = xfer->xroot;
2729	udev = info->udev;
2730
2731	/*
2732	* Only stall BULK and INTERRUPT endpoints.
2733	*/
2734	type = (ep->edesc->bmAttributes & UE_XFERTYPE);
2735	if ((type == UE_BULK) \|\|
2736	(type == UE_INTERRUPT)) {
2737	uint8_t did_stall;
2738
2739	did_stall = 1;
2740
2741	if (udev->flags.usb_mode == USB_MODE_DEVICE) {
2742	(udev->bus->methods->set_stall) (
2743	udev, ep, &did_stall);
2744	} else if (udev->ctrl_xfer[1]) {
2745	info = udev->ctrl_xfer[1]->xroot;
2746	usb_proc_msignal(
2747	USB_BUS_CS_PROC(info->bus),
2748	&udev->cs_msg[0], &udev->cs_msg[1]);
2749	} else {
2750	/* should not happen */
2751	DPRINTFN(0, "No stall handler\n");
2752	}
2753	/*
2754	* Check if we should stall. Some USB hardware
2755	* handles set- and clear-stall in hardware.
2756	*/
2757	if (did_stall) {
2758	/*
2759	* The transfer will be continued when
2760	* the clear-stall control endpoint
2761	* message is received.
2762	*/
2763	ep->is_stalled = 1;
2764	return;
2765	}
2766	} else if (type == UE_ISOCHRONOUS) {
2767
2768	/*
2769	* Make sure any FIFO overflow or other FIFO
2770	* error conditions go away by resetting the
2771	* endpoint FIFO through the clear stall
2772	* method.
2773	*/
2774	if (udev->flags.usb_mode == USB_MODE_DEVICE) {
2775	(udev->bus->methods->clear_stall) (udev, ep);
2776	}
2777	}
2778	}
2779	/* Set or clear stall complete - special case */
2780	if (xfer->nframes == 0) {
2781	/* we are complete */
2782	xfer->aframes = 0;
2783	usbd_transfer_done(xfer, 0);
2784	return;
2785	}
2786	/*
2787	* Handled cases:
2788	*
2789	* 1) Start the first transfer queued.
2790	*
2791	* 2) Re-start the current USB transfer.
2792	*/
2793	/*
2794	* Check if there should be any
2795	* pre transfer start delay:
2796	*/
2797	if (xfer->interval > 0) {
2798	type = (ep->edesc->bmAttributes & UE_XFERTYPE);
2799	if ((type == UE_BULK) \|\|
2800	(type == UE_CONTROL)) {
2801	usbd_transfer_timeout_ms(xfer,
2802	&usbd_transfer_start_cb,
2803	xfer->interval);
2804	return;
2805	}
2806	}
2807	DPRINTF("start\n");
2808
2809	#if USB_HAVE_PF
2810	usbpf_xfertap(xfer, USBPF_XFERTAP_SUBMIT);
2811	#endif
2812	/* the transfer can now be cancelled */
2813	xfer->flags_int.can_cancel_immed = 1;
2814
2815	/* start USB transfer, if no error */
2816	if (xfer->error == 0)
2817	(ep->methods->start) (xfer);
2818
2819	/* check for transfer error */
2820	if (xfer->error) {
2821	/* some error has happened */
2822	usbd_transfer_done(xfer, 0);
2823	}
2824	}
2825
2826	/------------------------------------------------------------------------
2827	* usbd_transfer_timeout_ms
2828	*
2829	* This function is used to setup a timeout on the given USB
2830	* transfer. If the timeout has been deferred the callback given by
2831	* "cb" will get called after "ms" milliseconds.
2832	------------------------------------------------------------------------/
2833	void
2834	usbd_transfer_timeout_ms(struct usb_xfer *xfer,
2835	void (cb) (void arg), usb_timeout_t ms)
2836	{
2837	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
2838
2839	/* defer delay */
2840	usb_callout_reset(&xfer->timeout_handle,
2841	USB_MS_TO_TICKS(ms) + USB_CALLOUT_ZERO_TICKS, cb, xfer);
2842	}
2843
2844	/------------------------------------------------------------------------
2845	* usbd_callback_wrapper_sub
2846	*
2847	* - This function will update variables in an USB transfer after
2848	* that the USB transfer is complete.
2849	*
2850	* - This function is used to start the next USB transfer on the
2851	* ep transfer queue, if any.
2852	*
2853	* NOTE: In some special cases the USB transfer will not be removed from
2854	* the pipe queue, but remain first. To enforce USB transfer removal call
2855	* this function passing the error code "USB_ERR_CANCELLED".
2856	*
2857	* Return values:
2858	* 0: Success.
2859	* Else: The callback has been deferred.
2860	------------------------------------------------------------------------/
2861	static uint8_t
2862	usbd_callback_wrapper_sub(struct usb_xfer *xfer)
2863	{
2864	struct usb_endpoint *ep;
2865	struct usb_bus *bus;
2866	usb_frcount_t x;
2867
2868	bus = xfer->xroot->bus;
2869
2870	if ((!xfer->flags_int.open) &&
2871	(!xfer->flags_int.did_close)) {
2872	DPRINTF("close\n");
2873	USB_BUS_LOCK(bus);
2874	(xfer->endpoint->methods->close) (xfer);
2875	USB_BUS_UNLOCK(bus);
2876	/* only close once */
2877	xfer->flags_int.did_close = 1;
2878	return (1); /* wait for new callback */
2879	}
2880	/*
2881	* If we have a non-hardware induced error we
2882	* need to do the DMA delay!
2883	*/
2884	if (xfer->error != 0 && !xfer->flags_int.did_dma_delay &&
2885	(xfer->error == USB_ERR_CANCELLED \|\|
2886	xfer->error == USB_ERR_TIMEOUT \|\|
2887	bus->methods->start_dma_delay != NULL)) {
2888
2889	usb_timeout_t temp;
2890
2891	/* only delay once */
2892	xfer->flags_int.did_dma_delay = 1;
2893
2894	/* we can not cancel this delay */
2895	xfer->flags_int.can_cancel_immed = 0;
2896
2897	temp = usbd_get_dma_delay(xfer->xroot->udev);
2898
2899	DPRINTFN(3, "DMA delay, %u ms, "
2900	"on %p\n", temp, xfer);
2901
2902	if (temp != 0) {
2903	USB_BUS_LOCK(bus);
2904	/*
2905	* Some hardware solutions have dedicated
2906	* events when it is safe to free DMA'ed
2907	* memory. For the other hardware platforms we
2908	* use a static delay.
2909	*/
2910	if (bus->methods->start_dma_delay != NULL) {
2911	(bus->methods->start_dma_delay) (xfer);
2912	} else {
2913	usbd_transfer_timeout_ms(xfer,
2914	(void ()(void ))&usb_dma_delay_done_cb,
2915	temp);
2916	}
2917	USB_BUS_UNLOCK(bus);
2918	return (1); /* wait for new callback */
2919	}
2920	}
2921	/* check actual number of frames */
2922	if (xfer->aframes > xfer->nframes) {
2923	if (xfer->error == 0) {
2924	panic("%s: actual number of frames, %d, is "
2925	"greater than initial number of frames, %d\n",
2926	__FUNCTION__, xfer->aframes, xfer->nframes);
2927	} else {
2928	/* just set some valid value */
2929	xfer->aframes = xfer->nframes;
2930	}
2931	}
2932	/* compute actual length */
2933	xfer->actlen = 0;
2934
2935	for (x = 0; x != xfer->aframes; x++) {
2936	xfer->actlen += xfer->frlengths[x];
2937	}
2938
2939	/*
2940	* Frames that were not transferred get zero actual length in
2941	* case the USB device driver does not check the actual number
2942	* of frames transferred, "xfer->aframes":
2943	*/
2944	for (; x < xfer->nframes; x++) {
2945	usbd_xfer_set_frame_len(xfer, x, 0);
2946	}
2947
2948	/* check actual length */
2949	if (xfer->actlen > xfer->sumlen) {
2950	if (xfer->error == 0) {
2951	panic("%s: actual length, %d, is greater than "
2952	"initial length, %d\n",
2953	__FUNCTION__, xfer->actlen, xfer->sumlen);
2954	} else {
2955	/* just set some valid value */
2956	xfer->actlen = xfer->sumlen;
2957	}
2958	}
2959	DPRINTFN(1, "xfer=%p endpoint=%p sts=%d alen=%d, slen=%d, afrm=%d, nfrm=%d\n",
2960	xfer, xfer->endpoint, xfer->error, xfer->actlen, xfer->sumlen,
2961	xfer->aframes, xfer->nframes);
2962
2963	if (xfer->error) {
2964	/* end of control transfer, if any */
2965	xfer->flags_int.control_act = 0;
2966
2967	#if USB_HAVE_TT_SUPPORT
2968	switch (xfer->error) {
2969	case USB_ERR_NORMAL_COMPLETION:
2970	case USB_ERR_SHORT_XFER:
2971	case USB_ERR_STALLED:
2972	case USB_ERR_CANCELLED:
2973	/* nothing to do */
2974	break;
2975	default:
2976	/* try to reset the TT, if any */
2977	USB_BUS_LOCK(bus);
2978	uhub_tt_buffer_reset_async_locked(xfer->xroot->udev, xfer->endpoint);
2979	USB_BUS_UNLOCK(bus);
2980	break;
2981	}
2982	#endif
2983	/* check if we should block the execution queue */
2984	if ((xfer->error != USB_ERR_CANCELLED) &&
2985	(xfer->flags.pipe_bof)) {
2986	DPRINTFN(2, "xfer=%p: Block On Failure "
2987	"on endpoint=%p\n", xfer, xfer->endpoint);
2988	goto done;
2989	}
2990	} else {
2991	/* check for short transfers */
2992	if (xfer->actlen < xfer->sumlen) {
2993
2994	/* end of control transfer, if any */
2995	xfer->flags_int.control_act = 0;
2996
2997	if (!xfer->flags_int.short_xfer_ok) {
2998	xfer->error = USB_ERR_SHORT_XFER;
2999	if (xfer->flags.pipe_bof) {
3000	DPRINTFN(2, "xfer=%p: Block On Failure on "
3001	"Short Transfer on endpoint %p.\n",
3002	xfer, xfer->endpoint);
3003	goto done;
3004	}
3005	}
3006	} else {
3007	/*
3008	* Check if we are in the middle of a
3009	* control transfer:
3010	*/
3011	if (xfer->flags_int.control_act) {
3012	DPRINTFN(5, "xfer=%p: Control transfer "
3013	"active on endpoint=%p\n", xfer, xfer->endpoint);
3014	goto done;
3015	}
3016	}
3017	}
3018
3019	ep = xfer->endpoint;
3020
3021	/*
3022	* If the current USB transfer is completing we need to start the
3023	* next one:
3024	*/
3025	USB_BUS_LOCK(bus);
3026	if (ep->endpoint_q[xfer->stream_id].curr == xfer) {
3027	usb_command_wrapper(&ep->endpoint_q[xfer->stream_id], NULL);
3028
3029	if (ep->endpoint_q[xfer->stream_id].curr != NULL \|\|
3030	TAILQ_FIRST(&ep->endpoint_q[xfer->stream_id].head) != NULL) {
3031	/* there is another USB transfer waiting */
3032	} else {
3033	/* this is the last USB transfer */
3034	/* clear isochronous sync flag */
3035	xfer->endpoint->is_synced = 0;
3036	}
3037	}
3038	USB_BUS_UNLOCK(bus);
3039	done:
3040	return (0);
3041	}
3042
3043	/------------------------------------------------------------------------
3044	* usb_command_wrapper
3045	*
3046	* This function is used to execute commands non-recursivly on an USB
3047	* transfer.
3048	------------------------------------------------------------------------/
3049	void
3050	usb_command_wrapper(struct usb_xfer_queue pq, struct usb_xfer xfer)
3051	{
3052	if (xfer) {
3053	/*
3054	* If the transfer is not already processing,
3055	* queue it!
3056	*/
3057	if (pq->curr != xfer) {
3058	usbd_transfer_enqueue(pq, xfer);
3059	if (pq->curr != NULL) {
3060	/* something is already processing */
3061	DPRINTFN(6, "busy %p\n", pq->curr);
3062	return;
3063	}
3064	}
3065	} else {
3066	/* Get next element in queue */
3067	pq->curr = NULL;
3068	}
3069
3070	if (!pq->recurse_1) {
3071
3072	/* clear third recurse flag */
3073	pq->recurse_3 = 0;
3074
3075	do {
3076	/* set two first recurse flags */
3077	pq->recurse_1 = 1;
3078	pq->recurse_2 = 1;
3079
3080	if (pq->curr == NULL) {
3081	xfer = TAILQ_FIRST(&pq->head);
3082	if (xfer) {
3083	TAILQ_REMOVE(&pq->head, xfer,
3084	wait_entry);
3085	xfer->wait_queue = NULL;
3086	pq->curr = xfer;
3087	} else {
3088	break;
3089	}
3090	}
3091	DPRINTFN(6, "cb %p (enter)\n", pq->curr);
3092	(pq->command) (pq);
3093	DPRINTFN(6, "cb %p (leave)\n", pq->curr);
3094
3095	/*
3096	* Set third recurse flag to indicate
3097	* recursion happened:
3098	*/
3099	pq->recurse_3 = 1;
3100
3101	} while (!pq->recurse_2);
3102
3103	/* clear first recurse flag */
3104	pq->recurse_1 = 0;
3105
3106	} else {
3107	/* clear second recurse flag */
3108	pq->recurse_2 = 0;
3109	}
3110	}
3111
3112	/------------------------------------------------------------------------
3113	* usbd_ctrl_transfer_setup
3114	*
3115	* This function is used to setup the default USB control endpoint
3116	* transfer.
3117	------------------------------------------------------------------------/
3118	void
3119	usbd_ctrl_transfer_setup(struct usb_device *udev)
3120	{
3121	struct usb_xfer *xfer;
3122	uint8_t no_resetup;
3123	uint8_t iface_index;
3124
3125	/* check for root HUB */
3126	if (udev->parent_hub == NULL)
3127	return;
3128	repeat:
3129
3130	xfer = udev->ctrl_xfer[0];
3131	if (xfer) {
3132	USB_XFER_LOCK(xfer);
3133	no_resetup =
3134	((xfer->address == udev->address) &&
3135	(udev->ctrl_ep_desc.wMaxPacketSize[0] ==
3136	udev->ddesc.bMaxPacketSize));
3137	if (udev->flags.usb_mode == USB_MODE_DEVICE) {
3138	if (no_resetup) {
3139	/*
3140	* NOTE: checking "xfer->address" and
3141	* starting the USB transfer must be
3142	* atomic!
3143	*/
3144	usbd_transfer_start(xfer);
3145	}
3146	}
3147	USB_XFER_UNLOCK(xfer);
3148	} else {
3149	no_resetup = 0;
3150	}
3151
3152	if (no_resetup) {
3153	/*
3154	* All parameters are exactly the same like before.
3155	* Just return.
3156	*/
3157	return;
3158	}
3159	/*
3160	* Update wMaxPacketSize for the default control endpoint:
3161	*/
3162	udev->ctrl_ep_desc.wMaxPacketSize[0] =
3163	udev->ddesc.bMaxPacketSize;
3164
3165	/*
3166	* Unsetup any existing USB transfer:
3167	*/
3168	usbd_transfer_unsetup(udev->ctrl_xfer, USB_CTRL_XFER_MAX);
3169
3170	/*
3171	* Reset clear stall error counter.
3172	*/
3173	udev->clear_stall_errors = 0;
3174
3175	/*
3176	* Try to setup a new USB transfer for the
3177	* default control endpoint:
3178	*/
3179	iface_index = 0;
3180	if (usbd_transfer_setup(udev, &iface_index,
3181	udev->ctrl_xfer, usb_control_ep_cfg, USB_CTRL_XFER_MAX, NULL,
3182	&udev->device_mtx)) {
3183	DPRINTFN(0, "could not setup default "
3184	"USB transfer\n");
3185	} else {
3186	goto repeat;
3187	}
3188	}
3189
3190	/------------------------------------------------------------------------
3191	* usbd_clear_data_toggle - factored out code
3192	*
3193	* NOTE: the intention of this function is not to reset the hardware
3194	* data toggle.
3195	------------------------------------------------------------------------/
3196	void
3197	usbd_clear_stall_locked(struct usb_device udev, struct usb_endpoint ep)
3198	{
3199	USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3200
3201	/* check that we have a valid case */
3202	if (udev->flags.usb_mode == USB_MODE_HOST &&
3203	udev->parent_hub != NULL &&
3204	udev->bus->methods->clear_stall != NULL &&
3205	ep->methods != NULL) {
3206	(udev->bus->methods->clear_stall) (udev, ep);
3207	}
3208	}
3209
3210	/------------------------------------------------------------------------
3211	* usbd_clear_data_toggle - factored out code
3212	*
3213	* NOTE: the intention of this function is not to reset the hardware
3214	* data toggle on the USB device side.
3215	------------------------------------------------------------------------/
3216	void
3217	usbd_clear_data_toggle(struct usb_device udev, struct usb_endpoint ep)
3218	{
3219	DPRINTFN(5, "udev=%p endpoint=%p\n", udev, ep);
3220
3221	USB_BUS_LOCK(udev->bus);
3222	ep->toggle_next = 0;
3223	/* some hardware needs a callback to clear the data toggle */
3224	usbd_clear_stall_locked(udev, ep);
3225	USB_BUS_UNLOCK(udev->bus);
3226	}
3227
3228	/------------------------------------------------------------------------
3229	* usbd_clear_stall_callback - factored out clear stall callback
3230	*
3231	* Input parameters:
3232	* xfer1: Clear Stall Control Transfer
3233	* xfer2: Stalled USB Transfer
3234	*
3235	* This function is NULL safe.
3236	*
3237	* Return values:
3238	* 0: In progress
3239	* Else: Finished
3240	*
3241	* Clear stall config example:
3242	*
3243	* static const struct usb_config my_clearstall = {
3244	* .type = UE_CONTROL,
3245	* .endpoint = 0,
3246	* .direction = UE_DIR_ANY,
3247	* .interval = 50, //50 milliseconds
3248	* .bufsize = sizeof(struct usb_device_request),
3249	* .timeout = 1000, //1.000 seconds
3250	* .callback = &my_clear_stall_callback, // **
3251	* .usb_mode = USB_MODE_HOST,
3252	* };
3253	*
3254	* ** "my_clear_stall_callback" calls "usbd_clear_stall_callback"
3255	* passing the correct parameters.
3256	------------------------------------------------------------------------/
3257	uint8_t
3258	usbd_clear_stall_callback(struct usb_xfer *xfer1,
3259	struct usb_xfer *xfer2)
3260	{
3261	struct usb_device_request req;
3262
3263	if (xfer2 == NULL) {
3264	/* looks like we are tearing down */
3265	DPRINTF("NULL input parameter\n");
3266	return (0);
3267	}
3268	USB_XFER_LOCK_ASSERT(xfer1, MA_OWNED);
3269	USB_XFER_LOCK_ASSERT(xfer2, MA_OWNED);
3270
3271	switch (USB_GET_STATE(xfer1)) {
3272	case USB_ST_SETUP:
3273
3274	/*
3275	* pre-clear the data toggle to DATA0 ("umass.c" and
3276	* "ata-usb.c" depends on this)
3277	*/
3278
3279	usbd_clear_data_toggle(xfer2->xroot->udev, xfer2->endpoint);
3280
3281	/* setup a clear-stall packet */
3282
3283	req.bmRequestType = UT_WRITE_ENDPOINT;
3284	req.bRequest = UR_CLEAR_FEATURE;
3285	USETW(req.wValue, UF_ENDPOINT_HALT);
3286	req.wIndex[0] = xfer2->endpoint->edesc->bEndpointAddress;
3287	req.wIndex[1] = 0;
3288	USETW(req.wLength, 0);
3289
3290	/*
3291	* "usbd_transfer_setup_sub()" will ensure that
3292	* we have sufficient room in the buffer for
3293	* the request structure!
3294	*/
3295
3296	/* copy in the transfer */
3297
3298	usbd_copy_in(xfer1->frbuffers, 0, &req, sizeof(req));
3299
3300	/* set length */
3301	xfer1->frlengths[0] = sizeof(req);
3302	xfer1->nframes = 1;
3303
3304	usbd_transfer_submit(xfer1);
3305	return (0);
3306
3307	case USB_ST_TRANSFERRED:
3308	break;
3309
3310	default: /* Error */
3311	if (xfer1->error == USB_ERR_CANCELLED) {
3312	return (0);
3313	}
3314	break;
3315	}
3316	return (1); /* Clear Stall Finished */
3317	}
3318
3319	/------------------------------------------------------------------------
3320	* usbd_transfer_poll
3321	*
3322	* The following function gets called from the USB keyboard driver and
3323	* UMASS when the system has paniced.
3324	*
3325	* NOTE: It is currently not possible to resume normal operation on
3326	* the USB controller which has been polled, due to clearing of the
3327	* "up_dsleep" and "up_msleep" flags.
3328	------------------------------------------------------------------------/
3329	void
3330	usbd_transfer_poll(struct usb_xfer **ppxfer, uint16_t max)
3331	{
3332	struct usb_xfer *xfer;
3333	struct usb_xfer_root *xroot;
3334	struct usb_device *udev;
3335	struct usb_proc_msg *pm;
3336	uint16_t n;
3337	uint16_t drop_bus;
3338	uint16_t drop_xfer;
3339
3340	for (n = 0; n != max; n++) {
3341	/* Extra checks to avoid panic */
3342	xfer = ppxfer[n];
3343	if (xfer == NULL)
3344	continue; /* no USB transfer */
3345	xroot = xfer->xroot;
3346	if (xroot == NULL)
3347	continue; /* no USB root */
3348	udev = xroot->udev;
3349	if (udev == NULL)
3350	continue; /* no USB device */
3351	if (udev->bus == NULL)
3352	continue; /* no BUS structure */
3353	if (udev->bus->methods == NULL)
3354	continue; /* no BUS methods */
3355	if (udev->bus->methods->xfer_poll == NULL)
3356	continue; /* no poll method */
3357
3358	/* make sure that the BUS mutex is not locked */
3359	drop_bus = 0;
3360	while (mtx_owned(&xroot->udev->bus->bus_mtx) && !SCHEDULER_STOPPED()) {
3361	mtx_unlock(&xroot->udev->bus->bus_mtx);
3362	drop_bus++;
3363	}
3364
3365	/* make sure that the transfer mutex is not locked */
3366	drop_xfer = 0;
3367	while (mtx_owned(xroot->xfer_mtx) && !SCHEDULER_STOPPED()) {
3368	mtx_unlock(xroot->xfer_mtx);
3369	drop_xfer++;
3370	}
3371
3372	#if USB_HAVE_PER_BUS_PROCESS
3373	/* Make sure cv_signal() and cv_broadcast() is not called */
3374	USB_BUS_CONTROL_XFER_PROC(udev->bus)->up_msleep = 0;
3375	USB_BUS_EXPLORE_PROC(udev->bus)->up_msleep = 0;
3376	USB_BUS_GIANT_PROC(udev->bus)->up_msleep = 0;
3377	USB_BUS_NON_GIANT_ISOC_PROC(udev->bus)->up_msleep = 0;
3378	USB_BUS_NON_GIANT_BULK_PROC(udev->bus)->up_msleep = 0;
3379	#endif
3380
3381	/* poll USB hardware */
3382	(udev->bus->methods->xfer_poll) (udev->bus);
3383
3384	USB_BUS_LOCK(xroot->bus);
3385
3386	/* check for clear stall */
3387	if (udev->ctrl_xfer[1] != NULL) {
3388
3389	/* poll clear stall start */
3390	pm = &udev->cs_msg[0].hdr;
3391	(pm->pm_callback) (pm);
3392	/* poll clear stall done thread */
3393	pm = &udev->ctrl_xfer[1]->
3394	xroot->done_m[0].hdr;
3395	(pm->pm_callback) (pm);
3396	}
3397
3398	/* poll done thread */
3399	pm = &xroot->done_m[0].hdr;
3400	(pm->pm_callback) (pm);
3401
3402	USB_BUS_UNLOCK(xroot->bus);
3403
3404	/* restore transfer mutex */
3405	while (drop_xfer--)
3406	mtx_lock(xroot->xfer_mtx);
3407
3408	/* restore BUS mutex */
3409	while (drop_bus--)
3410	mtx_lock(&xroot->udev->bus->bus_mtx);
3411	}
3412	}
3413
3414	static void
3415	usbd_get_std_packet_size(struct usb_std_packet_size *ptr,
3416	uint8_t type, enum usb_dev_speed speed)
3417	{
3418	static const uint16_t intr_range_max[USB_SPEED_MAX] = {
3419	[USB_SPEED_LOW] = 8,
3420	[USB_SPEED_FULL] = 64,
3421	[USB_SPEED_HIGH] = 1024,
3422	[USB_SPEED_VARIABLE] = 1024,
3423	[USB_SPEED_SUPER] = 1024,
3424	};
3425
3426	static const uint16_t isoc_range_max[USB_SPEED_MAX] = {
3427	[USB_SPEED_LOW] = 0, /* invalid */
3428	[USB_SPEED_FULL] = 1023,
3429	[USB_SPEED_HIGH] = 1024,
3430	[USB_SPEED_VARIABLE] = 3584,
3431	[USB_SPEED_SUPER] = 1024,
3432	};
3433
3434	static const uint16_t control_min[USB_SPEED_MAX] = {
3435	[USB_SPEED_LOW] = 8,
3436	[USB_SPEED_FULL] = 8,
3437	[USB_SPEED_HIGH] = 64,
3438	[USB_SPEED_VARIABLE] = 512,
3439	[USB_SPEED_SUPER] = 512,
3440	};
3441
3442	static const uint16_t bulk_min[USB_SPEED_MAX] = {
3443	[USB_SPEED_LOW] = 8,
3444	[USB_SPEED_FULL] = 8,
3445	[USB_SPEED_HIGH] = 512,
3446	[USB_SPEED_VARIABLE] = 512,
3447	[USB_SPEED_SUPER] = 1024,
3448	};
3449
3450	uint16_t temp;
3451
3452	memset(ptr, 0, sizeof(*ptr));
3453
3454	switch (type) {
3455	case UE_INTERRUPT:
3456	ptr->range.max = intr_range_max[speed];
3457	break;
3458	case UE_ISOCHRONOUS:
3459	ptr->range.max = isoc_range_max[speed];
3460	break;
3461	default:
3462	if (type == UE_BULK)
3463	temp = bulk_min[speed];
3464	else /* UE_CONTROL */
3465	temp = control_min[speed];
3466
3467	/* default is fixed */
3468	ptr->fixed[0] = temp;
3469	ptr->fixed[1] = temp;
3470	ptr->fixed[2] = temp;
3471	ptr->fixed[3] = temp;
3472
3473	if (speed == USB_SPEED_FULL) {
3474	/* multiple sizes */
3475	ptr->fixed[1] = 16;
3476	ptr->fixed[2] = 32;
3477	ptr->fixed[3] = 64;
3478	}
3479	if ((speed == USB_SPEED_VARIABLE) &&
3480	(type == UE_BULK)) {
3481	/* multiple sizes */
3482	ptr->fixed[2] = 1024;
3483	ptr->fixed[3] = 1536;
3484	}
3485	break;
3486	}
3487	}
3488
3489	void *
3490	usbd_xfer_softc(struct usb_xfer *xfer)
3491	{
3492	return (xfer->priv_sc);
3493	}
3494
3495	void *
3496	usbd_xfer_get_priv(struct usb_xfer *xfer)
3497	{
3498	return (xfer->priv_fifo);
3499	}
3500
3501	void
3502	usbd_xfer_set_priv(struct usb_xfer xfer, void ptr)
3503	{
3504	xfer->priv_fifo = ptr;
3505	}
3506
3507	uint8_t
3508	usbd_xfer_state(struct usb_xfer *xfer)
3509	{
3510	return (xfer->usb_state);
3511	}
3512
3513	void
3514	usbd_xfer_set_flag(struct usb_xfer *xfer, int flag)
3515	{
3516	switch (flag) {
3517	case USB_FORCE_SHORT_XFER:
3518	xfer->flags.force_short_xfer = 1;
3519	break;
3520	case USB_SHORT_XFER_OK:
3521	xfer->flags.short_xfer_ok = 1;
3522	break;
3523	case USB_MULTI_SHORT_OK:
3524	xfer->flags.short_frames_ok = 1;
3525	break;
3526	case USB_MANUAL_STATUS:
3527	xfer->flags.manual_status = 1;
3528	break;
3529	}
3530	}
3531
3532	void
3533	usbd_xfer_clr_flag(struct usb_xfer *xfer, int flag)
3534	{
3535	switch (flag) {
3536	case USB_FORCE_SHORT_XFER:
3537	xfer->flags.force_short_xfer = 0;
3538	break;
3539	case USB_SHORT_XFER_OK:
3540	xfer->flags.short_xfer_ok = 0;
3541	break;
3542	case USB_MULTI_SHORT_OK:
3543	xfer->flags.short_frames_ok = 0;
3544	break;
3545	case USB_MANUAL_STATUS:
3546	xfer->flags.manual_status = 0;
3547	break;
3548	}
3549	}
3550
3551	/*
3552	* The following function returns in milliseconds when the isochronous
3553	* transfer was completed by the hardware. The returned value wraps
3554	* around 65536 milliseconds.
3555	*/
3556	uint16_t
3557	usbd_xfer_get_timestamp(struct usb_xfer *xfer)
3558	{
3559	return (xfer->isoc_time_complete);
3560	}
3561
3562	/*
3563	* The following function returns non-zero if the max packet size
3564	* field was clamped to a valid value. Else it returns zero.
3565	*/
3566	uint8_t
3567	usbd_xfer_maxp_was_clamped(struct usb_xfer *xfer)
3568	{
3569	return (xfer->flags_int.maxp_was_clamped);
3570	}

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: rtems-libbsd/freebsd/sys/dev/usb/usb_transfer.c @ 8e65e1b

Download in other formats: