Context Navigation

source: rtems-libbsd/freebsd/sys/kern/sys_socket.c @ 0913210

55-freebsd-126-freebsd-12

Last change on this file since 0913210 was 0913210, checked in by Sebastian Huber <sebastian.huber@…>, on 07/28/17 at 06:36:33
SOCKET(2): Add missing file handler
Property mode set to `100644`
File size: 25.6 KB

Line
1	#include <machine/rtems-bsd-kernel-space.h>
2
3	/*-
4	* Copyright (c) 1982, 1986, 1990, 1993
5	* The Regents of the University of California. All rights reserved.
6	*
7	* Redistribution and use in source and binary forms, with or without
8	* modification, are permitted provided that the following conditions
9	* are met:
10	* 1. Redistributions of source code must retain the above copyright
11	* notice, this list of conditions and the following disclaimer.
12	* 2. Redistributions in binary form must reproduce the above copyright
13	* notice, this list of conditions and the following disclaimer in the
14	* documentation and/or other materials provided with the distribution.
15	* 3. Neither the name of the University nor the names of its contributors
16	* may be used to endorse or promote products derived from this software
17	* without specific prior written permission.
18	*
19	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
20	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
23	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29	* SUCH DAMAGE.
30	*
31	* @(#)sys_socket.c 8.1 (Berkeley) 6/10/93
32	*/
33
34	#include <sys/cdefs.h>
35	__FBSDID("$FreeBSD$");
36
37	#include <sys/param.h>
38	#include <sys/systm.h>
39	#include <sys/aio.h>
40	#include <sys/domain.h>
41	#include <sys/file.h>
42	#include <sys/filedesc.h>
43	#include <sys/kernel.h>
44	#include <sys/kthread.h>
45	#include <sys/malloc.h>
46	#include <sys/proc.h>
47	#include <sys/protosw.h>
48	#include <sys/sigio.h>
49	#include <sys/signal.h>
50	#include <sys/signalvar.h>
51	#include <sys/socket.h>
52	#include <sys/socketvar.h>
53	#include <sys/filio.h> /* XXX */
54	#include <sys/sockio.h>
55	#include <sys/stat.h>
56	#include <sys/sysctl.h>
57	#include <sys/sysproto.h>
58	#include <sys/taskqueue.h>
59	#include <sys/uio.h>
60	#include <sys/ucred.h>
61	#include <sys/un.h>
62	#include <sys/unpcb.h>
63	#include <sys/user.h>
64
65	#include <net/if.h>
66	#include <net/if_var.h>
67	#include <net/route.h>
68	#include <net/vnet.h>
69
70	#include <netinet/in.h>
71	#include <netinet/in_pcb.h>
72
73	#include <security/mac/mac_framework.h>
74
75	#ifndef __rtems__
76	#include <vm/vm.h>
77	#include <vm/pmap.h>
78	#include <vm/vm_extern.h>
79	#include <vm/vm_map.h>
80
81	static SYSCTL_NODE(_kern_ipc, OID_AUTO, aio, CTLFLAG_RD, NULL,
82	"socket AIO stats");
83
84	static int empty_results;
85	SYSCTL_INT(_kern_ipc_aio, OID_AUTO, empty_results, CTLFLAG_RD, &empty_results,
86	0, "socket operation returned EAGAIN");
87
88	static int empty_retries;
89	SYSCTL_INT(_kern_ipc_aio, OID_AUTO, empty_retries, CTLFLAG_RD, &empty_retries,
90	0, "socket operation retries");
91
92	static fo_rdwr_t soo_read;
93	static fo_rdwr_t soo_write;
94	static fo_ioctl_t soo_ioctl;
95	static fo_poll_t soo_poll;
96	extern fo_kqfilter_t soo_kqfilter;
97	static fo_stat_t soo_stat;
98	static fo_close_t soo_close;
99	static fo_fill_kinfo_t soo_fill_kinfo;
100	static fo_aio_queue_t soo_aio_queue;
101
102	static void soo_aio_cancel(struct kaiocb *job);
103
104	struct fileops socketops = {
105	.fo_read = soo_read,
106	.fo_write = soo_write,
107	.fo_truncate = invfo_truncate,
108	.fo_ioctl = soo_ioctl,
109	.fo_poll = soo_poll,
110	.fo_kqfilter = soo_kqfilter,
111	.fo_stat = soo_stat,
112	.fo_close = soo_close,
113	.fo_chmod = invfo_chmod,
114	.fo_chown = invfo_chown,
115	.fo_sendfile = invfo_sendfile,
116	.fo_fill_kinfo = soo_fill_kinfo,
117	.fo_aio_queue = soo_aio_queue,
118	.fo_flags = DFLAG_PASSABLE
119	};
120	#endif /* __rtems__ */
121
122	#ifdef __rtems__
123	int
124	#else /* __rtems__ */
125	static int
126	#endif /* __rtems__ */
127	soo_read(struct file fp, struct uio uio, struct ucred *active_cred,
128	int flags, struct thread *td)
129	{
130	struct socket *so = fp->f_data;
131	int error;
132
133	#ifdef MAC
134	error = mac_socket_check_receive(active_cred, so);
135	if (error)
136	return (error);
137	#endif
138	error = soreceive(so, 0, uio, 0, 0, 0);
139	return (error);
140	}
141	#ifdef __rtems__
142	static ssize_t
143	rtems_bsd_soo_read(rtems_libio_t iop, void buffer, size_t count)
144	{
145	struct thread *td = rtems_bsd_get_curthread_or_null();
146	struct file *fp = rtems_bsd_iop_to_fp(iop);
147	struct iovec iov = {
148	.iov_base = buffer,
149	.iov_len = count
150	};
151	struct uio auio = {
152	.uio_iov = &iov,
153	.uio_iovcnt = 1,
154	.uio_offset = 0,
155	.uio_resid = count,
156	.uio_segflg = UIO_USERSPACE,
157	.uio_rw = UIO_READ,
158	.uio_td = td
159	};
160	int error;
161
162	if (td != NULL) {
163	error = soo_read(fp, &auio, NULL, 0, NULL);
164	} else {
165	error = ENOMEM;
166	}
167
168	if (error == 0) {
169	return (count - auio.uio_resid);
170	} else {
171	rtems_set_errno_and_return_minus_one(error);
172	}
173	}
174
175	static ssize_t
176	rtems_bsd_soo_readv(rtems_libio_t iop, const struct iovec iov,
177	int iovcnt, ssize_t total)
178	{
179	struct thread *td = rtems_bsd_get_curthread_or_null();
180	struct file *fp = rtems_bsd_iop_to_fp(iop);
181	struct uio auio = {
182	.uio_iov = __DECONST(struct iovec *, iov),
183	.uio_iovcnt = iovcnt,
184	.uio_offset = 0,
185	.uio_resid = total,
186	.uio_segflg = UIO_USERSPACE,
187	.uio_rw = UIO_READ,
188	.uio_td = td
189	};
190	int error;
191
192	if (td != NULL) {
193	error = soo_read(fp, &auio, NULL, 0, NULL);
194	} else {
195	error = ENOMEM;
196	}
197
198	if (error == 0) {
199	return (total - auio.uio_resid);
200	} else {
201	rtems_set_errno_and_return_minus_one(error);
202	}
203	}
204	#endif /* __rtems__ */
205
206	#ifdef __rtems__
207	int
208	#else /* __rtems__ */
209	static int
210	#endif /* __rtems__ */
211	soo_write(struct file fp, struct uio uio, struct ucred *active_cred,
212	int flags, struct thread *td)
213	{
214	struct socket *so = fp->f_data;
215	int error;
216
217	#ifdef MAC
218	error = mac_socket_check_send(active_cred, so);
219	if (error)
220	return (error);
221	#endif
222	error = sosend(so, 0, uio, 0, 0, 0, uio->uio_td);
223	if (error == EPIPE && (so->so_options & SO_NOSIGPIPE) == 0) {
224	#ifndef __rtems__
225	PROC_LOCK(uio->uio_td->td_proc);
226	tdsignal(uio->uio_td, SIGPIPE);
227	PROC_UNLOCK(uio->uio_td->td_proc);
228	#else /* __rtems__ */
229	/* FIXME: Determine if we really want to use signals */
230	#endif /* __rtems__ */
231	}
232	return (error);
233	}
234	#ifdef __rtems__
235	static ssize_t
236	rtems_bsd_soo_write(rtems_libio_t iop, const void buffer, size_t count)
237	{
238	struct thread *td = rtems_bsd_get_curthread_or_null();
239	struct file *fp = rtems_bsd_iop_to_fp(iop);
240	struct iovec iov = {
241	.iov_base = __DECONST(void *, buffer),
242	.iov_len = count
243	};
244	struct uio auio = {
245	.uio_iov = &iov,
246	.uio_iovcnt = 1,
247	.uio_offset = 0,
248	.uio_resid = count,
249	.uio_segflg = UIO_USERSPACE,
250	.uio_rw = UIO_WRITE,
251	.uio_td = td
252	};
253	int error;
254
255	if (td != NULL) {
256	error = soo_write(fp, &auio, NULL, 0, NULL);
257	} else {
258	error = ENOMEM;
259	}
260
261	if (error == 0) {
262	return (count - auio.uio_resid);
263	} else {
264	rtems_set_errno_and_return_minus_one(error);
265	}
266	}
267
268	static ssize_t
269	rtems_bsd_soo_writev(rtems_libio_t iop, const struct iovec iov,
270	int iovcnt, ssize_t total)
271	{
272	struct thread *td = rtems_bsd_get_curthread_or_null();
273	struct file *fp = rtems_bsd_iop_to_fp(iop);
274	struct uio auio = {
275	.uio_iov = __DECONST(struct iovec *, iov),
276	.uio_iovcnt = iovcnt,
277	.uio_offset = 0,
278	.uio_resid = total,
279	.uio_segflg = UIO_USERSPACE,
280	.uio_rw = UIO_WRITE,
281	.uio_td = td
282	};
283	int error;
284
285	if (td != NULL) {
286	error = soo_write(fp, &auio, NULL, 0, NULL);
287	} else {
288	error = ENOMEM;
289	}
290
291	if (error == 0) {
292	return (total - auio.uio_resid);
293	} else {
294	rtems_set_errno_and_return_minus_one(error);
295	}
296	}
297	#endif /* __rtems__ */
298
299	#ifdef __rtems__
300	int
301	#else /* __rtems__ */
302	static int
303	#endif /* __rtems__ */
304	soo_ioctl(struct file fp, u_long cmd, void data, struct ucred *active_cred,
305	struct thread *td)
306	{
307	struct socket *so = fp->f_data;
308	int error = 0;
309
310	switch (cmd) {
311	case FIONBIO:
312	SOCK_LOCK(so);
313	if ((int )data)
314	so->so_state \|= SS_NBIO;
315	else
316	so->so_state &= ~SS_NBIO;
317	SOCK_UNLOCK(so);
318	break;
319
320	case FIOASYNC:
321	/*
322	* XXXRW: This code separately acquires SOCK_LOCK(so) and
323	* SOCKBUF_LOCK(&so->so_rcv) even though they are the same
324	* mutex to avoid introducing the assumption that they are
325	* the same.
326	*/
327	if ((int )data) {
328	SOCK_LOCK(so);
329	so->so_state \|= SS_ASYNC;
330	SOCK_UNLOCK(so);
331	SOCKBUF_LOCK(&so->so_rcv);
332	so->so_rcv.sb_flags \|= SB_ASYNC;
333	SOCKBUF_UNLOCK(&so->so_rcv);
334	SOCKBUF_LOCK(&so->so_snd);
335	so->so_snd.sb_flags \|= SB_ASYNC;
336	SOCKBUF_UNLOCK(&so->so_snd);
337	} else {
338	SOCK_LOCK(so);
339	so->so_state &= ~SS_ASYNC;
340	SOCK_UNLOCK(so);
341	SOCKBUF_LOCK(&so->so_rcv);
342	so->so_rcv.sb_flags &= ~SB_ASYNC;
343	SOCKBUF_UNLOCK(&so->so_rcv);
344	SOCKBUF_LOCK(&so->so_snd);
345	so->so_snd.sb_flags &= ~SB_ASYNC;
346	SOCKBUF_UNLOCK(&so->so_snd);
347	}
348	break;
349
350	case FIONREAD:
351	/* Unlocked read. */
352	(int )data = sbavail(&so->so_rcv);
353	break;
354
355	case FIONWRITE:
356	/* Unlocked read. */
357	(int )data = sbavail(&so->so_snd);
358	break;
359
360	case FIONSPACE:
361	/* Unlocked read. */
362	if ((so->so_snd.sb_hiwat < sbused(&so->so_snd)) \|\|
363	(so->so_snd.sb_mbmax < so->so_snd.sb_mbcnt))
364	(int )data = 0;
365	else
366	(int )data = sbspace(&so->so_snd);
367	break;
368
369	case FIOSETOWN:
370	error = fsetown((int )data, &so->so_sigio);
371	break;
372
373	case FIOGETOWN:
374	(int )data = fgetown(&so->so_sigio);
375	break;
376
377	case SIOCSPGRP:
378	error = fsetown(-((int )data), &so->so_sigio);
379	break;
380
381	case SIOCGPGRP:
382	(int )data = -fgetown(&so->so_sigio);
383	break;
384
385	case SIOCATMARK:
386	/* Unlocked read. */
387	(int )data = (so->so_rcv.sb_state & SBS_RCVATMARK) != 0;
388	break;
389	default:
390	/*
391	* Interface/routing/protocol specific ioctls: interface and
392	* routing ioctls should have a different entry since a
393	* socket is unnecessary.
394	*/
395	if (IOCGROUP(cmd) == 'i')
396	error = ifioctl(so, cmd, data, td);
397	else if (IOCGROUP(cmd) == 'r') {
398	CURVNET_SET(so->so_vnet);
399	error = rtioctl_fib(cmd, data, so->so_fibnum);
400	CURVNET_RESTORE();
401	} else {
402	CURVNET_SET(so->so_vnet);
403	error = ((*so->so_proto->pr_usrreqs->pru_control)
404	(so, cmd, data, 0, td));
405	CURVNET_RESTORE();
406	}
407	break;
408	}
409	return (error);
410	}
411	#ifdef __rtems__
412	static int
413	rtems_bsd_soo_ioctl(rtems_libio_t iop, ioctl_command_t request, void buffer)
414	{
415	struct thread *td = rtems_bsd_get_curthread_or_null();
416	struct file *fp = rtems_bsd_iop_to_fp(iop);
417	int error;
418
419	if (td != NULL) {
420	error = soo_ioctl(fp, request, buffer, NULL, td);
421	} else {
422	error = ENOMEM;
423	}
424
425	return rtems_bsd_error_to_status_and_errno(error);
426	}
427	#endif /* __rtems__ */
428
429	#ifdef __rtems__
430	int
431	#else /* __rtems__ */
432	static int
433	#endif /* __rtems__ */
434	soo_poll(struct file fp, int events, struct ucred active_cred,
435	struct thread *td)
436	{
437	struct socket *so = fp->f_data;
438	#ifdef MAC
439	int error;
440
441	error = mac_socket_check_poll(active_cred, so);
442	if (error)
443	return (error);
444	#endif
445	#ifndef __rtems__
446	return (sopoll(so, events, fp->f_cred, td));
447	#else /* __rtems__ */
448	return (sopoll(so, events, NULL, td));
449	#endif /* __rtems__ */
450	}
451	#ifdef __rtems__
452	static int
453	rtems_bsd_soo_poll(rtems_libio_t *iop, int events)
454	{
455	struct thread *td = rtems_bsd_get_curthread_or_null();
456	struct file *fp = rtems_bsd_iop_to_fp(iop);
457	int error;
458
459	if (td != NULL) {
460	error = soo_poll(fp, events, NULL, td);
461	} else {
462	error = ENOMEM;
463	}
464
465	return (error);
466	}
467	#endif /* __rtems__ */
468
469	#ifndef __rtems__
470	int
471	soo_stat(struct file fp, struct stat ub, struct ucred *active_cred,
472	struct thread *td)
473	{
474	struct socket *so = fp->f_data;
475	#else /* __rtems__ */
476	static int
477	soo_stat(struct socket so, struct stat ub)
478	{
479	#endif /* __rtems__ */
480	struct sockbuf *sb;
481	#ifdef MAC
482	int error;
483	#endif
484
485	#ifndef __rtems__
486	bzero((caddr_t)ub, sizeof (*ub));
487	#endif /* __rtems__ */
488	ub->st_mode = S_IFSOCK;
489	#ifdef MAC
490	error = mac_socket_check_stat(active_cred, so);
491	if (error)
492	return (error);
493	#endif
494	/*
495	* If SBS_CANTRCVMORE is set, but there's still data left in the
496	* receive buffer, the socket is still readable.
497	*/
498	sb = &so->so_rcv;
499	SOCKBUF_LOCK(sb);
500	if ((sb->sb_state & SBS_CANTRCVMORE) == 0 \|\| sbavail(sb))
501	ub->st_mode \|= S_IRUSR \| S_IRGRP \| S_IROTH;
502	ub->st_size = sbavail(sb) - sb->sb_ctl;
503	SOCKBUF_UNLOCK(sb);
504
505	sb = &so->so_snd;
506	SOCKBUF_LOCK(sb);
507	if ((sb->sb_state & SBS_CANTSENDMORE) == 0)
508	ub->st_mode \|= S_IWUSR \| S_IWGRP \| S_IWOTH;
509	SOCKBUF_UNLOCK(sb);
510	#ifndef __rtems__
511	ub->st_uid = so->so_cred->cr_uid;
512	ub->st_gid = so->so_cred->cr_gid;
513	#else /* __rtems__ */
514	ub->st_uid = BSD_DEFAULT_UID;
515	ub->st_gid = BSD_DEFAULT_GID;
516	#endif /* __rtems__ */
517	return (*so->so_proto->pr_usrreqs->pru_sense)(so, ub);
518	}
519	#ifdef __rtems__
520	static int
521	rtems_bsd_soo_stat(
522	const rtems_filesystem_location_info_t *loc,
523	struct stat *buf
524	)
525	{
526	struct socket *so = rtems_bsd_loc_to_f_data(loc);
527	int error = soo_stat(so, buf);
528
529	return rtems_bsd_error_to_status_and_errno(error);
530	}
531	#endif /* __rtems__ */
532
533	/*
534	* API socket close on file pointer. We call soclose() to close the socket
535	* (including initiating closing protocols). soclose() will sorele() the
536	* file reference but the actual socket will not go away until the socket's
537	* ref count hits 0.
538	*/
539	#ifdef __rtems__
540	int
541	#else /* __rtems__ */
542	static int
543	#endif /* __rtems__ */
544	soo_close(struct file fp, struct thread td)
545	{
546	int error = 0;
547	struct socket *so;
548
549	#ifdef __rtems__
550	/* FIXME: Move this to the RTEMS close() function */
551	knote_fdclose(td, rtems_bsd_fp_to_fd(fp));
552	#endif /* __rtems__ */
553
554	so = fp->f_data;
555	#ifndef __rtems__
556	fp->f_ops = &badfileops;
557	#else /* __rtems__ */
558	fp->f_io.pathinfo.handlers = &rtems_filesystem_handlers_default;
559	#endif /* __rtems__ */
560	fp->f_data = NULL;
561
562	if (so)
563	error = soclose(so);
564	return (error);
565	}
566
567	#ifndef __rtems__
568	static int
569	soo_fill_kinfo(struct file fp, struct kinfo_file kif, struct filedesc *fdp)
570	{
571	struct sockaddr *sa;
572	struct inpcb *inpcb;
573	struct unpcb *unpcb;
574	struct socket *so;
575	int error;
576
577	kif->kf_type = KF_TYPE_SOCKET;
578	so = fp->f_data;
579	kif->kf_sock_domain = so->so_proto->pr_domain->dom_family;
580	kif->kf_sock_type = so->so_type;
581	kif->kf_sock_protocol = so->so_proto->pr_protocol;
582	kif->kf_un.kf_sock.kf_sock_pcb = (uintptr_t)so->so_pcb;
583	switch (kif->kf_sock_domain) {
584	case AF_INET:
585	case AF_INET6:
586	if (kif->kf_sock_protocol == IPPROTO_TCP) {
587	if (so->so_pcb != NULL) {
588	inpcb = (struct inpcb *)(so->so_pcb);
589	kif->kf_un.kf_sock.kf_sock_inpcb =
590	(uintptr_t)inpcb->inp_ppcb;
591	}
592	}
593	break;
594	case AF_UNIX:
595	if (so->so_pcb != NULL) {
596	unpcb = (struct unpcb *)(so->so_pcb);
597	if (unpcb->unp_conn) {
598	kif->kf_un.kf_sock.kf_sock_unpconn =
599	(uintptr_t)unpcb->unp_conn;
600	kif->kf_un.kf_sock.kf_sock_rcv_sb_state =
601	so->so_rcv.sb_state;
602	kif->kf_un.kf_sock.kf_sock_snd_sb_state =
603	so->so_snd.sb_state;
604	}
605	}
606	break;
607	}
608	error = so->so_proto->pr_usrreqs->pru_sockaddr(so, &sa);
609	if (error == 0 && sa->sa_len <= sizeof(kif->kf_sa_local)) {
610	bcopy(sa, &kif->kf_sa_local, sa->sa_len);
611	free(sa, M_SONAME);
612	}
613	error = so->so_proto->pr_usrreqs->pru_peeraddr(so, &sa);
614	if (error == 0 && sa->sa_len <= sizeof(kif->kf_sa_peer)) {
615	bcopy(sa, &kif->kf_sa_peer, sa->sa_len);
616	free(sa, M_SONAME);
617	}
618	strncpy(kif->kf_path, so->so_proto->pr_domain->dom_name,
619	sizeof(kif->kf_path));
620	return (0);
621	}
622
623	/*
624	* Use the 'backend3' field in AIO jobs to store the amount of data
625	* completed by the AIO job so far.
626	*/
627	#define aio_done backend3
628
629	static STAILQ_HEAD(, task) soaio_jobs;
630	static struct mtx soaio_jobs_lock;
631	static struct task soaio_kproc_task;
632	static int soaio_starting, soaio_idle, soaio_queued;
633	static struct unrhdr *soaio_kproc_unr;
634
635	static int soaio_max_procs = MAX_AIO_PROCS;
636	SYSCTL_INT(_kern_ipc_aio, OID_AUTO, max_procs, CTLFLAG_RW, &soaio_max_procs, 0,
637	"Maximum number of kernel processes to use for async socket IO");
638
639	static int soaio_num_procs;
640	SYSCTL_INT(_kern_ipc_aio, OID_AUTO, num_procs, CTLFLAG_RD, &soaio_num_procs, 0,
641	"Number of active kernel processes for async socket IO");
642
643	static int soaio_target_procs = TARGET_AIO_PROCS;
644	SYSCTL_INT(_kern_ipc_aio, OID_AUTO, target_procs, CTLFLAG_RD,
645	&soaio_target_procs, 0,
646	"Preferred number of ready kernel processes for async socket IO");
647
648	static int soaio_lifetime;
649	SYSCTL_INT(_kern_ipc_aio, OID_AUTO, lifetime, CTLFLAG_RW, &soaio_lifetime, 0,
650	"Maximum lifetime for idle aiod");
651
652	static void
653	soaio_kproc_loop(void *arg)
654	{
655	struct proc *p;
656	struct vmspace *myvm;
657	struct task *task;
658	int error, id, pending;
659
660	id = (intptr_t)arg;
661
662	/*
663	* Grab an extra reference on the daemon's vmspace so that it
664	* doesn't get freed by jobs that switch to a different
665	* vmspace.
666	*/
667	p = curproc;
668	myvm = vmspace_acquire_ref(p);
669
670	mtx_lock(&soaio_jobs_lock);
671	MPASS(soaio_starting > 0);
672	soaio_starting--;
673	for (;;) {
674	while (!STAILQ_EMPTY(&soaio_jobs)) {
675	task = STAILQ_FIRST(&soaio_jobs);
676	STAILQ_REMOVE_HEAD(&soaio_jobs, ta_link);
677	soaio_queued--;
678	pending = task->ta_pending;
679	task->ta_pending = 0;
680	mtx_unlock(&soaio_jobs_lock);
681
682	task->ta_func(task->ta_context, pending);
683
684	mtx_lock(&soaio_jobs_lock);
685	}
686	MPASS(soaio_queued == 0);
687
688	if (p->p_vmspace != myvm) {
689	mtx_unlock(&soaio_jobs_lock);
690	vmspace_switch_aio(myvm);
691	mtx_lock(&soaio_jobs_lock);
692	continue;
693	}
694
695	soaio_idle++;
696	error = mtx_sleep(&soaio_idle, &soaio_jobs_lock, 0, "-",
697	soaio_lifetime);
698	soaio_idle--;
699	if (error == EWOULDBLOCK && STAILQ_EMPTY(&soaio_jobs) &&
700	soaio_num_procs > soaio_target_procs)
701	break;
702	}
703	soaio_num_procs--;
704	mtx_unlock(&soaio_jobs_lock);
705	free_unr(soaio_kproc_unr, id);
706	kproc_exit(0);
707	}
708
709	static void
710	soaio_kproc_create(void *context, int pending)
711	{
712	struct proc *p;
713	int error, id;
714
715	mtx_lock(&soaio_jobs_lock);
716	for (;;) {
717	if (soaio_num_procs < soaio_target_procs) {
718	/* Must create */
719	} else if (soaio_num_procs >= soaio_max_procs) {
720	/*
721	* Hit the limit on kernel processes, don't
722	* create another one.
723	*/
724	break;
725	} else if (soaio_queued <= soaio_idle + soaio_starting) {
726	/*
727	* No more AIO jobs waiting for a process to be
728	* created, so stop.
729	*/
730	break;
731	}
732	soaio_starting++;
733	mtx_unlock(&soaio_jobs_lock);
734
735	id = alloc_unr(soaio_kproc_unr);
736	error = kproc_create(soaio_kproc_loop, (void *)(intptr_t)id,
737	&p, 0, 0, "soaiod%d", id);
738	if (error != 0) {
739	free_unr(soaio_kproc_unr, id);
740	mtx_lock(&soaio_jobs_lock);
741	soaio_starting--;
742	break;
743	}
744
745	mtx_lock(&soaio_jobs_lock);
746	soaio_num_procs++;
747	}
748	mtx_unlock(&soaio_jobs_lock);
749	}
750
751	void
752	soaio_enqueue(struct task *task)
753	{
754
755	mtx_lock(&soaio_jobs_lock);
756	MPASS(task->ta_pending == 0);
757	task->ta_pending++;
758	STAILQ_INSERT_TAIL(&soaio_jobs, task, ta_link);
759	soaio_queued++;
760	if (soaio_queued <= soaio_idle)
761	wakeup_one(&soaio_idle);
762	else if (soaio_num_procs < soaio_max_procs)
763	taskqueue_enqueue(taskqueue_thread, &soaio_kproc_task);
764	mtx_unlock(&soaio_jobs_lock);
765	}
766
767	static void
768	soaio_init(void)
769	{
770
771	soaio_lifetime = AIOD_LIFETIME_DEFAULT;
772	STAILQ_INIT(&soaio_jobs);
773	mtx_init(&soaio_jobs_lock, "soaio jobs", NULL, MTX_DEF);
774	soaio_kproc_unr = new_unrhdr(1, INT_MAX, NULL);
775	TASK_INIT(&soaio_kproc_task, 0, soaio_kproc_create, NULL);
776	if (soaio_target_procs > 0)
777	taskqueue_enqueue(taskqueue_thread, &soaio_kproc_task);
778	}
779	SYSINIT(soaio, SI_SUB_VFS, SI_ORDER_ANY, soaio_init, NULL);
780
781	static __inline int
782	soaio_ready(struct socket so, struct sockbuf sb)
783	{
784	return (sb == &so->so_rcv ? soreadable(so) : sowriteable(so));
785	}
786
787	static void
788	soaio_process_job(struct socket so, struct sockbuf sb, struct kaiocb *job)
789	{
790	struct ucred *td_savedcred;
791	struct thread *td;
792	struct file *fp;
793	struct uio uio;
794	struct iovec iov;
795	size_t cnt, done;
796	long ru_before;
797	int error, flags;
798
799	SOCKBUF_UNLOCK(sb);
800	aio_switch_vmspace(job);
801	td = curthread;
802	fp = job->fd_file;
803	retry:
804	td_savedcred = td->td_ucred;
805	td->td_ucred = job->cred;
806
807	done = job->aio_done;
808	cnt = job->uaiocb.aio_nbytes - done;
809	iov.iov_base = (void *)((uintptr_t)job->uaiocb.aio_buf + done);
810	iov.iov_len = cnt;
811	uio.uio_iov = &iov;
812	uio.uio_iovcnt = 1;
813	uio.uio_offset = 0;
814	uio.uio_resid = cnt;
815	uio.uio_segflg = UIO_USERSPACE;
816	uio.uio_td = td;
817	flags = MSG_NBIO;
818
819	/*
820	* For resource usage accounting, only count a completed request
821	* as a single message to avoid counting multiple calls to
822	* sosend/soreceive on a blocking socket.
823	*/
824
825	if (sb == &so->so_rcv) {
826	uio.uio_rw = UIO_READ;
827	ru_before = td->td_ru.ru_msgrcv;
828	#ifdef MAC
829	error = mac_socket_check_receive(fp->f_cred, so);
830	if (error == 0)
831
832	#endif
833	error = soreceive(so, NULL, &uio, NULL, NULL, &flags);
834	if (td->td_ru.ru_msgrcv != ru_before)
835	job->msgrcv = 1;
836	} else {
837	if (!TAILQ_EMPTY(&sb->sb_aiojobq))
838	flags \|= MSG_MORETOCOME;
839	uio.uio_rw = UIO_WRITE;
840	ru_before = td->td_ru.ru_msgsnd;
841	#ifdef MAC
842	error = mac_socket_check_send(fp->f_cred, so);
843	if (error == 0)
844	#endif
845	error = sosend(so, NULL, &uio, NULL, NULL, flags, td);
846	if (td->td_ru.ru_msgsnd != ru_before)
847	job->msgsnd = 1;
848	if (error == EPIPE && (so->so_options & SO_NOSIGPIPE) == 0) {
849	PROC_LOCK(job->userproc);
850	kern_psignal(job->userproc, SIGPIPE);
851	PROC_UNLOCK(job->userproc);
852	}
853	}
854
855	done += cnt - uio.uio_resid;
856	job->aio_done = done;
857	td->td_ucred = td_savedcred;
858
859	if (error == EWOULDBLOCK) {
860	/*
861	* The request was either partially completed or not
862	* completed at all due to racing with a read() or
863	* write() on the socket. If the socket is
864	* non-blocking, return with any partial completion.
865	* If the socket is blocking or if no progress has
866	* been made, requeue this request at the head of the
867	* queue to try again when the socket is ready.
868	*/
869	MPASS(done != job->uaiocb.aio_nbytes);
870	SOCKBUF_LOCK(sb);
871	if (done == 0 \|\| !(so->so_state & SS_NBIO)) {
872	empty_results++;
873	if (soaio_ready(so, sb)) {
874	empty_retries++;
875	SOCKBUF_UNLOCK(sb);
876	goto retry;
877	}
878
879	if (!aio_set_cancel_function(job, soo_aio_cancel)) {
880	SOCKBUF_UNLOCK(sb);
881	if (done != 0)
882	aio_complete(job, done, 0);
883	else
884	aio_cancel(job);
885	SOCKBUF_LOCK(sb);
886	} else {
887	TAILQ_INSERT_HEAD(&sb->sb_aiojobq, job, list);
888	}
889	return;
890	}
891	SOCKBUF_UNLOCK(sb);
892	}
893	if (done != 0 && (error == ERESTART \|\| error == EINTR \|\|
894	error == EWOULDBLOCK))
895	error = 0;
896	if (error)
897	aio_complete(job, -1, error);
898	else
899	aio_complete(job, done, 0);
900	SOCKBUF_LOCK(sb);
901	}
902
903	static void
904	soaio_process_sb(struct socket so, struct sockbuf sb)
905	{
906	struct kaiocb *job;
907
908	SOCKBUF_LOCK(sb);
909	while (!TAILQ_EMPTY(&sb->sb_aiojobq) && soaio_ready(so, sb)) {
910	job = TAILQ_FIRST(&sb->sb_aiojobq);
911	TAILQ_REMOVE(&sb->sb_aiojobq, job, list);
912	if (!aio_clear_cancel_function(job))
913	continue;
914
915	soaio_process_job(so, sb, job);
916	}
917
918	/*
919	* If there are still pending requests, the socket must not be
920	* ready so set SB_AIO to request a wakeup when the socket
921	* becomes ready.
922	*/
923	if (!TAILQ_EMPTY(&sb->sb_aiojobq))
924	sb->sb_flags \|= SB_AIO;
925	sb->sb_flags &= ~SB_AIO_RUNNING;
926	SOCKBUF_UNLOCK(sb);
927
928	ACCEPT_LOCK();
929	SOCK_LOCK(so);
930	sorele(so);
931	}
932
933	void
934	soaio_rcv(void *context, int pending)
935	{
936	struct socket *so;
937
938	so = context;
939	soaio_process_sb(so, &so->so_rcv);
940	}
941
942	void
943	soaio_snd(void *context, int pending)
944	{
945	struct socket *so;
946
947	so = context;
948	soaio_process_sb(so, &so->so_snd);
949	}
950
951	void
952	sowakeup_aio(struct socket so, struct sockbuf sb)
953	{
954
955	SOCKBUF_LOCK_ASSERT(sb);
956	sb->sb_flags &= ~SB_AIO;
957	if (sb->sb_flags & SB_AIO_RUNNING)
958	return;
959	sb->sb_flags \|= SB_AIO_RUNNING;
960	if (sb == &so->so_snd)
961	SOCK_LOCK(so);
962	soref(so);
963	if (sb == &so->so_snd)
964	SOCK_UNLOCK(so);
965	soaio_enqueue(&sb->sb_aiotask);
966	}
967
968	static void
969	soo_aio_cancel(struct kaiocb *job)
970	{
971	struct socket *so;
972	struct sockbuf *sb;
973	long done;
974	int opcode;
975
976	so = job->fd_file->f_data;
977	opcode = job->uaiocb.aio_lio_opcode;
978	if (opcode == LIO_READ)
979	sb = &so->so_rcv;
980	else {
981	MPASS(opcode == LIO_WRITE);
982	sb = &so->so_snd;
983	}
984
985	SOCKBUF_LOCK(sb);
986	if (!aio_cancel_cleared(job))
987	TAILQ_REMOVE(&sb->sb_aiojobq, job, list);
988	if (TAILQ_EMPTY(&sb->sb_aiojobq))
989	sb->sb_flags &= ~SB_AIO;
990	SOCKBUF_UNLOCK(sb);
991
992	done = job->aio_done;
993	if (done != 0)
994	aio_complete(job, done, 0);
995	else
996	aio_cancel(job);
997	}
998
999	static int
1000	soo_aio_queue(struct file fp, struct kaiocb job)
1001	{
1002	struct socket *so;
1003	struct sockbuf *sb;
1004	int error;
1005
1006	so = fp->f_data;
1007	error = (*so->so_proto->pr_usrreqs->pru_aio_queue)(so, job);
1008	if (error == 0)
1009	return (0);
1010
1011	switch (job->uaiocb.aio_lio_opcode) {
1012	case LIO_READ:
1013	sb = &so->so_rcv;
1014	break;
1015	case LIO_WRITE:
1016	sb = &so->so_snd;
1017	break;
1018	default:
1019	return (EINVAL);
1020	}
1021
1022	SOCKBUF_LOCK(sb);
1023	if (!aio_set_cancel_function(job, soo_aio_cancel))
1024	panic("new job was cancelled");
1025	TAILQ_INSERT_TAIL(&sb->sb_aiojobq, job, list);
1026	if (!(sb->sb_flags & SB_AIO_RUNNING)) {
1027	if (soaio_ready(so, sb))
1028	sowakeup_aio(so, sb);
1029	else
1030	sb->sb_flags \|= SB_AIO;
1031	}
1032	SOCKBUF_UNLOCK(sb);
1033	return (0);
1034	}
1035	#endif /* __rtems__ */
1036	#ifdef __rtems__
1037	static int
1038	rtems_bsd_soo_open(rtems_libio_t iop, const char path, int oflag,
1039	mode_t mode)
1040	{
1041	return rtems_bsd_error_to_status_and_errno(ENXIO);
1042	}
1043
1044	static int
1045	rtems_bsd_soo_close(rtems_libio_t *iop)
1046	{
1047	struct file *fp = rtems_bsd_iop_to_fp(iop);
1048	int error = soo_close(fp, NULL);
1049
1050	return rtems_bsd_error_to_status_and_errno(error);
1051	}
1052
1053	static int
1054	rtems_bsd_soo_fcntl(rtems_libio_t *iop, int cmd)
1055	{
1056	int error = 0;
1057
1058	if (cmd == F_SETFL) {
1059	struct file *fp = rtems_bsd_iop_to_fp(iop);
1060	int nbio = iop->flags & LIBIO_FLAGS_NO_DELAY;
1061
1062	error = soo_ioctl(fp, FIONBIO, &nbio, NULL, NULL);
1063	}
1064
1065	return rtems_bsd_error_to_status_and_errno(error);
1066	}
1067
1068	const rtems_filesystem_file_handlers_r socketops = {
1069	.open_h = rtems_bsd_soo_open,
1070	.close_h = rtems_bsd_soo_close,
1071	.read_h = rtems_bsd_soo_read,
1072	.write_h = rtems_bsd_soo_write,
1073	.ioctl_h = rtems_bsd_soo_ioctl,
1074	.lseek_h = rtems_filesystem_default_lseek,
1075	.fstat_h = rtems_bsd_soo_stat,
1076	.ftruncate_h = rtems_filesystem_default_ftruncate,
1077	.fsync_h = rtems_filesystem_default_fsync_or_fdatasync,
1078	.fdatasync_h = rtems_filesystem_default_fsync_or_fdatasync,
1079	.fcntl_h = rtems_bsd_soo_fcntl,
1080	.poll_h = rtems_bsd_soo_poll,
1081	.kqfilter_h = rtems_bsd_soo_kqfilter,
1082	.readv_h = rtems_bsd_soo_readv,
1083	.writev_h = rtems_bsd_soo_writev,
1084	.mmap_h = rtems_filesystem_default_mmap
1085	};
1086	#endif /* __rtems__ */

Note: See TracBrowser for help on using the repository browser.

Download in other formats: