Context Navigation

source: rtems-libbsd/freebsd/sys/kern/kern_event.c @ 0c9f27b

4.1155-freebsd-126-freebsd-12freebsd-9.3

Last change on this file since 0c9f27b was 0c9f27b, checked in by Sebastian Huber <sebastian.huber@…>, on 10/28/13 at 14:40:53
Use kqueue() and kevent() from FreeBSD
Property mode set to `100644`
File size: 55.4 KB

Line
1	#include <machine/rtems-bsd-config.h>
2
3	/*-
4	* Copyright (c) 1999,2000,2001 Jonathan Lemon <jlemon@FreeBSD.org>
5	* Copyright 2004 John-Mark Gurney <jmg@FreeBSD.org>
6	* Copyright (c) 2009 Apple, Inc.
7	* All rights reserved.
8	*
9	* Redistribution and use in source and binary forms, with or without
10	* modification, are permitted provided that the following conditions
11	* are met:
12	* 1. Redistributions of source code must retain the above copyright
13	* notice, this list of conditions and the following disclaimer.
14	* 2. Redistributions in binary form must reproduce the above copyright
15	* notice, this list of conditions and the following disclaimer in the
16	* documentation and/or other materials provided with the distribution.
17	*
18	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28	* SUCH DAMAGE.
29	*/
30
31	#include <sys/cdefs.h>
32	__FBSDID("$FreeBSD$");
33
34	#include <rtems/bsd/local/opt_ktrace.h>
35
36	#include <rtems/bsd/sys/param.h>
37	#include <sys/systm.h>
38	#include <sys/kernel.h>
39	#include <rtems/bsd/sys/lock.h>
40	#include <sys/mutex.h>
41	#include <sys/proc.h>
42	#include <sys/malloc.h>
43	#include <rtems/bsd/sys/unistd.h>
44	#include <sys/file.h>
45	#include <sys/filedesc.h>
46	#include <sys/filio.h>
47	#include <sys/fcntl.h>
48	#include <sys/kthread.h>
49	#include <sys/selinfo.h>
50	#include <sys/queue.h>
51	#include <sys/event.h>
52	#include <sys/eventvar.h>
53	#include <sys/poll.h>
54	#include <sys/protosw.h>
55	#include <sys/sigio.h>
56	#include <sys/signalvar.h>
57	#include <sys/socket.h>
58	#include <sys/socketvar.h>
59	#include <sys/stat.h>
60	#include <sys/sysctl.h>
61	#include <sys/sysproto.h>
62	#include <sys/syscallsubr.h>
63	#include <sys/taskqueue.h>
64	#include <sys/uio.h>
65	#ifdef KTRACE
66	#include <sys/ktrace.h>
67	#endif
68
69	#include <vm/uma.h>
70	#ifdef __rtems__
71	#include <machine/rtems-bsd-syscall-api.h>
72
73	/* Maintain a global kqueue list on RTEMS */
74	static struct kqlist fd_kqlist;
75	#endif /* __rtems__ */
76
77	static MALLOC_DEFINE(M_KQUEUE, "kqueue", "memory for kqueue system");
78
79	/*
80	* This lock is used if multiple kq locks are required. This possibly
81	* should be made into a per proc lock.
82	*/
83	static struct mtx kq_global;
84	MTX_SYSINIT(kq_global, &kq_global, "kqueue order", MTX_DEF);
85	#define KQ_GLOBAL_LOCK(lck, haslck) do { \
86	if (!haslck) \
87	mtx_lock(lck); \
88	haslck = 1; \
89	} while (0)
90	#define KQ_GLOBAL_UNLOCK(lck, haslck) do { \
91	if (haslck) \
92	mtx_unlock(lck); \
93	haslck = 0; \
94	} while (0)
95
96	TASKQUEUE_DEFINE_THREAD(kqueue);
97
98	static int kevent_copyout(void arg, struct kevent kevp, int count);
99	static int kevent_copyin(void arg, struct kevent kevp, int count);
100	static int kqueue_register(struct kqueue kq, struct kevent kev,
101	struct thread *td, int waitok);
102	static int kqueue_acquire(struct file fp, struct kqueue *kqp);
103	static void kqueue_release(struct kqueue *kq, int locked);
104	static int kqueue_expand(struct kqueue kq, struct filterops fops,
105	uintptr_t ident, int waitok);
106	static void kqueue_task(void *arg, int pending);
107	static int kqueue_scan(struct kqueue *kq, int maxevents,
108	struct kevent_copyops *k_ops,
109	const struct timespec *timeout,
110	struct kevent keva, struct thread td);
111	static void kqueue_wakeup(struct kqueue *kq);
112	static struct filterops *kqueue_fo_find(int filt);
113	static void kqueue_fo_release(int filt);
114
115	#ifndef __rtems__
116	static fo_rdwr_t kqueue_read;
117	static fo_rdwr_t kqueue_write;
118	static fo_truncate_t kqueue_truncate;
119	static fo_ioctl_t kqueue_ioctl;
120	static fo_poll_t kqueue_poll;
121	static fo_kqfilter_t kqueue_kqfilter;
122	static fo_stat_t kqueue_stat;
123	static fo_close_t kqueue_close;
124
125	static struct fileops kqueueops = {
126	.fo_read = kqueue_read,
127	.fo_write = kqueue_write,
128	.fo_truncate = kqueue_truncate,
129	.fo_ioctl = kqueue_ioctl,
130	.fo_poll = kqueue_poll,
131	.fo_kqfilter = kqueue_kqfilter,
132	.fo_stat = kqueue_stat,
133	.fo_close = kqueue_close,
134	};
135	#else /* __rtems__ */
136	static const rtems_filesystem_file_handlers_r kqueueops;
137	#endif /* __rtems__ */
138
139	static int knote_attach(struct knote kn, struct kqueue kq);
140	static void knote_drop(struct knote kn, struct thread td);
141	static void knote_enqueue(struct knote *kn);
142	static void knote_dequeue(struct knote *kn);
143	static void knote_init(void);
144	static struct knote *knote_alloc(int waitok);
145	static void knote_free(struct knote *kn);
146
147	static void filt_kqdetach(struct knote *kn);
148	static int filt_kqueue(struct knote *kn, long hint);
149	#ifndef __rtems__
150	static int filt_procattach(struct knote *kn);
151	static void filt_procdetach(struct knote *kn);
152	static int filt_proc(struct knote *kn, long hint);
153	#endif /* __rtems__ */
154	static int filt_fileattach(struct knote *kn);
155	static void filt_timerexpire(void *knx);
156	static int filt_timerattach(struct knote *kn);
157	static void filt_timerdetach(struct knote *kn);
158	static int filt_timer(struct knote *kn, long hint);
159	static int filt_userattach(struct knote *kn);
160	static void filt_userdetach(struct knote *kn);
161	static int filt_user(struct knote *kn, long hint);
162	static void filt_usertouch(struct knote kn, struct kevent kev,
163	u_long type);
164
165	static struct filterops file_filtops =
166	{ 1, filt_fileattach, NULL, NULL };
167	static struct filterops kqread_filtops =
168	{ 1, NULL, filt_kqdetach, filt_kqueue };
169	/* XXX - move to kern_proc.c? */
170	#ifndef __rtems__
171	static struct filterops proc_filtops =
172	{ 0, filt_procattach, filt_procdetach, filt_proc };
173	#endif /* __rtems__ */
174	static struct filterops timer_filtops =
175	{ 0, filt_timerattach, filt_timerdetach, filt_timer };
176	static struct filterops user_filtops = {
177	.f_attach = filt_userattach,
178	.f_detach = filt_userdetach,
179	.f_event = filt_user,
180	.f_touch = filt_usertouch,
181	};
182
183	static uma_zone_t knote_zone;
184	static int kq_ncallouts = 0;
185	static int kq_calloutmax = (4 * 1024);
186	SYSCTL_INT(_kern, OID_AUTO, kq_calloutmax, CTLFLAG_RW,
187	&kq_calloutmax, 0, "Maximum number of callouts allocated for kqueue");
188
189	/* XXX - ensure not KN_INFLUX?? */
190	#define KNOTE_ACTIVATE(kn, islock) do { \
191	if ((islock)) \
192	mtx_assert(&(kn)->kn_kq->kq_lock, MA_OWNED); \
193	else \
194	KQ_LOCK((kn)->kn_kq); \
195	(kn)->kn_status \|= KN_ACTIVE; \
196	if (((kn)->kn_status & (KN_QUEUED \| KN_DISABLED)) == 0) \
197	knote_enqueue((kn)); \
198	if (!(islock)) \
199	KQ_UNLOCK((kn)->kn_kq); \
200	} while(0)
201	#define KQ_LOCK(kq) do { \
202	mtx_lock(&(kq)->kq_lock); \
203	} while (0)
204	#define KQ_FLUX_WAKEUP(kq) do { \
205	if (((kq)->kq_state & KQ_FLUXWAIT) == KQ_FLUXWAIT) { \
206	(kq)->kq_state &= ~KQ_FLUXWAIT; \
207	wakeup((kq)); \
208	} \
209	} while (0)
210	#define KQ_UNLOCK_FLUX(kq) do { \
211	KQ_FLUX_WAKEUP(kq); \
212	mtx_unlock(&(kq)->kq_lock); \
213	} while (0)
214	#define KQ_UNLOCK(kq) do { \
215	mtx_unlock(&(kq)->kq_lock); \
216	} while (0)
217	#define KQ_OWNED(kq) do { \
218	mtx_assert(&(kq)->kq_lock, MA_OWNED); \
219	} while (0)
220	#define KQ_NOTOWNED(kq) do { \
221	mtx_assert(&(kq)->kq_lock, MA_NOTOWNED); \
222	} while (0)
223	#define KN_LIST_LOCK(kn) do { \
224	if (kn->kn_knlist != NULL) \
225	kn->kn_knlist->kl_lock(kn->kn_knlist->kl_lockarg); \
226	} while (0)
227	#define KN_LIST_UNLOCK(kn) do { \
228	if (kn->kn_knlist != NULL) \
229	kn->kn_knlist->kl_unlock(kn->kn_knlist->kl_lockarg); \
230	} while (0)
231	#define KNL_ASSERT_LOCK(knl, islocked) do { \
232	if (islocked) \
233	KNL_ASSERT_LOCKED(knl); \
234	else \
235	KNL_ASSERT_UNLOCKED(knl); \
236	} while (0)
237	#ifdef INVARIANTS
238	#define KNL_ASSERT_LOCKED(knl) do { \
239	knl->kl_assert_locked((knl)->kl_lockarg); \
240	} while (0)
241	#define KNL_ASSERT_UNLOCKED(knl) do { \
242	knl->kl_assert_unlocked((knl)->kl_lockarg); \
243	} while (0)
244	#else /* !INVARIANTS */
245	#define KNL_ASSERT_LOCKED(knl) do {} while(0)
246	#define KNL_ASSERT_UNLOCKED(knl) do {} while (0)
247	#endif /* INVARIANTS */
248
249	#define KN_HASHSIZE 64 /* XXX should be tunable */
250	#define KN_HASH(val, mask) (((val) ^ (val >> 8)) & (mask))
251
252	static int
253	filt_nullattach(struct knote *kn)
254	{
255
256	return (ENXIO);
257	};
258
259	struct filterops null_filtops =
260	{ 0, filt_nullattach, NULL, NULL };
261
262	/* XXX - make SYSINIT to add these, and move into respective modules. */
263	extern struct filterops sig_filtops;
264	extern struct filterops fs_filtops;
265
266	/*
267	* Table for for all system-defined filters.
268	*/
269	static struct mtx filterops_lock;
270	MTX_SYSINIT(kqueue_filterops, &filterops_lock, "protect sysfilt_ops",
271	MTX_DEF);
272	static struct {
273	struct filterops *for_fop;
274	int for_refcnt;
275	} sysfilt_ops[EVFILT_SYSCOUNT] = {
276	{ &file_filtops }, /* EVFILT_READ */
277	{ &file_filtops }, /* EVFILT_WRITE */
278	{ &null_filtops }, /* EVFILT_AIO */
279	{ &file_filtops }, /* EVFILT_VNODE */
280	#ifndef __rtems__
281	{ &proc_filtops }, /* EVFILT_PROC */
282	{ &sig_filtops }, /* EVFILT_SIGNAL */
283	#else /* __rtems__ */
284	{ &null_filtops }, /* EVFILT_PROC */
285	{ &null_filtops }, /* EVFILT_SIGNAL */
286	#endif /* __rtems__ */
287	{ &timer_filtops }, /* EVFILT_TIMER */
288	{ &null_filtops }, /* former EVFILT_NETDEV */
289	#ifndef __rtems__
290	{ &fs_filtops }, /* EVFILT_FS */
291	#else /* __rtems__ */
292	{ &null_filtops }, /* EVFILT_FS */
293	#endif /* __rtems__ */
294	{ &null_filtops }, /* EVFILT_LIO */
295	{ &user_filtops }, /* EVFILT_USER */
296	};
297
298	/*
299	* Simple redirection for all cdevsw style objects to call their fo_kqfilter
300	* method.
301	*/
302	static int
303	filt_fileattach(struct knote *kn)
304	{
305
306	return (fo_kqfilter(kn->kn_fp, kn));
307	}
308
309	/ARGSUSED/
310	static int
311	kqueue_kqfilter(struct file fp, struct knote kn)
312	{
313	struct kqueue *kq = kn->kn_fp->f_data;
314
315	if (kn->kn_filter != EVFILT_READ)
316	return (EINVAL);
317
318	kn->kn_status \|= KN_KQUEUE;
319	kn->kn_fop = &kqread_filtops;
320	knlist_add(&kq->kq_sel.si_note, kn, 0);
321
322	return (0);
323	}
324	#ifdef __rtems__
325	static int
326	rtems_bsd_kqueue_kqfilter(rtems_libio_t iop, struct knote kn)
327	{
328	struct file *fp = rtems_bsd_iop_to_fp(iop);
329
330	return kqueue_kqfilter(fp, kn);
331	}
332	#endif /* __rtems__ */
333
334	static void
335	filt_kqdetach(struct knote *kn)
336	{
337	struct kqueue *kq = kn->kn_fp->f_data;
338
339	knlist_remove(&kq->kq_sel.si_note, kn, 0);
340	}
341
342	/ARGSUSED/
343	static int
344	filt_kqueue(struct knote *kn, long hint)
345	{
346	struct kqueue *kq = kn->kn_fp->f_data;
347
348	kn->kn_data = kq->kq_count;
349	return (kn->kn_data > 0);
350	}
351
352	#ifndef __rtems__
353	/* XXX - move to kern_proc.c? */
354	static int
355	filt_procattach(struct knote *kn)
356	{
357	struct proc *p;
358	int immediate;
359	int error;
360
361	immediate = 0;
362	p = pfind(kn->kn_id);
363	if (p == NULL && (kn->kn_sfflags & NOTE_EXIT)) {
364	p = zpfind(kn->kn_id);
365	immediate = 1;
366	} else if (p != NULL && (p->p_flag & P_WEXIT)) {
367	immediate = 1;
368	}
369
370	if (p == NULL)
371	return (ESRCH);
372	if ((error = p_cansee(curthread, p))) {
373	PROC_UNLOCK(p);
374	return (error);
375	}
376
377	kn->kn_ptr.p_proc = p;
378	kn->kn_flags \|= EV_CLEAR; /* automatically set */
379
380	/*
381	* internal flag indicating registration done by kernel
382	*/
383	if (kn->kn_flags & EV_FLAG1) {
384	kn->kn_data = kn->kn_sdata; /* ppid */
385	kn->kn_fflags = NOTE_CHILD;
386	kn->kn_flags &= ~EV_FLAG1;
387	}
388
389	if (immediate == 0)
390	knlist_add(&p->p_klist, kn, 1);
391
392	/*
393	* Immediately activate any exit notes if the target process is a
394	* zombie. This is necessary to handle the case where the target
395	* process, e.g. a child, dies before the kevent is registered.
396	*/
397	if (immediate && filt_proc(kn, NOTE_EXIT))
398	KNOTE_ACTIVATE(kn, 0);
399
400	PROC_UNLOCK(p);
401
402	return (0);
403	}
404
405	/*
406	* The knote may be attached to a different process, which may exit,
407	* leaving nothing for the knote to be attached to. So when the process
408	* exits, the knote is marked as DETACHED and also flagged as ONESHOT so
409	* it will be deleted when read out. However, as part of the knote deletion,
410	* this routine is called, so a check is needed to avoid actually performing
411	* a detach, because the original process does not exist any more.
412	*/
413	/* XXX - move to kern_proc.c? */
414	static void
415	filt_procdetach(struct knote *kn)
416	{
417	struct proc *p;
418
419	p = kn->kn_ptr.p_proc;
420	knlist_remove(&p->p_klist, kn, 0);
421	kn->kn_ptr.p_proc = NULL;
422	}
423
424	/* XXX - move to kern_proc.c? */
425	static int
426	filt_proc(struct knote *kn, long hint)
427	{
428	struct proc *p = kn->kn_ptr.p_proc;
429	u_int event;
430
431	/*
432	* mask off extra data
433	*/
434	event = (u_int)hint & NOTE_PCTRLMASK;
435
436	/*
437	* if the user is interested in this event, record it.
438	*/
439	if (kn->kn_sfflags & event)
440	kn->kn_fflags \|= event;
441
442	/*
443	* process is gone, so flag the event as finished.
444	*/
445	if (event == NOTE_EXIT) {
446	if (!(kn->kn_status & KN_DETACHED))
447	knlist_remove_inevent(&p->p_klist, kn);
448	kn->kn_flags \|= (EV_EOF \| EV_ONESHOT);
449	kn->kn_data = p->p_xstat;
450	kn->kn_ptr.p_proc = NULL;
451	return (1);
452	}
453
454	return (kn->kn_fflags != 0);
455	}
456
457	/*
458	* Called when the process forked. It mostly does the same as the
459	* knote(), activating all knotes registered to be activated when the
460	* process forked. Additionally, for each knote attached to the
461	* parent, check whether user wants to track the new process. If so
462	* attach a new knote to it, and immediately report an event with the
463	* child's pid.
464	*/
465	void
466	knote_fork(struct knlist *list, int pid)
467	{
468	struct kqueue *kq;
469	struct knote *kn;
470	struct kevent kev;
471	int error;
472
473	if (list == NULL)
474	return;
475	list->kl_lock(list->kl_lockarg);
476
477	SLIST_FOREACH(kn, &list->kl_list, kn_selnext) {
478	if ((kn->kn_status & KN_INFLUX) == KN_INFLUX)
479	continue;
480	kq = kn->kn_kq;
481	KQ_LOCK(kq);
482	if ((kn->kn_status & KN_INFLUX) == KN_INFLUX) {
483	KQ_UNLOCK(kq);
484	continue;
485	}
486
487	/*
488	* The same as knote(), activate the event.
489	*/
490	if ((kn->kn_sfflags & NOTE_TRACK) == 0) {
491	kn->kn_status \|= KN_HASKQLOCK;
492	if (kn->kn_fop->f_event(kn, NOTE_FORK \| pid))
493	KNOTE_ACTIVATE(kn, 1);
494	kn->kn_status &= ~KN_HASKQLOCK;
495	KQ_UNLOCK(kq);
496	continue;
497	}
498
499	/*
500	* The NOTE_TRACK case. In addition to the activation
501	* of the event, we need to register new event to
502	* track the child. Drop the locks in preparation for
503	* the call to kqueue_register().
504	*/
505	kn->kn_status \|= KN_INFLUX;
506	KQ_UNLOCK(kq);
507	list->kl_unlock(list->kl_lockarg);
508
509	/*
510	* Activate existing knote and register a knote with
511	* new process.
512	*/
513	kev.ident = pid;
514	kev.filter = kn->kn_filter;
515	kev.flags = kn->kn_flags \| EV_ADD \| EV_ENABLE \| EV_FLAG1;
516	kev.fflags = kn->kn_sfflags;
517	kev.data = kn->kn_id; /* parent */
518	kev.udata = kn->kn_kevent.udata;/* preserve udata */
519	error = kqueue_register(kq, &kev, NULL, 0);
520	if (kn->kn_fop->f_event(kn, NOTE_FORK \| pid))
521	KNOTE_ACTIVATE(kn, 0);
522	if (error)
523	kn->kn_fflags \|= NOTE_TRACKERR;
524	KQ_LOCK(kq);
525	kn->kn_status &= ~KN_INFLUX;
526	KQ_UNLOCK_FLUX(kq);
527	list->kl_lock(list->kl_lockarg);
528	}
529	list->kl_unlock(list->kl_lockarg);
530	}
531	#endif /* __rtems__ */
532
533	static int
534	timertoticks(intptr_t data)
535	{
536	struct timeval tv;
537	int tticks;
538
539	tv.tv_sec = data / 1000;
540	tv.tv_usec = (data % 1000) * 1000;
541	tticks = tvtohz(&tv);
542
543	return tticks;
544	}
545
546	/* XXX - move to kern_timeout.c? */
547	static void
548	filt_timerexpire(void *knx)
549	{
550	struct knote *kn = knx;
551	struct callout *calloutp;
552
553	kn->kn_data++;
554	KNOTE_ACTIVATE(kn, 0); /* XXX - handle locking */
555
556	if ((kn->kn_flags & EV_ONESHOT) != EV_ONESHOT) {
557	calloutp = (struct callout *)kn->kn_hook;
558	callout_reset_curcpu(calloutp, timertoticks(kn->kn_sdata),
559	filt_timerexpire, kn);
560	}
561	}
562
563	/*
564	* data contains amount of time to sleep, in milliseconds
565	*/
566	/* XXX - move to kern_timeout.c? */
567	static int
568	filt_timerattach(struct knote *kn)
569	{
570	struct callout *calloutp;
571
572	atomic_add_int(&kq_ncallouts, 1);
573
574	if (kq_ncallouts >= kq_calloutmax) {
575	atomic_add_int(&kq_ncallouts, -1);
576	return (ENOMEM);
577	}
578
579	kn->kn_flags \|= EV_CLEAR; /* automatically set */
580	kn->kn_status &= ~KN_DETACHED; /* knlist_add usually sets it */
581	calloutp = malloc(sizeof(*calloutp), M_KQUEUE, M_WAITOK);
582	callout_init(calloutp, CALLOUT_MPSAFE);
583	kn->kn_hook = calloutp;
584	callout_reset_curcpu(calloutp, timertoticks(kn->kn_sdata),
585	filt_timerexpire, kn);
586
587	return (0);
588	}
589
590	/* XXX - move to kern_timeout.c? */
591	static void
592	filt_timerdetach(struct knote *kn)
593	{
594	struct callout *calloutp;
595
596	calloutp = (struct callout *)kn->kn_hook;
597	callout_drain(calloutp);
598	free(calloutp, M_KQUEUE);
599	atomic_add_int(&kq_ncallouts, -1);
600	kn->kn_status \|= KN_DETACHED; /* knlist_remove usually clears it */
601	}
602
603	/* XXX - move to kern_timeout.c? */
604	static int
605	filt_timer(struct knote *kn, long hint)
606	{
607
608	return (kn->kn_data != 0);
609	}
610
611	static int
612	filt_userattach(struct knote *kn)
613	{
614
615	/*
616	* EVFILT_USER knotes are not attached to anything in the kernel.
617	*/
618	kn->kn_hook = NULL;
619	if (kn->kn_fflags & NOTE_TRIGGER)
620	kn->kn_hookid = 1;
621	else
622	kn->kn_hookid = 0;
623	return (0);
624	}
625
626	static void
627	filt_userdetach(__unused struct knote *kn)
628	{
629
630	/*
631	* EVFILT_USER knotes are not attached to anything in the kernel.
632	*/
633	}
634
635	static int
636	filt_user(struct knote *kn, __unused long hint)
637	{
638
639	return (kn->kn_hookid);
640	}
641
642	static void
643	filt_usertouch(struct knote kn, struct kevent kev, u_long type)
644	{
645	u_int ffctrl;
646
647	switch (type) {
648	case EVENT_REGISTER:
649	if (kev->fflags & NOTE_TRIGGER)
650	kn->kn_hookid = 1;
651
652	ffctrl = kev->fflags & NOTE_FFCTRLMASK;
653	kev->fflags &= NOTE_FFLAGSMASK;
654	switch (ffctrl) {
655	case NOTE_FFNOP:
656	break;
657
658	case NOTE_FFAND:
659	kn->kn_sfflags &= kev->fflags;
660	break;
661
662	case NOTE_FFOR:
663	kn->kn_sfflags \|= kev->fflags;
664	break;
665
666	case NOTE_FFCOPY:
667	kn->kn_sfflags = kev->fflags;
668	break;
669
670	default:
671	/* XXX Return error? */
672	break;
673	}
674	kn->kn_sdata = kev->data;
675	if (kev->flags & EV_CLEAR) {
676	kn->kn_hookid = 0;
677	kn->kn_data = 0;
678	kn->kn_fflags = 0;
679	}
680	break;
681
682	case EVENT_PROCESS:
683	*kev = kn->kn_kevent;
684	kev->fflags = kn->kn_sfflags;
685	kev->data = kn->kn_sdata;
686	if (kn->kn_flags & EV_CLEAR) {
687	kn->kn_hookid = 0;
688	kn->kn_data = 0;
689	kn->kn_fflags = 0;
690	}
691	break;
692
693	default:
694	panic("filt_usertouch() - invalid type (%ld)", type);
695	break;
696	}
697	}
698
699	#ifndef __rtems__
700	int
701	kqueue(struct thread td, struct kqueue_args uap)
702	#else /* __rtems__ */
703	static int
704	rtems_bsd_kqueue(struct thread td, struct kqueue_args uap)
705	#endif /* __rtems__ */
706	{
707	struct filedesc *fdp;
708	struct kqueue *kq;
709	struct file *fp;
710	int fd, error;
711
712	#ifndef __rtems__
713	fdp = td->td_proc->p_fd;
714	#else /* __rtems__ */
715	(void) fdp;
716	#endif /* __rtems__ */
717	error = falloc(td, &fp, &fd);
718	if (error)
719	goto done2;
720
721	/* An extra reference on `nfp' has been held for us by falloc(). */
722	kq = malloc(sizeof *kq, M_KQUEUE, M_WAITOK \| M_ZERO);
723	mtx_init(&kq->kq_lock, "kqueue", NULL, MTX_DEF\|MTX_DUPOK);
724	TAILQ_INIT(&kq->kq_head);
725	#ifndef __rtems__
726	kq->kq_fdp = fdp;
727	#endif /* __rtems__ */
728	knlist_init_mtx(&kq->kq_sel.si_note, &kq->kq_lock);
729	TASK_INIT(&kq->kq_task, 0, kqueue_task, kq);
730
731	#ifndef __rtems__
732	FILEDESC_XLOCK(fdp);
733	SLIST_INSERT_HEAD(&fdp->fd_kqlist, kq, kq_list);
734	FILEDESC_XUNLOCK(fdp);
735	#else /* __rtems__ */
736	rtems_libio_lock();
737	SLIST_INSERT_HEAD(&fd_kqlist, kq, kq_list);
738	rtems_libio_unlock();
739	#endif /* __rtems__ */
740
741	finit(fp, FREAD \| FWRITE, DTYPE_KQUEUE, kq, &kqueueops);
742	fdrop(fp, td);
743
744	td->td_retval[0] = fd;
745	done2:
746	return (error);
747	}
748	#ifdef __rtems__
749	int
750	kqueue(void)
751	{
752	struct thread *td = rtems_bsd_get_curthread_or_null();
753	struct kqueue_args ua;
754	int error;
755
756	if (td != NULL) {
757	error = rtems_bsd_kqueue(td, &ua);
758	} else {
759	error = ENOMEM;
760	}
761
762	if (error == 0) {
763	return td->td_retval[0];
764	} else {
765	rtems_set_errno_and_return_minus_one(error);
766	}
767	}
768	#endif /* __rtems__ */
769
770	#ifndef _SYS_SYSPROTO_H_
771	struct kevent_args {
772	int fd;
773	const struct kevent *changelist;
774	int nchanges;
775	struct kevent *eventlist;
776	int nevents;
777	const struct timespec *timeout;
778	};
779	#endif
780	#ifndef __rtems__
781	int
782	kevent(struct thread td, struct kevent_args uap)
783	#else /* __rtems__ */
784	static int
785	kern_kevent(struct thread *td, int fd, int nchanges, int nevents, struct
786	kevent_copyops k_ops, const struct timespec timeout);
787
788	static int
789	rtems_bsd_kevent(struct thread td, struct kevent_args uap)
790	#endif /* __rtems__ */
791	{
792	struct timespec ts, *tsp;
793	struct kevent_copyops k_ops = { uap,
794	kevent_copyout,
795	kevent_copyin};
796	int error;
797	#ifdef KTRACE
798	struct uio ktruio;
799	struct iovec ktriov;
800	struct uio *ktruioin = NULL;
801	struct uio *ktruioout = NULL;
802	#endif
803
804	if (uap->timeout != NULL) {
805	error = copyin(uap->timeout, &ts, sizeof(ts));
806	if (error)
807	return (error);
808	tsp = &ts;
809	} else
810	tsp = NULL;
811
812	#ifdef KTRACE
813	if (KTRPOINT(td, KTR_GENIO)) {
814	ktriov.iov_base = uap->changelist;
815	ktriov.iov_len = uap->nchanges * sizeof(struct kevent);
816	ktruio = (struct uio){ .uio_iov = &ktriov, .uio_iovcnt = 1,
817	.uio_segflg = UIO_USERSPACE, .uio_rw = UIO_READ,
818	.uio_td = td };
819	ktruioin = cloneuio(&ktruio);
820	ktriov.iov_base = uap->eventlist;
821	ktriov.iov_len = uap->nevents * sizeof(struct kevent);
822	ktruioout = cloneuio(&ktruio);
823	}
824	#endif
825
826	error = kern_kevent(td, uap->fd, uap->nchanges, uap->nevents,
827	&k_ops, tsp);
828
829	#ifdef KTRACE
830	if (ktruioin != NULL) {
831	ktruioin->uio_resid = uap->nchanges * sizeof(struct kevent);
832	ktrgenio(uap->fd, UIO_WRITE, ktruioin, 0);
833	ktruioout->uio_resid = td->td_retval[0] * sizeof(struct kevent);
834	ktrgenio(uap->fd, UIO_READ, ktruioout, error);
835	}
836	#endif
837
838	return (error);
839	}
840	#ifdef __rtems__
841	__weak_reference(kevent, _kevent);
842
843	int
844	kevent(int kq, const struct kevent *changelist, int nchanges,
845	struct kevent *eventlist, int nevents,
846	const struct timespec *timeout)
847	{
848	struct thread *td = rtems_bsd_get_curthread_or_null();
849	struct kevent_args ua = {
850	.fd = kq,
851	.changelist = changelist,
852	.nchanges = nchanges,
853	.eventlist = eventlist,
854	.nevents = nevents,
855	.timeout = timeout
856	};
857	int error;
858
859	if (td != NULL) {
860	error = rtems_bsd_kevent(td, &ua);
861	} else {
862	error = ENOMEM;
863	}
864
865	if (error == 0) {
866	return td->td_retval[0];
867	} else {
868	rtems_set_errno_and_return_minus_one(error);
869	}
870	}
871	#endif /* __rtems__ */
872
873	/*
874	* Copy 'count' items into the destination list pointed to by uap->eventlist.
875	*/
876	static int
877	kevent_copyout(void arg, struct kevent kevp, int count)
878	{
879	struct kevent_args *uap;
880	int error;
881
882	KASSERT(count <= KQ_NEVENTS, ("count (%d) > KQ_NEVENTS", count));
883	uap = (struct kevent_args *)arg;
884
885	error = copyout(kevp, uap->eventlist, count * sizeof *kevp);
886	if (error == 0)
887	uap->eventlist += count;
888	return (error);
889	}
890
891	/*
892	* Copy 'count' items from the list pointed to by uap->changelist.
893	*/
894	static int
895	kevent_copyin(void arg, struct kevent kevp, int count)
896	{
897	struct kevent_args *uap;
898	int error;
899
900	KASSERT(count <= KQ_NEVENTS, ("count (%d) > KQ_NEVENTS", count));
901	uap = (struct kevent_args *)arg;
902
903	error = copyin(uap->changelist, kevp, count * sizeof *kevp);
904	if (error == 0)
905	uap->changelist += count;
906	return (error);
907	}
908
909	int
910	kern_kevent(struct thread *td, int fd, int nchanges, int nevents,
911	struct kevent_copyops k_ops, const struct timespec timeout)
912	{
913	struct kevent keva[KQ_NEVENTS];
914	struct kevent kevp, changes;
915	struct kqueue *kq;
916	struct file *fp;
917	int i, n, nerrors, error;
918
919	if ((error = fget(td, fd, &fp)) != 0)
920	return (error);
921	if ((error = kqueue_acquire(fp, &kq)) != 0)
922	goto done_norel;
923
924	nerrors = 0;
925
926	while (nchanges > 0) {
927	n = nchanges > KQ_NEVENTS ? KQ_NEVENTS : nchanges;
928	error = k_ops->k_copyin(k_ops->arg, keva, n);
929	if (error)
930	goto done;
931	changes = keva;
932	for (i = 0; i < n; i++) {
933	kevp = &changes[i];
934	if (!kevp->filter)
935	continue;
936	kevp->flags &= ~EV_SYSFLAGS;
937	error = kqueue_register(kq, kevp, td, 1);
938	if (error \|\| (kevp->flags & EV_RECEIPT)) {
939	if (nevents != 0) {
940	kevp->flags = EV_ERROR;
941	kevp->data = error;
942	(void) k_ops->k_copyout(k_ops->arg,
943	kevp, 1);
944	nevents--;
945	nerrors++;
946	} else {
947	goto done;
948	}
949	}
950	}
951	nchanges -= n;
952	}
953	if (nerrors) {
954	td->td_retval[0] = nerrors;
955	error = 0;
956	goto done;
957	}
958
959	error = kqueue_scan(kq, nevents, k_ops, timeout, keva, td);
960	done:
961	kqueue_release(kq, 0);
962	done_norel:
963	fdrop(fp, td);
964	return (error);
965	}
966
967	int
968	kqueue_add_filteropts(int filt, struct filterops *filtops)
969	{
970	int error;
971
972	error = 0;
973	if (filt > 0 \|\| filt + EVFILT_SYSCOUNT < 0) {
974	printf(
975	"trying to add a filterop that is out of range: %d is beyond %d\n",
976	~filt, EVFILT_SYSCOUNT);
977	return EINVAL;
978	}
979	mtx_lock(&filterops_lock);
980	if (sysfilt_ops[~filt].for_fop != &null_filtops &&
981	sysfilt_ops[~filt].for_fop != NULL)
982	error = EEXIST;
983	else {
984	sysfilt_ops[~filt].for_fop = filtops;
985	sysfilt_ops[~filt].for_refcnt = 0;
986	}
987	mtx_unlock(&filterops_lock);
988
989	return (error);
990	}
991
992	int
993	kqueue_del_filteropts(int filt)
994	{
995	int error;
996
997	error = 0;
998	if (filt > 0 \|\| filt + EVFILT_SYSCOUNT < 0)
999	return EINVAL;
1000
1001	mtx_lock(&filterops_lock);
1002	if (sysfilt_ops[~filt].for_fop == &null_filtops \|\|
1003	sysfilt_ops[~filt].for_fop == NULL)
1004	error = EINVAL;
1005	else if (sysfilt_ops[~filt].for_refcnt != 0)
1006	error = EBUSY;
1007	else {
1008	sysfilt_ops[~filt].for_fop = &null_filtops;
1009	sysfilt_ops[~filt].for_refcnt = 0;
1010	}
1011	mtx_unlock(&filterops_lock);
1012
1013	return error;
1014	}
1015
1016	static struct filterops *
1017	kqueue_fo_find(int filt)
1018	{
1019
1020	if (filt > 0 \|\| filt + EVFILT_SYSCOUNT < 0)
1021	return NULL;
1022
1023	mtx_lock(&filterops_lock);
1024	sysfilt_ops[~filt].for_refcnt++;
1025	if (sysfilt_ops[~filt].for_fop == NULL)
1026	sysfilt_ops[~filt].for_fop = &null_filtops;
1027	mtx_unlock(&filterops_lock);
1028
1029	return sysfilt_ops[~filt].for_fop;
1030	}
1031
1032	static void
1033	kqueue_fo_release(int filt)
1034	{
1035
1036	if (filt > 0 \|\| filt + EVFILT_SYSCOUNT < 0)
1037	return;
1038
1039	mtx_lock(&filterops_lock);
1040	KASSERT(sysfilt_ops[~filt].for_refcnt > 0,
1041	("filter object refcount not valid on release"));
1042	sysfilt_ops[~filt].for_refcnt--;
1043	mtx_unlock(&filterops_lock);
1044	}
1045
1046	/*
1047	* A ref to kq (obtained via kqueue_acquire) must be held. waitok will
1048	* influence if memory allocation should wait. Make sure it is 0 if you
1049	* hold any mutexes.
1050	*/
1051	static int
1052	kqueue_register(struct kqueue kq, struct kevent kev, struct thread *td, int waitok)
1053	{
1054	struct filterops *fops;
1055	struct file *fp;
1056	struct knote kn, tkn;
1057	int error, filt, event;
1058	int haskqglobal;
1059
1060	fp = NULL;
1061	kn = NULL;
1062	error = 0;
1063	haskqglobal = 0;
1064
1065	filt = kev->filter;
1066	fops = kqueue_fo_find(filt);
1067	if (fops == NULL)
1068	return EINVAL;
1069
1070	tkn = knote_alloc(waitok); /* prevent waiting with locks */
1071
1072	findkn:
1073	if (fops->f_isfd) {
1074	KASSERT(td != NULL, ("td is NULL"));
1075	error = fget(td, kev->ident, &fp);
1076	if (error)
1077	goto done;
1078
1079	if ((kev->flags & EV_ADD) == EV_ADD && kqueue_expand(kq, fops,
1080	kev->ident, 0) != 0) {
1081	/* try again */
1082	fdrop(fp, td);
1083	fp = NULL;
1084	error = kqueue_expand(kq, fops, kev->ident, waitok);
1085	if (error)
1086	goto done;
1087	goto findkn;
1088	}
1089
1090	#ifndef __rtems__
1091	if (fp->f_type == DTYPE_KQUEUE) {
1092	#else /* __rtems__ */
1093	if (fp->f_io.pathinfo.handlers == &kqueueops) {
1094	#endif /* __rtems__ */
1095	/*
1096	* if we add some inteligence about what we are doing,
1097	* we should be able to support events on ourselves.
1098	* We need to know when we are doing this to prevent
1099	* getting both the knlist lock and the kq lock since
1100	* they are the same thing.
1101	*/
1102	if (fp->f_data == kq) {
1103	error = EINVAL;
1104	goto done;
1105	}
1106
1107	KQ_GLOBAL_LOCK(&kq_global, haskqglobal);
1108	}
1109
1110	KQ_LOCK(kq);
1111	if (kev->ident < kq->kq_knlistsize) {
1112	SLIST_FOREACH(kn, &kq->kq_knlist[kev->ident], kn_link)
1113	if (kev->filter == kn->kn_filter)
1114	break;
1115	}
1116	} else {
1117	if ((kev->flags & EV_ADD) == EV_ADD)
1118	kqueue_expand(kq, fops, kev->ident, waitok);
1119
1120	KQ_LOCK(kq);
1121	if (kq->kq_knhashmask != 0) {
1122	struct klist *list;
1123
1124	list = &kq->kq_knhash[
1125	KN_HASH((u_long)kev->ident, kq->kq_knhashmask)];
1126	SLIST_FOREACH(kn, list, kn_link)
1127	if (kev->ident == kn->kn_id &&
1128	kev->filter == kn->kn_filter)
1129	break;
1130	}
1131	}
1132
1133	/* knote is in the process of changing, wait for it to stablize. */
1134	if (kn != NULL && (kn->kn_status & KN_INFLUX) == KN_INFLUX) {
1135	KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
1136	kq->kq_state \|= KQ_FLUXWAIT;
1137	msleep(kq, &kq->kq_lock, PSOCK \| PDROP, "kqflxwt", 0);
1138	if (fp != NULL) {
1139	fdrop(fp, td);
1140	fp = NULL;
1141	}
1142	goto findkn;
1143	}
1144
1145	/*
1146	* kn now contains the matching knote, or NULL if no match
1147	*/
1148	if (kn == NULL) {
1149	if (kev->flags & EV_ADD) {
1150	kn = tkn;
1151	tkn = NULL;
1152	if (kn == NULL) {
1153	KQ_UNLOCK(kq);
1154	error = ENOMEM;
1155	goto done;
1156	}
1157	kn->kn_fp = fp;
1158	kn->kn_kq = kq;
1159	kn->kn_fop = fops;
1160	/*
1161	* apply reference counts to knote structure, and
1162	* do not release it at the end of this routine.
1163	*/
1164	fops = NULL;
1165	fp = NULL;
1166
1167	kn->kn_sfflags = kev->fflags;
1168	kn->kn_sdata = kev->data;
1169	kev->fflags = 0;
1170	kev->data = 0;
1171	kn->kn_kevent = *kev;
1172	kn->kn_kevent.flags &= ~(EV_ADD \| EV_DELETE \|
1173	EV_ENABLE \| EV_DISABLE);
1174	kn->kn_status = KN_INFLUX\|KN_DETACHED;
1175
1176	error = knote_attach(kn, kq);
1177	KQ_UNLOCK(kq);
1178	if (error != 0) {
1179	tkn = kn;
1180	goto done;
1181	}
1182
1183	if ((error = kn->kn_fop->f_attach(kn)) != 0) {
1184	knote_drop(kn, td);
1185	goto done;
1186	}
1187	KN_LIST_LOCK(kn);
1188	goto done_ev_add;
1189	} else {
1190	/* No matching knote and the EV_ADD flag is not set. */
1191	KQ_UNLOCK(kq);
1192	error = ENOENT;
1193	goto done;
1194	}
1195	}
1196
1197	if (kev->flags & EV_DELETE) {
1198	kn->kn_status \|= KN_INFLUX;
1199	KQ_UNLOCK(kq);
1200	if (!(kn->kn_status & KN_DETACHED))
1201	kn->kn_fop->f_detach(kn);
1202	knote_drop(kn, td);
1203	goto done;
1204	}
1205
1206	/*
1207	* The user may change some filter values after the initial EV_ADD,
1208	* but doing so will not reset any filter which has already been
1209	* triggered.
1210	*/
1211	kn->kn_status \|= KN_INFLUX;
1212	KQ_UNLOCK(kq);
1213	KN_LIST_LOCK(kn);
1214	kn->kn_kevent.udata = kev->udata;
1215	if (!fops->f_isfd && fops->f_touch != NULL) {
1216	fops->f_touch(kn, kev, EVENT_REGISTER);
1217	} else {
1218	kn->kn_sfflags = kev->fflags;
1219	kn->kn_sdata = kev->data;
1220	}
1221
1222	/*
1223	* We can get here with kn->kn_knlist == NULL. This can happen when
1224	* the initial attach event decides that the event is "completed"
1225	* already. i.e. filt_procattach is called on a zombie process. It
1226	* will call filt_proc which will remove it from the list, and NULL
1227	* kn_knlist.
1228	*/
1229	done_ev_add:
1230	event = kn->kn_fop->f_event(kn, 0);
1231	KQ_LOCK(kq);
1232	if (event)
1233	KNOTE_ACTIVATE(kn, 1);
1234	kn->kn_status &= ~KN_INFLUX;
1235	KN_LIST_UNLOCK(kn);
1236
1237	if ((kev->flags & EV_DISABLE) &&
1238	((kn->kn_status & KN_DISABLED) == 0)) {
1239	kn->kn_status \|= KN_DISABLED;
1240	}
1241
1242	if ((kev->flags & EV_ENABLE) && (kn->kn_status & KN_DISABLED)) {
1243	kn->kn_status &= ~KN_DISABLED;
1244	if ((kn->kn_status & KN_ACTIVE) &&
1245	((kn->kn_status & KN_QUEUED) == 0))
1246	knote_enqueue(kn);
1247	}
1248	KQ_UNLOCK_FLUX(kq);
1249
1250	done:
1251	KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
1252	if (fp != NULL)
1253	fdrop(fp, td);
1254	if (tkn != NULL)
1255	knote_free(tkn);
1256	if (fops != NULL)
1257	kqueue_fo_release(filt);
1258	return (error);
1259	}
1260
1261	static int
1262	kqueue_acquire(struct file fp, struct kqueue *kqp)
1263	{
1264	int error;
1265	struct kqueue *kq;
1266
1267	error = 0;
1268
1269	kq = fp->f_data;
1270	#ifndef __rtems__
1271	if (fp->f_type != DTYPE_KQUEUE \|\| kq == NULL)
1272	#else /* __rtems__ */
1273	if (fp->f_io.pathinfo.handlers != &kqueueops \|\| kq == NULL)
1274	#endif /* __rtems__ */
1275	return (EBADF);
1276	*kqp = kq;
1277	KQ_LOCK(kq);
1278	if ((kq->kq_state & KQ_CLOSING) == KQ_CLOSING) {
1279	KQ_UNLOCK(kq);
1280	return (EBADF);
1281	}
1282	kq->kq_refcnt++;
1283	KQ_UNLOCK(kq);
1284
1285	return error;
1286	}
1287
1288	static void
1289	kqueue_release(struct kqueue *kq, int locked)
1290	{
1291	if (locked)
1292	KQ_OWNED(kq);
1293	else
1294	KQ_LOCK(kq);
1295	kq->kq_refcnt--;
1296	if (kq->kq_refcnt == 1)
1297	wakeup(&kq->kq_refcnt);
1298	if (!locked)
1299	KQ_UNLOCK(kq);
1300	}
1301
1302	static void
1303	kqueue_schedtask(struct kqueue *kq)
1304	{
1305
1306	KQ_OWNED(kq);
1307	KASSERT(((kq->kq_state & KQ_TASKDRAIN) != KQ_TASKDRAIN),
1308	("scheduling kqueue task while draining"));
1309
1310	if ((kq->kq_state & KQ_TASKSCHED) != KQ_TASKSCHED) {
1311	taskqueue_enqueue(taskqueue_kqueue, &kq->kq_task);
1312	kq->kq_state \|= KQ_TASKSCHED;
1313	}
1314	}
1315
1316	/*
1317	* Expand the kq to make sure we have storage for fops/ident pair.
1318	*
1319	* Return 0 on success (or no work necessary), return errno on failure.
1320	*
1321	* Not calling hashinit w/ waitok (proper malloc flag) should be safe.
1322	* If kqueue_register is called from a non-fd context, there usually/should
1323	* be no locks held.
1324	*/
1325	static int
1326	kqueue_expand(struct kqueue kq, struct filterops fops, uintptr_t ident,
1327	int waitok)
1328	{
1329	struct klist list, tmp_knhash, *to_free;
1330	u_long tmp_knhashmask;
1331	int size;
1332	int fd;
1333	int mflag = waitok ? M_WAITOK : M_NOWAIT;
1334
1335	KQ_NOTOWNED(kq);
1336
1337	to_free = NULL;
1338	if (fops->f_isfd) {
1339	fd = ident;
1340	if (kq->kq_knlistsize <= fd) {
1341	size = kq->kq_knlistsize;
1342	while (size <= fd)
1343	size += KQEXTENT;
1344	list = malloc(size * sizeof list, M_KQUEUE, mflag);
1345	if (list == NULL)
1346	return ENOMEM;
1347	KQ_LOCK(kq);
1348	if (kq->kq_knlistsize > fd) {
1349	to_free = list;
1350	list = NULL;
1351	} else {
1352	if (kq->kq_knlist != NULL) {
1353	bcopy(kq->kq_knlist, list,
1354	kq->kq_knlistsize * sizeof list);
1355	to_free = kq->kq_knlist;
1356	kq->kq_knlist = NULL;
1357	}
1358	bzero((caddr_t)list +
1359	kq->kq_knlistsize * sizeof list,
1360	(size - kq->kq_knlistsize) * sizeof list);
1361	kq->kq_knlistsize = size;
1362	kq->kq_knlist = list;
1363	}
1364	KQ_UNLOCK(kq);
1365	}
1366	} else {
1367	if (kq->kq_knhashmask == 0) {
1368	tmp_knhash = hashinit(KN_HASHSIZE, M_KQUEUE,
1369	&tmp_knhashmask);
1370	if (tmp_knhash == NULL)
1371	return ENOMEM;
1372	KQ_LOCK(kq);
1373	if (kq->kq_knhashmask == 0) {
1374	kq->kq_knhash = tmp_knhash;
1375	kq->kq_knhashmask = tmp_knhashmask;
1376	} else {
1377	to_free = tmp_knhash;
1378	}
1379	KQ_UNLOCK(kq);
1380	}
1381	}
1382	free(to_free, M_KQUEUE);
1383
1384	KQ_NOTOWNED(kq);
1385	return 0;
1386	}
1387
1388	static void
1389	kqueue_task(void *arg, int pending)
1390	{
1391	struct kqueue *kq;
1392	int haskqglobal;
1393
1394	haskqglobal = 0;
1395	kq = arg;
1396
1397	KQ_GLOBAL_LOCK(&kq_global, haskqglobal);
1398	KQ_LOCK(kq);
1399
1400	KNOTE_LOCKED(&kq->kq_sel.si_note, 0);
1401
1402	kq->kq_state &= ~KQ_TASKSCHED;
1403	if ((kq->kq_state & KQ_TASKDRAIN) == KQ_TASKDRAIN) {
1404	wakeup(&kq->kq_state);
1405	}
1406	KQ_UNLOCK(kq);
1407	KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
1408	}
1409
1410	/*
1411	* Scan, update kn_data (if not ONESHOT), and copyout triggered events.
1412	* We treat KN_MARKER knotes as if they are INFLUX.
1413	*/
1414	static int
1415	kqueue_scan(struct kqueue kq, int maxevents, struct kevent_copyops k_ops,
1416	const struct timespec tsp, struct kevent keva, struct thread *td)
1417	{
1418	struct kevent *kevp;
1419	struct timeval atv, rtv, ttv;
1420	struct knote kn, marker;
1421	int count, timeout, nkev, error, influx;
1422	int haskqglobal, touch;
1423
1424	count = maxevents;
1425	nkev = 0;
1426	error = 0;
1427	haskqglobal = 0;
1428
1429	if (maxevents == 0)
1430	goto done_nl;
1431
1432	if (tsp != NULL) {
1433	TIMESPEC_TO_TIMEVAL(&atv, tsp);
1434	if (itimerfix(&atv)) {
1435	error = EINVAL;
1436	goto done_nl;
1437	}
1438	if (tsp->tv_sec == 0 && tsp->tv_nsec == 0)
1439	timeout = -1;
1440	else
1441	timeout = atv.tv_sec > 24 * 60 * 60 ?
1442	24 * 60 * 60 * hz : tvtohz(&atv);
1443	getmicrouptime(&rtv);
1444	timevaladd(&atv, &rtv);
1445	} else {
1446	atv.tv_sec = 0;
1447	atv.tv_usec = 0;
1448	timeout = 0;
1449	}
1450	marker = knote_alloc(1);
1451	if (marker == NULL) {
1452	error = ENOMEM;
1453	goto done_nl;
1454	}
1455	marker->kn_status = KN_MARKER;
1456	KQ_LOCK(kq);
1457	goto start;
1458
1459	retry:
1460	if (atv.tv_sec \|\| atv.tv_usec) {
1461	getmicrouptime(&rtv);
1462	if (timevalcmp(&rtv, &atv, >=))
1463	goto done;
1464	ttv = atv;
1465	timevalsub(&ttv, &rtv);
1466	timeout = ttv.tv_sec > 24 * 60 * 60 ?
1467	24 * 60 * 60 * hz : tvtohz(&ttv);
1468	}
1469
1470	start:
1471	kevp = keva;
1472	if (kq->kq_count == 0) {
1473	if (timeout < 0) {
1474	error = EWOULDBLOCK;
1475	} else {
1476	kq->kq_state \|= KQ_SLEEP;
1477	error = msleep(kq, &kq->kq_lock, PSOCK \| PCATCH,
1478	"kqread", timeout);
1479	}
1480	if (error == 0)
1481	goto retry;
1482	/* don't restart after signals... */
1483	if (error == ERESTART)
1484	error = EINTR;
1485	else if (error == EWOULDBLOCK)
1486	error = 0;
1487	goto done;
1488	}
1489
1490	TAILQ_INSERT_TAIL(&kq->kq_head, marker, kn_tqe);
1491	influx = 0;
1492	while (count) {
1493	KQ_OWNED(kq);
1494	kn = TAILQ_FIRST(&kq->kq_head);
1495
1496	if ((kn->kn_status == KN_MARKER && kn != marker) \|\|
1497	(kn->kn_status & KN_INFLUX) == KN_INFLUX) {
1498	if (influx) {
1499	influx = 0;
1500	KQ_FLUX_WAKEUP(kq);
1501	}
1502	kq->kq_state \|= KQ_FLUXWAIT;
1503	error = msleep(kq, &kq->kq_lock, PSOCK,
1504	"kqflxwt", 0);
1505	continue;
1506	}
1507
1508	TAILQ_REMOVE(&kq->kq_head, kn, kn_tqe);
1509	if ((kn->kn_status & KN_DISABLED) == KN_DISABLED) {
1510	kn->kn_status &= ~KN_QUEUED;
1511	kq->kq_count--;
1512	continue;
1513	}
1514	if (kn == marker) {
1515	KQ_FLUX_WAKEUP(kq);
1516	if (count == maxevents)
1517	goto retry;
1518	goto done;
1519	}
1520	KASSERT((kn->kn_status & KN_INFLUX) == 0,
1521	("KN_INFLUX set when not suppose to be"));
1522
1523	if ((kn->kn_flags & EV_ONESHOT) == EV_ONESHOT) {
1524	kn->kn_status &= ~KN_QUEUED;
1525	kn->kn_status \|= KN_INFLUX;
1526	kq->kq_count--;
1527	KQ_UNLOCK(kq);
1528	/*
1529	* We don't need to lock the list since we've marked
1530	* it _INFLUX.
1531	*/
1532	*kevp = kn->kn_kevent;
1533	if (!(kn->kn_status & KN_DETACHED))
1534	kn->kn_fop->f_detach(kn);
1535	knote_drop(kn, td);
1536	KQ_LOCK(kq);
1537	kn = NULL;
1538	} else {
1539	kn->kn_status \|= KN_INFLUX;
1540	KQ_UNLOCK(kq);
1541	if ((kn->kn_status & KN_KQUEUE) == KN_KQUEUE)
1542	KQ_GLOBAL_LOCK(&kq_global, haskqglobal);
1543	KN_LIST_LOCK(kn);
1544	if (kn->kn_fop->f_event(kn, 0) == 0) {
1545	KQ_LOCK(kq);
1546	KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
1547	kn->kn_status &=
1548	~(KN_QUEUED \| KN_ACTIVE \| KN_INFLUX);
1549	kq->kq_count--;
1550	KN_LIST_UNLOCK(kn);
1551	influx = 1;
1552	continue;
1553	}
1554	touch = (!kn->kn_fop->f_isfd &&
1555	kn->kn_fop->f_touch != NULL);
1556	if (touch)
1557	kn->kn_fop->f_touch(kn, kevp, EVENT_PROCESS);
1558	else
1559	*kevp = kn->kn_kevent;
1560	KQ_LOCK(kq);
1561	KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
1562	if (kn->kn_flags & (EV_CLEAR \| EV_DISPATCH)) {
1563	/*
1564	* Manually clear knotes who weren't
1565	* 'touch'ed.
1566	*/
1567	if (touch == 0 && kn->kn_flags & EV_CLEAR) {
1568	kn->kn_data = 0;
1569	kn->kn_fflags = 0;
1570	}
1571	if (kn->kn_flags & EV_DISPATCH)
1572	kn->kn_status \|= KN_DISABLED;
1573	kn->kn_status &= ~(KN_QUEUED \| KN_ACTIVE);
1574	kq->kq_count--;
1575	} else
1576	TAILQ_INSERT_TAIL(&kq->kq_head, kn, kn_tqe);
1577
1578	kn->kn_status &= ~(KN_INFLUX);
1579	KN_LIST_UNLOCK(kn);
1580	influx = 1;
1581	}
1582
1583	/* we are returning a copy to the user */
1584	kevp++;
1585	nkev++;
1586	count--;
1587
1588	if (nkev == KQ_NEVENTS) {
1589	influx = 0;
1590	KQ_UNLOCK_FLUX(kq);
1591	error = k_ops->k_copyout(k_ops->arg, keva, nkev);
1592	nkev = 0;
1593	kevp = keva;
1594	KQ_LOCK(kq);
1595	if (error)
1596	break;
1597	}
1598	}
1599	TAILQ_REMOVE(&kq->kq_head, marker, kn_tqe);
1600	done:
1601	KQ_OWNED(kq);
1602	KQ_UNLOCK_FLUX(kq);
1603	knote_free(marker);
1604	done_nl:
1605	KQ_NOTOWNED(kq);
1606	if (nkev != 0)
1607	error = k_ops->k_copyout(k_ops->arg, keva, nkev);
1608	td->td_retval[0] = maxevents - count;
1609	return (error);
1610	}
1611
1612	#ifndef __rtems__
1613	/*
1614	* XXX
1615	* This could be expanded to call kqueue_scan, if desired.
1616	*/
1617	/ARGSUSED/
1618	static int
1619	kqueue_read(struct file fp, struct uio uio, struct ucred *active_cred,
1620	int flags, struct thread *td)
1621	{
1622	return (ENXIO);
1623	}
1624
1625	/ARGSUSED/
1626	static int
1627	kqueue_write(struct file fp, struct uio uio, struct ucred *active_cred,
1628	int flags, struct thread *td)
1629	{
1630	return (ENXIO);
1631	}
1632
1633	/ARGSUSED/
1634	static int
1635	kqueue_truncate(struct file fp, off_t length, struct ucred active_cred,
1636	struct thread *td)
1637	{
1638
1639	return (EINVAL);
1640	}
1641
1642	/ARGSUSED/
1643	static int
1644	kqueue_ioctl(struct file fp, u_long cmd, void data,
1645	struct ucred active_cred, struct thread td)
1646	{
1647	/*
1648	* Enabling sigio causes two major problems:
1649	* 1) infinite recursion:
1650	* Synopsys: kevent is being used to track signals and have FIOASYNC
1651	* set. On receipt of a signal this will cause a kqueue to recurse
1652	* into itself over and over. Sending the sigio causes the kqueue
1653	* to become ready, which in turn posts sigio again, forever.
1654	* Solution: this can be solved by setting a flag in the kqueue that
1655	* we have a SIGIO in progress.
1656	* 2) locking problems:
1657	* Synopsys: Kqueue is a leaf subsystem, but adding signalling puts
1658	* us above the proc and pgrp locks.
1659	* Solution: Post a signal using an async mechanism, being sure to
1660	* record a generation count in the delivery so that we do not deliver
1661	* a signal to the wrong process.
1662	*
1663	* Note, these two mechanisms are somewhat mutually exclusive!
1664	*/
1665	#if 0
1666	struct kqueue *kq;
1667
1668	kq = fp->f_data;
1669	switch (cmd) {
1670	case FIOASYNC:
1671	if ((int )data) {
1672	kq->kq_state \|= KQ_ASYNC;
1673	} else {
1674	kq->kq_state &= ~KQ_ASYNC;
1675	}
1676	return (0);
1677
1678	case FIOSETOWN:
1679	return (fsetown((int )data, &kq->kq_sigio));
1680
1681	case FIOGETOWN:
1682	(int )data = fgetown(&kq->kq_sigio);
1683	return (0);
1684	}
1685	#endif
1686
1687	return (ENOTTY);
1688	}
1689	#endif /* __rtems__ */
1690
1691	/ARGSUSED/
1692	static int
1693	kqueue_poll(struct file fp, int events, struct ucred active_cred,
1694	struct thread *td)
1695	{
1696	struct kqueue *kq;
1697	int revents = 0;
1698	int error;
1699
1700	if ((error = kqueue_acquire(fp, &kq)))
1701	return POLLERR;
1702
1703	KQ_LOCK(kq);
1704	if (events & (POLLIN \| POLLRDNORM)) {
1705	if (kq->kq_count) {
1706	revents \|= events & (POLLIN \| POLLRDNORM);
1707	} else {
1708	selrecord(td, &kq->kq_sel);
1709	if (SEL_WAITING(&kq->kq_sel))
1710	kq->kq_state \|= KQ_SEL;
1711	}
1712	}
1713	kqueue_release(kq, 1);
1714	KQ_UNLOCK(kq);
1715	return (revents);
1716	}
1717	#ifdef __rtems__
1718	static int
1719	rtems_bsd_kqueue_poll(rtems_libio_t *iop, int events)
1720	{
1721	struct thread *td = rtems_bsd_get_curthread_or_null();
1722	struct file *fp = rtems_bsd_iop_to_fp(iop);
1723	int error;
1724
1725	if (td != NULL) {
1726	error = kqueue_poll(fp, events, NULL, td);
1727	} else {
1728	error = ENOMEM;
1729	}
1730
1731	return error;
1732	}
1733	#endif /* __rtems__ */
1734
1735	/ARGSUSED/
1736	#ifndef __rtems__
1737	static int
1738	kqueue_stat(struct file fp, struct stat st, struct ucred *active_cred,
1739	struct thread *td)
1740	{
1741
1742	bzero((void )st, sizeof st);
1743	#else /* __rtems__ */
1744	static int
1745	rtems_bsd_kqueue_stat(const rtems_filesystem_location_info_t *loc,
1746	struct stat *st)
1747	{
1748	(void) loc;
1749	#endif /* __rtems__ */
1750	/*
1751	* We no longer return kq_count because the unlocked value is useless.
1752	* If you spent all this time getting the count, why not spend your
1753	* syscall better by calling kevent?
1754	*
1755	* XXX - This is needed for libc_r.
1756	*/
1757	st->st_mode = S_IFIFO;
1758	return (0);
1759	}
1760
1761	/ARGSUSED/
1762	static int
1763	kqueue_close(struct file fp, struct thread td)
1764	{
1765	struct kqueue *kq = fp->f_data;
1766	struct filedesc *fdp;
1767	struct knote *kn;
1768	int i;
1769	int error;
1770
1771	#ifdef __rtems__
1772	/* FIXME: Move this to the RTEMS close() function */
1773	knote_fdclose(td, rtems_bsd_fp_to_fd(fp));
1774	#endif /* __rtems__ */
1775
1776	if ((error = kqueue_acquire(fp, &kq)))
1777	return error;
1778
1779	KQ_LOCK(kq);
1780
1781	KASSERT((kq->kq_state & KQ_CLOSING) != KQ_CLOSING,
1782	("kqueue already closing"));
1783	kq->kq_state \|= KQ_CLOSING;
1784	if (kq->kq_refcnt > 1)
1785	msleep(&kq->kq_refcnt, &kq->kq_lock, PSOCK, "kqclose", 0);
1786
1787	KASSERT(kq->kq_refcnt == 1, ("other refs are out there!"));
1788	#ifndef __rtems__
1789	fdp = kq->kq_fdp;
1790	#else /* __rtems__ */
1791	(void) fdp;
1792	#endif /* __rtems__ */
1793
1794	KASSERT(knlist_empty(&kq->kq_sel.si_note),
1795	("kqueue's knlist not empty"));
1796
1797	for (i = 0; i < kq->kq_knlistsize; i++) {
1798	while ((kn = SLIST_FIRST(&kq->kq_knlist[i])) != NULL) {
1799	if ((kn->kn_status & KN_INFLUX) == KN_INFLUX) {
1800	kq->kq_state \|= KQ_FLUXWAIT;
1801	msleep(kq, &kq->kq_lock, PSOCK, "kqclo1", 0);
1802	continue;
1803	}
1804	kn->kn_status \|= KN_INFLUX;
1805	KQ_UNLOCK(kq);
1806	if (!(kn->kn_status & KN_DETACHED))
1807	kn->kn_fop->f_detach(kn);
1808	knote_drop(kn, td);
1809	KQ_LOCK(kq);
1810	}
1811	}
1812	if (kq->kq_knhashmask != 0) {
1813	for (i = 0; i <= kq->kq_knhashmask; i++) {
1814	while ((kn = SLIST_FIRST(&kq->kq_knhash[i])) != NULL) {
1815	if ((kn->kn_status & KN_INFLUX) == KN_INFLUX) {
1816	kq->kq_state \|= KQ_FLUXWAIT;
1817	msleep(kq, &kq->kq_lock, PSOCK,
1818	"kqclo2", 0);
1819	continue;
1820	}
1821	kn->kn_status \|= KN_INFLUX;
1822	KQ_UNLOCK(kq);
1823	if (!(kn->kn_status & KN_DETACHED))
1824	kn->kn_fop->f_detach(kn);
1825	knote_drop(kn, td);
1826	KQ_LOCK(kq);
1827	}
1828	}
1829	}
1830
1831	if ((kq->kq_state & KQ_TASKSCHED) == KQ_TASKSCHED) {
1832	kq->kq_state \|= KQ_TASKDRAIN;
1833	msleep(&kq->kq_state, &kq->kq_lock, PSOCK, "kqtqdr", 0);
1834	}
1835
1836	if ((kq->kq_state & KQ_SEL) == KQ_SEL) {
1837	selwakeuppri(&kq->kq_sel, PSOCK);
1838	if (!SEL_WAITING(&kq->kq_sel))
1839	kq->kq_state &= ~KQ_SEL;
1840	}
1841
1842	KQ_UNLOCK(kq);
1843
1844	#ifndef __rtems__
1845	FILEDESC_XLOCK(fdp);
1846	SLIST_REMOVE(&fdp->fd_kqlist, kq, kqueue, kq_list);
1847	FILEDESC_XUNLOCK(fdp);
1848	#else /* __rtems__ */
1849	rtems_libio_lock();
1850	SLIST_REMOVE(&fd_kqlist, kq, kqueue, kq_list);
1851	rtems_libio_unlock();
1852	#endif /* __rtems__ */
1853
1854	knlist_destroy(&kq->kq_sel.si_note);
1855	mtx_destroy(&kq->kq_lock);
1856	#ifndef __rtems__
1857	kq->kq_fdp = NULL;
1858	#endif /* __rtems__ */
1859
1860	if (kq->kq_knhash != NULL)
1861	free(kq->kq_knhash, M_KQUEUE);
1862	if (kq->kq_knlist != NULL)
1863	free(kq->kq_knlist, M_KQUEUE);
1864
1865	funsetown(&kq->kq_sigio);
1866	free(kq, M_KQUEUE);
1867	fp->f_data = NULL;
1868
1869	return (0);
1870	}
1871	#ifdef __rtems__
1872	static int
1873	rtems_bsd_kqueue_close(rtems_libio_t *iop)
1874	{
1875	struct thread *td = rtems_bsd_get_curthread_or_null();
1876	struct file *fp = rtems_bsd_iop_to_fp(iop);
1877	int error;
1878
1879	if (td != NULL) {
1880	error = kqueue_close(fp, td);
1881	} else {
1882	error = ENOMEM;
1883	}
1884
1885	return rtems_bsd_error_to_status_and_errno(error);
1886	}
1887	#endif /* __rtems__ */
1888
1889	static void
1890	kqueue_wakeup(struct kqueue *kq)
1891	{
1892	KQ_OWNED(kq);
1893
1894	if ((kq->kq_state & KQ_SLEEP) == KQ_SLEEP) {
1895	kq->kq_state &= ~KQ_SLEEP;
1896	wakeup(kq);
1897	}
1898	if ((kq->kq_state & KQ_SEL) == KQ_SEL) {
1899	selwakeuppri(&kq->kq_sel, PSOCK);
1900	if (!SEL_WAITING(&kq->kq_sel))
1901	kq->kq_state &= ~KQ_SEL;
1902	}
1903	if (!knlist_empty(&kq->kq_sel.si_note))
1904	kqueue_schedtask(kq);
1905	if ((kq->kq_state & KQ_ASYNC) == KQ_ASYNC) {
1906	#ifndef __rtems__
1907	pgsigio(&kq->kq_sigio, SIGIO, 0);
1908	#else /* __rtems__ */
1909	BSD_ASSERT(0);
1910	#endif /* __rtems__ */
1911	}
1912	}
1913
1914	/*
1915	* Walk down a list of knotes, activating them if their event has triggered.
1916	*
1917	* There is a possibility to optimize in the case of one kq watching another.
1918	* Instead of scheduling a task to wake it up, you could pass enough state
1919	* down the chain to make up the parent kqueue. Make this code functional
1920	* first.
1921	*/
1922	void
1923	knote(struct knlist *list, long hint, int lockflags)
1924	{
1925	struct kqueue *kq;
1926	struct knote *kn;
1927	int error;
1928
1929	if (list == NULL)
1930	return;
1931
1932	KNL_ASSERT_LOCK(list, lockflags & KNF_LISTLOCKED);
1933
1934	if ((lockflags & KNF_LISTLOCKED) == 0)
1935	list->kl_lock(list->kl_lockarg);
1936
1937	/*
1938	* If we unlock the list lock (and set KN_INFLUX), we can eliminate
1939	* the kqueue scheduling, but this will introduce four
1940	* lock/unlock's for each knote to test. If we do, continue to use
1941	* SLIST_FOREACH, SLIST_FOREACH_SAFE is not safe in our case, it is
1942	* only safe if you want to remove the current item, which we are
1943	* not doing.
1944	*/
1945	SLIST_FOREACH(kn, &list->kl_list, kn_selnext) {
1946	kq = kn->kn_kq;
1947	if ((kn->kn_status & KN_INFLUX) != KN_INFLUX) {
1948	KQ_LOCK(kq);
1949	if ((kn->kn_status & KN_INFLUX) == KN_INFLUX) {
1950	KQ_UNLOCK(kq);
1951	} else if ((lockflags & KNF_NOKQLOCK) != 0) {
1952	kn->kn_status \|= KN_INFLUX;
1953	KQ_UNLOCK(kq);
1954	error = kn->kn_fop->f_event(kn, hint);
1955	KQ_LOCK(kq);
1956	kn->kn_status &= ~KN_INFLUX;
1957	if (error)
1958	KNOTE_ACTIVATE(kn, 1);
1959	KQ_UNLOCK_FLUX(kq);
1960	} else {
1961	kn->kn_status \|= KN_HASKQLOCK;
1962	if (kn->kn_fop->f_event(kn, hint))
1963	KNOTE_ACTIVATE(kn, 1);
1964	kn->kn_status &= ~KN_HASKQLOCK;
1965	KQ_UNLOCK(kq);
1966	}
1967	}
1968	kq = NULL;
1969	}
1970	if ((lockflags & KNF_LISTLOCKED) == 0)
1971	list->kl_unlock(list->kl_lockarg);
1972	}
1973
1974	/*
1975	* add a knote to a knlist
1976	*/
1977	void
1978	knlist_add(struct knlist knl, struct knote kn, int islocked)
1979	{
1980	KNL_ASSERT_LOCK(knl, islocked);
1981	KQ_NOTOWNED(kn->kn_kq);
1982	KASSERT((kn->kn_status & (KN_INFLUX\|KN_DETACHED)) ==
1983	(KN_INFLUX\|KN_DETACHED), ("knote not KN_INFLUX and KN_DETACHED"));
1984	if (!islocked)
1985	knl->kl_lock(knl->kl_lockarg);
1986	SLIST_INSERT_HEAD(&knl->kl_list, kn, kn_selnext);
1987	if (!islocked)
1988	knl->kl_unlock(knl->kl_lockarg);
1989	KQ_LOCK(kn->kn_kq);
1990	kn->kn_knlist = knl;
1991	kn->kn_status &= ~KN_DETACHED;
1992	KQ_UNLOCK(kn->kn_kq);
1993	}
1994
1995	static void
1996	knlist_remove_kq(struct knlist knl, struct knote kn, int knlislocked, int kqislocked)
1997	{
1998	KASSERT(!(!!kqislocked && !knlislocked), ("kq locked w/o knl locked"));
1999	KNL_ASSERT_LOCK(knl, knlislocked);
2000	mtx_assert(&kn->kn_kq->kq_lock, kqislocked ? MA_OWNED : MA_NOTOWNED);
2001	if (!kqislocked)
2002	KASSERT((kn->kn_status & (KN_INFLUX\|KN_DETACHED)) == KN_INFLUX,
2003	("knlist_remove called w/o knote being KN_INFLUX or already removed"));
2004	if (!knlislocked)
2005	knl->kl_lock(knl->kl_lockarg);
2006	SLIST_REMOVE(&knl->kl_list, kn, knote, kn_selnext);
2007	kn->kn_knlist = NULL;
2008	if (!knlislocked)
2009	knl->kl_unlock(knl->kl_lockarg);
2010	if (!kqislocked)
2011	KQ_LOCK(kn->kn_kq);
2012	kn->kn_status \|= KN_DETACHED;
2013	if (!kqislocked)
2014	KQ_UNLOCK(kn->kn_kq);
2015	}
2016
2017	/*
2018	* remove all knotes from a specified klist
2019	*/
2020	void
2021	knlist_remove(struct knlist knl, struct knote kn, int islocked)
2022	{
2023
2024	knlist_remove_kq(knl, kn, islocked, 0);
2025	}
2026
2027	/*
2028	* remove knote from a specified klist while in f_event handler.
2029	*/
2030	void
2031	knlist_remove_inevent(struct knlist knl, struct knote kn)
2032	{
2033
2034	knlist_remove_kq(knl, kn, 1,
2035	(kn->kn_status & KN_HASKQLOCK) == KN_HASKQLOCK);
2036	}
2037
2038	int
2039	knlist_empty(struct knlist *knl)
2040	{
2041	KNL_ASSERT_LOCKED(knl);
2042	return SLIST_EMPTY(&knl->kl_list);
2043	}
2044
2045	static struct mtx knlist_lock;
2046	MTX_SYSINIT(knlist_lock, &knlist_lock, "knlist lock for lockless objects",
2047	MTX_DEF);
2048	static void knlist_mtx_lock(void *arg);
2049	static void knlist_mtx_unlock(void *arg);
2050
2051	static void
2052	knlist_mtx_lock(void *arg)
2053	{
2054	mtx_lock((struct mtx *)arg);
2055	}
2056
2057	static void
2058	knlist_mtx_unlock(void *arg)
2059	{
2060	mtx_unlock((struct mtx *)arg);
2061	}
2062
2063	static void
2064	knlist_mtx_assert_locked(void *arg)
2065	{
2066	mtx_assert((struct mtx *)arg, MA_OWNED);
2067	}
2068
2069	static void
2070	knlist_mtx_assert_unlocked(void *arg)
2071	{
2072	mtx_assert((struct mtx *)arg, MA_NOTOWNED);
2073	}
2074
2075	void
2076	knlist_init(struct knlist knl, void lock, void (kl_lock)(void ),
2077	void (kl_unlock)(void ),
2078	void (kl_assert_locked)(void ), void (kl_assert_unlocked)(void ))
2079	{
2080
2081	if (lock == NULL)
2082	knl->kl_lockarg = &knlist_lock;
2083	else
2084	knl->kl_lockarg = lock;
2085
2086	if (kl_lock == NULL)
2087	knl->kl_lock = knlist_mtx_lock;
2088	else
2089	knl->kl_lock = kl_lock;
2090	if (kl_unlock == NULL)
2091	knl->kl_unlock = knlist_mtx_unlock;
2092	else
2093	knl->kl_unlock = kl_unlock;
2094	if (kl_assert_locked == NULL)
2095	knl->kl_assert_locked = knlist_mtx_assert_locked;
2096	else
2097	knl->kl_assert_locked = kl_assert_locked;
2098	if (kl_assert_unlocked == NULL)
2099	knl->kl_assert_unlocked = knlist_mtx_assert_unlocked;
2100	else
2101	knl->kl_assert_unlocked = kl_assert_unlocked;
2102
2103	SLIST_INIT(&knl->kl_list);
2104	}
2105
2106	void
2107	knlist_init_mtx(struct knlist knl, struct mtx lock)
2108	{
2109
2110	knlist_init(knl, lock, NULL, NULL, NULL, NULL);
2111	}
2112
2113	void
2114	knlist_destroy(struct knlist *knl)
2115	{
2116
2117	#ifdef INVARIANTS
2118	/*
2119	* if we run across this error, we need to find the offending
2120	* driver and have it call knlist_clear.
2121	*/
2122	if (!SLIST_EMPTY(&knl->kl_list))
2123	printf("WARNING: destroying knlist w/ knotes on it!\n");
2124	#endif
2125
2126	knl->kl_lockarg = knl->kl_lock = knl->kl_unlock = NULL;
2127	SLIST_INIT(&knl->kl_list);
2128	}
2129
2130	/*
2131	* Even if we are locked, we may need to drop the lock to allow any influx
2132	* knotes time to "settle".
2133	*/
2134	void
2135	knlist_cleardel(struct knlist knl, struct thread td, int islocked, int killkn)
2136	{
2137	struct knote kn, kn2;
2138	struct kqueue *kq;
2139
2140	if (islocked)
2141	KNL_ASSERT_LOCKED(knl);
2142	else {
2143	KNL_ASSERT_UNLOCKED(knl);
2144	again: /* need to reacquire lock since we have dropped it */
2145	knl->kl_lock(knl->kl_lockarg);
2146	}
2147
2148	SLIST_FOREACH_SAFE(kn, &knl->kl_list, kn_selnext, kn2) {
2149	kq = kn->kn_kq;
2150	KQ_LOCK(kq);
2151	if ((kn->kn_status & KN_INFLUX)) {
2152	KQ_UNLOCK(kq);
2153	continue;
2154	}
2155	knlist_remove_kq(knl, kn, 1, 1);
2156	if (killkn) {
2157	kn->kn_status \|= KN_INFLUX \| KN_DETACHED;
2158	KQ_UNLOCK(kq);
2159	knote_drop(kn, td);
2160	} else {
2161	/* Make sure cleared knotes disappear soon */
2162	kn->kn_flags \|= (EV_EOF \| EV_ONESHOT);
2163	KQ_UNLOCK(kq);
2164	}
2165	kq = NULL;
2166	}
2167
2168	if (!SLIST_EMPTY(&knl->kl_list)) {
2169	/* there are still KN_INFLUX remaining */
2170	kn = SLIST_FIRST(&knl->kl_list);
2171	kq = kn->kn_kq;
2172	KQ_LOCK(kq);
2173	KASSERT(kn->kn_status & KN_INFLUX,
2174	("knote removed w/o list lock"));
2175	knl->kl_unlock(knl->kl_lockarg);
2176	kq->kq_state \|= KQ_FLUXWAIT;
2177	msleep(kq, &kq->kq_lock, PSOCK \| PDROP, "kqkclr", 0);
2178	kq = NULL;
2179	goto again;
2180	}
2181
2182	if (islocked)
2183	KNL_ASSERT_LOCKED(knl);
2184	else {
2185	knl->kl_unlock(knl->kl_lockarg);
2186	KNL_ASSERT_UNLOCKED(knl);
2187	}
2188	}
2189
2190	/*
2191	* Remove all knotes referencing a specified fd must be called with FILEDESC
2192	* lock. This prevents a race where a new fd comes along and occupies the
2193	* entry and we attach a knote to the fd.
2194	*/
2195	void
2196	knote_fdclose(struct thread *td, int fd)
2197	{
2198	#ifndef __rtems__
2199	struct filedesc *fdp = td->td_proc->p_fd;
2200	#endif /* __rtems__ */
2201	struct kqueue *kq;
2202	struct knote *kn;
2203	int influx;
2204
2205	FILEDESC_XLOCK_ASSERT(fdp);
2206
2207	/*
2208	* We shouldn't have to worry about new kevents appearing on fd
2209	* since filedesc is locked.
2210	*/
2211	#ifndef __rtems__
2212	SLIST_FOREACH(kq, &fdp->fd_kqlist, kq_list) {
2213	#else /* __rtems__ */
2214	/* FIXME: Use separate lock? */
2215	rtems_libio_lock();
2216	SLIST_FOREACH(kq, &fd_kqlist, kq_list) {
2217	#endif /* __rtems__ */
2218	KQ_LOCK(kq);
2219
2220	again:
2221	influx = 0;
2222	while (kq->kq_knlistsize > fd &&
2223	(kn = SLIST_FIRST(&kq->kq_knlist[fd])) != NULL) {
2224	if (kn->kn_status & KN_INFLUX) {
2225	/* someone else might be waiting on our knote */
2226	if (influx)
2227	wakeup(kq);
2228	kq->kq_state \|= KQ_FLUXWAIT;
2229	msleep(kq, &kq->kq_lock, PSOCK, "kqflxwt", 0);
2230	goto again;
2231	}
2232	kn->kn_status \|= KN_INFLUX;
2233	KQ_UNLOCK(kq);
2234	if (!(kn->kn_status & KN_DETACHED))
2235	kn->kn_fop->f_detach(kn);
2236	knote_drop(kn, td);
2237	influx = 1;
2238	KQ_LOCK(kq);
2239	}
2240	KQ_UNLOCK_FLUX(kq);
2241	}
2242	#ifdef __rtems__
2243	rtems_libio_unlock();
2244	#endif /* __rtems__ */
2245	}
2246
2247	static int
2248	knote_attach(struct knote kn, struct kqueue kq)
2249	{
2250	struct klist *list;
2251
2252	KASSERT(kn->kn_status & KN_INFLUX, ("knote not marked INFLUX"));
2253	KQ_OWNED(kq);
2254
2255	if (kn->kn_fop->f_isfd) {
2256	if (kn->kn_id >= kq->kq_knlistsize)
2257	return ENOMEM;
2258	list = &kq->kq_knlist[kn->kn_id];
2259	} else {
2260	if (kq->kq_knhash == NULL)
2261	return ENOMEM;
2262	list = &kq->kq_knhash[KN_HASH(kn->kn_id, kq->kq_knhashmask)];
2263	}
2264
2265	SLIST_INSERT_HEAD(list, kn, kn_link);
2266
2267	return 0;
2268	}
2269
2270	/*
2271	* knote must already have been detached using the f_detach method.
2272	* no lock need to be held, it is assumed that the KN_INFLUX flag is set
2273	* to prevent other removal.
2274	*/
2275	static void
2276	knote_drop(struct knote kn, struct thread td)
2277	{
2278	struct kqueue *kq;
2279	struct klist *list;
2280
2281	kq = kn->kn_kq;
2282
2283	KQ_NOTOWNED(kq);
2284	KASSERT((kn->kn_status & KN_INFLUX) == KN_INFLUX,
2285	("knote_drop called without KN_INFLUX set in kn_status"));
2286
2287	KQ_LOCK(kq);
2288	if (kn->kn_fop->f_isfd)
2289	list = &kq->kq_knlist[kn->kn_id];
2290	else
2291	list = &kq->kq_knhash[KN_HASH(kn->kn_id, kq->kq_knhashmask)];
2292
2293	if (!SLIST_EMPTY(list))
2294	SLIST_REMOVE(list, kn, knote, kn_link);
2295	if (kn->kn_status & KN_QUEUED)
2296	knote_dequeue(kn);
2297	KQ_UNLOCK_FLUX(kq);
2298
2299	if (kn->kn_fop->f_isfd) {
2300	fdrop(kn->kn_fp, td);
2301	kn->kn_fp = NULL;
2302	}
2303	kqueue_fo_release(kn->kn_kevent.filter);
2304	kn->kn_fop = NULL;
2305	knote_free(kn);
2306	}
2307
2308	static void
2309	knote_enqueue(struct knote *kn)
2310	{
2311	struct kqueue *kq = kn->kn_kq;
2312
2313	KQ_OWNED(kn->kn_kq);
2314	KASSERT((kn->kn_status & KN_QUEUED) == 0, ("knote already queued"));
2315
2316	TAILQ_INSERT_TAIL(&kq->kq_head, kn, kn_tqe);
2317	kn->kn_status \|= KN_QUEUED;
2318	kq->kq_count++;
2319	kqueue_wakeup(kq);
2320	}
2321
2322	static void
2323	knote_dequeue(struct knote *kn)
2324	{
2325	struct kqueue *kq = kn->kn_kq;
2326
2327	KQ_OWNED(kn->kn_kq);
2328	KASSERT(kn->kn_status & KN_QUEUED, ("knote not queued"));
2329
2330	TAILQ_REMOVE(&kq->kq_head, kn, kn_tqe);
2331	kn->kn_status &= ~KN_QUEUED;
2332	kq->kq_count--;
2333	}
2334
2335	static void
2336	knote_init(void)
2337	{
2338
2339	knote_zone = uma_zcreate("KNOTE", sizeof(struct knote), NULL, NULL,
2340	NULL, NULL, UMA_ALIGN_PTR, 0);
2341	}
2342	SYSINIT(knote, SI_SUB_PSEUDO, SI_ORDER_ANY, knote_init, NULL);
2343
2344	static struct knote *
2345	knote_alloc(int waitok)
2346	{
2347	return ((struct knote *)uma_zalloc(knote_zone,
2348	(waitok ? M_WAITOK : M_NOWAIT)\|M_ZERO));
2349	}
2350
2351	static void
2352	knote_free(struct knote *kn)
2353	{
2354	if (kn != NULL)
2355	uma_zfree(knote_zone, kn);
2356	}
2357
2358	/*
2359	* Register the kev w/ the kq specified by fd.
2360	*/
2361	int
2362	kqfd_register(int fd, struct kevent kev, struct thread td, int waitok)
2363	{
2364	struct kqueue *kq;
2365	struct file *fp;
2366	int error;
2367
2368	if ((error = fget(td, fd, &fp)) != 0)
2369	return (error);
2370	if ((error = kqueue_acquire(fp, &kq)) != 0)
2371	goto noacquire;
2372
2373	error = kqueue_register(kq, kev, td, waitok);
2374
2375	kqueue_release(kq, 0);
2376
2377	noacquire:
2378	fdrop(fp, td);
2379
2380	return error;
2381	}
2382	#ifdef __rtems__
2383	static const rtems_filesystem_file_handlers_r kqueueops = {
2384	.open_h = rtems_filesystem_default_open,
2385	.close_h = rtems_bsd_kqueue_close,
2386	.read_h = rtems_filesystem_default_read,
2387	.write_h = rtems_filesystem_default_write,
2388	.ioctl_h = rtems_filesystem_default_ioctl,
2389	.lseek_h = rtems_filesystem_default_lseek,
2390	.fstat_h = rtems_bsd_kqueue_stat,
2391	.ftruncate_h = rtems_filesystem_default_ftruncate,
2392	.fsync_h = rtems_filesystem_default_fsync_or_fdatasync,
2393	.fdatasync_h = rtems_filesystem_default_fsync_or_fdatasync,
2394	.fcntl_h = rtems_filesystem_default_fcntl,
2395	.poll_h = rtems_bsd_kqueue_poll,
2396	.kqfilter_h = rtems_bsd_kqueue_kqfilter
2397	};
2398	#endif /* __rtems__ */

Note: See TracBrowser for help on using the repository browser.

Download in other formats: