Context Navigation

kern_timeout.c @ a85d900

55-freebsd-126-freebsd-12

Last change on this file since a85d900 was a85d900, checked in by Sebastian Huber <sebastian.huber@…>, on 08/04/17 at 12:32:19

Optimize callout handline via static timer wheel

The number of callouts is a compile-time constant in libbsd. Use this
in struct callout_cpu and avoid dynamic allocation of tables. This
signficantly reduces the count of load instructions in the callout
handling.

Property mode set to 100644

File size: 51.6 KB

Line
1	#include <machine/rtems-bsd-kernel-space.h>
2
3	/*-
4	* Copyright (c) 1982, 1986, 1991, 1993
5	* The Regents of the University of California. All rights reserved.
6	* (c) UNIX System Laboratories, Inc.
7	* All or some portions of this file are derived from material licensed
8	* to the University of California by American Telephone and Telegraph
9	* Co. or Unix System Laboratories, Inc. and are reproduced herein with
10	* the permission of UNIX System Laboratories, Inc.
11	*
12	* Redistribution and use in source and binary forms, with or without
13	* modification, are permitted provided that the following conditions
14	* are met:
15	* 1. Redistributions of source code must retain the above copyright
16	* notice, this list of conditions and the following disclaimer.
17	* 2. Redistributions in binary form must reproduce the above copyright
18	* notice, this list of conditions and the following disclaimer in the
19	* documentation and/or other materials provided with the distribution.
20	* 3. Neither the name of the University nor the names of its contributors
21	* may be used to endorse or promote products derived from this software
22	* without specific prior written permission.
23	*
24	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34	* SUCH DAMAGE.
35	*
36	* From: @(#)kern_clock.c 8.5 (Berkeley) 1/21/94
37	*/
38
39	#include <sys/cdefs.h>
40	__FBSDID("$FreeBSD$");
41
42	#include <rtems/bsd/local/opt_callout_profiling.h>
43	#include <rtems/bsd/local/opt_ddb.h>
44	#if defined(__arm__) \|\| defined(__rtems__)
45	#include <rtems/bsd/local/opt_timer.h>
46	#endif
47	#include <rtems/bsd/local/opt_rss.h>
48
49	#include <sys/param.h>
50	#include <sys/systm.h>
51	#include <sys/bus.h>
52	#include <sys/callout.h>
53	#include <sys/file.h>
54	#include <sys/interrupt.h>
55	#include <sys/kernel.h>
56	#include <sys/ktr.h>
57	#include <sys/lock.h>
58	#include <sys/malloc.h>
59	#include <sys/mutex.h>
60	#include <sys/proc.h>
61	#include <sys/sdt.h>
62	#include <sys/sleepqueue.h>
63	#include <sys/sysctl.h>
64	#include <sys/smp.h>
65
66	#ifdef DDB
67	#include <ddb/ddb.h>
68	#include <machine/_inttypes.h>
69	#endif
70
71	#ifdef SMP
72	#include <machine/cpu.h>
73	#endif
74
75	#ifndef NO_EVENTTIMERS
76	DPCPU_DECLARE(sbintime_t, hardclocktime);
77	#endif
78
79	SDT_PROVIDER_DEFINE(callout_execute);
80	SDT_PROBE_DEFINE1(callout_execute, , , callout__start, "struct callout *");
81	SDT_PROBE_DEFINE1(callout_execute, , , callout__end, "struct callout *");
82
83	#ifdef CALLOUT_PROFILING
84	static int avg_depth;
85	SYSCTL_INT(_debug, OID_AUTO, to_avg_depth, CTLFLAG_RD, &avg_depth, 0,
86	"Average number of items examined per softclock call. Units = 1/1000");
87	static int avg_gcalls;
88	SYSCTL_INT(_debug, OID_AUTO, to_avg_gcalls, CTLFLAG_RD, &avg_gcalls, 0,
89	"Average number of Giant callouts made per softclock call. Units = 1/1000");
90	static int avg_lockcalls;
91	SYSCTL_INT(_debug, OID_AUTO, to_avg_lockcalls, CTLFLAG_RD, &avg_lockcalls, 0,
92	"Average number of lock callouts made per softclock call. Units = 1/1000");
93	static int avg_mpcalls;
94	SYSCTL_INT(_debug, OID_AUTO, to_avg_mpcalls, CTLFLAG_RD, &avg_mpcalls, 0,
95	"Average number of MP callouts made per softclock call. Units = 1/1000");
96	static int avg_depth_dir;
97	SYSCTL_INT(_debug, OID_AUTO, to_avg_depth_dir, CTLFLAG_RD, &avg_depth_dir, 0,
98	"Average number of direct callouts examined per callout_process call. "
99	"Units = 1/1000");
100	static int avg_lockcalls_dir;
101	SYSCTL_INT(_debug, OID_AUTO, to_avg_lockcalls_dir, CTLFLAG_RD,
102	&avg_lockcalls_dir, 0, "Average number of lock direct callouts made per "
103	"callout_process call. Units = 1/1000");
104	static int avg_mpcalls_dir;
105	SYSCTL_INT(_debug, OID_AUTO, to_avg_mpcalls_dir, CTLFLAG_RD, &avg_mpcalls_dir,
106	0, "Average number of MP direct callouts made per callout_process call. "
107	"Units = 1/1000");
108	#endif
109
110	#ifndef __rtems__
111	static int ncallout;
112	SYSCTL_INT(_kern, OID_AUTO, ncallout, CTLFLAG_RDTUN \| CTLFLAG_NOFETCH, &ncallout, 0,
113	"Number of entries in callwheel and size of timeout() preallocation");
114	#else /* __rtems__ */
115	#define ncallout 16
116	#endif /* __rtems__ */
117
118	#ifdef RSS
119	static int pin_default_swi = 1;
120	static int pin_pcpu_swi = 1;
121	#else
122	static int pin_default_swi = 0;
123	static int pin_pcpu_swi = 0;
124	#endif
125
126	SYSCTL_INT(_kern, OID_AUTO, pin_default_swi, CTLFLAG_RDTUN \| CTLFLAG_NOFETCH, &pin_default_swi,
127	0, "Pin the default (non-per-cpu) swi (shared with PCPU 0 swi)");
128	SYSCTL_INT(_kern, OID_AUTO, pin_pcpu_swi, CTLFLAG_RDTUN \| CTLFLAG_NOFETCH, &pin_pcpu_swi,
129	0, "Pin the per-CPU swis (except PCPU 0, which is also default");
130
131	#ifndef __rtems__
132	/*
133	* TODO:
134	* allocate more timeout table slots when table overflows.
135	*/
136	u_int callwheelsize, callwheelmask;
137	#else /* __rtems__ */
138	#define callwheelsize (2 * ncallout)
139	#define callwheelmask (callwheelsize - 1)
140	#endif /* __rtems__ */
141
142	/*
143	* The callout cpu exec entities represent informations necessary for
144	* describing the state of callouts currently running on the CPU and the ones
145	* necessary for migrating callouts to the new callout cpu. In particular,
146	* the first entry of the array cc_exec_entity holds informations for callout
147	* running in SWI thread context, while the second one holds informations
148	* for callout running directly from hardware interrupt context.
149	* The cached informations are very important for deferring migration when
150	* the migrating callout is already running.
151	*/
152	struct cc_exec {
153	struct callout *cc_curr;
154	void (cc_drain)(void );
155	#ifdef SMP
156	void (ce_migration_func)(void );
157	void *ce_migration_arg;
158	int ce_migration_cpu;
159	sbintime_t ce_migration_time;
160	sbintime_t ce_migration_prec;
161	#endif
162	bool cc_cancel;
163	bool cc_waiting;
164	};
165
166	/*
167	* There is one struct callout_cpu per cpu, holding all relevant
168	* state for the callout processing thread on the individual CPU.
169	*/
170	struct callout_cpu {
171	struct mtx_padalign cc_lock;
172	#ifndef __rtems__
173	struct cc_exec cc_exec_entity[2];
174	#else /* __rtems__ */
175	struct cc_exec cc_exec_entity;
176	#endif /* __rtems__ */
177	struct callout *cc_next;
178	#ifndef __rtems__
179	struct callout *cc_callout;
180	struct callout_list *cc_callwheel;
181	struct callout_tailq cc_expireq;
182	#else /* __rtems__ */
183	struct callout cc_callout[ncallout];
184	struct callout_list cc_callwheel[callwheelsize];
185	#endif /* __rtems__ */
186	struct callout_slist cc_callfree;
187	sbintime_t cc_firstevent;
188	sbintime_t cc_lastscan;
189	void *cc_cookie;
190	u_int cc_bucket;
191	u_int cc_inited;
192	char cc_ktr_event_name[20];
193	};
194
195	#ifndef __rtems__
196	#define callout_migrating(c) ((c)->c_iflags & CALLOUT_DFRMIGRATION)
197	#endif /* __rtems__ */
198
199	#ifndef __rtems__
200	#define cc_exec_curr(cc, dir) cc->cc_exec_entity[dir].cc_curr
201	#define cc_exec_drain(cc, dir) cc->cc_exec_entity[dir].cc_drain
202	#else /* __rtems__ */
203	#define cc_exec_curr(cc, dir) cc->cc_exec_entity.cc_curr
204	#define cc_exec_drain(cc, dir) cc->cc_exec_entity.cc_drain
205	#endif /* __rtems__ */
206	#define cc_exec_next(cc) cc->cc_next
207	#ifndef __rtems__
208	#define cc_exec_cancel(cc, dir) cc->cc_exec_entity[dir].cc_cancel
209	#define cc_exec_waiting(cc, dir) cc->cc_exec_entity[dir].cc_waiting
210	#else /* __rtems__ */
211	#define cc_exec_cancel(cc, dir) cc->cc_exec_entity.cc_cancel
212	#define cc_exec_waiting(cc, dir) cc->cc_exec_entity.cc_waiting
213	#endif /* __rtems__ */
214	#ifdef SMP
215	#define cc_migration_func(cc, dir) cc->cc_exec_entity[dir].ce_migration_func
216	#define cc_migration_arg(cc, dir) cc->cc_exec_entity[dir].ce_migration_arg
217	#define cc_migration_cpu(cc, dir) cc->cc_exec_entity[dir].ce_migration_cpu
218	#define cc_migration_time(cc, dir) cc->cc_exec_entity[dir].ce_migration_time
219	#define cc_migration_prec(cc, dir) cc->cc_exec_entity[dir].ce_migration_prec
220
221	struct callout_cpu cc_cpu[MAXCPU];
222	#define CPUBLOCK MAXCPU
223	#define CC_CPU(cpu) (&cc_cpu[(cpu)])
224	#define CC_SELF() CC_CPU(PCPU_GET(cpuid))
225	#else
226	struct callout_cpu cc_cpu;
227	#define CC_CPU(cpu) &cc_cpu
228	#define CC_SELF() &cc_cpu
229	#endif
230	#define CC_LOCK(cc) mtx_lock_spin(&(cc)->cc_lock)
231	#define CC_UNLOCK(cc) mtx_unlock_spin(&(cc)->cc_lock)
232	#define CC_LOCK_ASSERT(cc) mtx_assert(&(cc)->cc_lock, MA_OWNED)
233
234	#ifndef __rtems__
235	static int timeout_cpu;
236	#else /* __rtems__ */
237	#define timeout_cpu 0
238	#endif /* __rtems__ */
239
240	static void callout_cpu_init(struct callout_cpu *cc, int cpu);
241	static void softclock_call_cc(struct callout c, struct callout_cpu cc,
242	#ifdef CALLOUT_PROFILING
243	int mpcalls, int lockcalls, int *gcalls,
244	#endif
245	int direct);
246
247	static MALLOC_DEFINE(M_CALLOUT, "callout", "Callout datastructures");
248
249	/**
250	* Locked by cc_lock:
251	* cc_curr - If a callout is in progress, it is cc_curr.
252	* If cc_curr is non-NULL, threads waiting in
253	* callout_drain() will be woken up as soon as the
254	* relevant callout completes.
255	* cc_cancel - Changing to 1 with both callout_lock and cc_lock held
256	* guarantees that the current callout will not run.
257	* The softclock() function sets this to 0 before it
258	* drops callout_lock to acquire c_lock, and it calls
259	* the handler only if curr_cancelled is still 0 after
260	* cc_lock is successfully acquired.
261	* cc_waiting - If a thread is waiting in callout_drain(), then
262	* callout_wait is nonzero. Set only when
263	* cc_curr is non-NULL.
264	*/
265
266	/*
267	* Resets the execution entity tied to a specific callout cpu.
268	*/
269	static void
270	cc_cce_cleanup(struct callout_cpu *cc, int direct)
271	{
272
273	cc_exec_curr(cc, direct) = NULL;
274	cc_exec_cancel(cc, direct) = false;
275	cc_exec_waiting(cc, direct) = false;
276	#ifdef SMP
277	cc_migration_cpu(cc, direct) = CPUBLOCK;
278	cc_migration_time(cc, direct) = 0;
279	cc_migration_prec(cc, direct) = 0;
280	cc_migration_func(cc, direct) = NULL;
281	cc_migration_arg(cc, direct) = NULL;
282	#endif
283	}
284
285	#ifndef __rtems__
286	/*
287	* Checks if migration is requested by a specific callout cpu.
288	*/
289	static int
290	cc_cce_migrating(struct callout_cpu *cc, int direct)
291	{
292
293	#ifdef SMP
294	return (cc_migration_cpu(cc, direct) != CPUBLOCK);
295	#else
296	return (0);
297	#endif
298	}
299	#endif /* __rtems__ */
300
301	/*
302	* Kernel low level callwheel initialization
303	* called on cpu0 during kernel startup.
304	*/
305	#ifdef __rtems__
306	static void rtems_bsd_timeout_init_early(void *);
307
308	static void
309	rtems_bsd_callout_timer(rtems_id id, void *arg)
310	{
311	rtems_status_code sc;
312
313	(void) arg;
314
315	sc = rtems_timer_reset(id);
316	BSD_ASSERT(sc == RTEMS_SUCCESSFUL);
317
318	callout_process(sbinuptime());
319	}
320
321	static void
322	rtems_bsd_timeout_init_late(void *unused)
323	{
324	rtems_status_code sc;
325	rtems_id id;
326
327	(void) unused;
328
329	sc = rtems_timer_create(rtems_build_name('_', 'C', 'L', 'O'), &id);
330	BSD_ASSERT(sc == RTEMS_SUCCESSFUL);
331
332	sc = rtems_timer_server_fire_after(id, 1, rtems_bsd_callout_timer, NULL);
333	BSD_ASSERT(sc == RTEMS_SUCCESSFUL);
334	}
335
336	SYSINIT(rtems_bsd_timeout_early, SI_SUB_VM, SI_ORDER_FIRST,
337	rtems_bsd_timeout_init_early, NULL);
338
339	SYSINIT(rtems_bsd_timeout_late, SI_SUB_LAST, SI_ORDER_FIRST,
340	rtems_bsd_timeout_init_late, NULL);
341
342	static void
343	rtems_bsd_timeout_init_early(void *dummy)
344	#else /* __rtems__ */
345	static void
346	callout_callwheel_init(void *dummy)
347	#endif /* __rtems__ */
348	{
349	struct callout_cpu *cc;
350	#ifdef __rtems__
351	(void) dummy;
352	#endif /* __rtems__ */
353
354	/*
355	* Calculate the size of the callout wheel and the preallocated
356	* timeout() structures.
357	* XXX: Clip callout to result of previous function of maxusers
358	* maximum 384. This is still huge, but acceptable.
359	*/
360	memset(CC_CPU(0), 0, sizeof(cc_cpu));
361	#ifndef __rtems__
362	ncallout = imin(16 + maxproc + maxfiles, 18508);
363	TUNABLE_INT_FETCH("kern.ncallout", &ncallout);
364	#endif /* __rtems__ */
365
366	/*
367	* Calculate callout wheel size, should be next power of two higher
368	* than 'ncallout'.
369	*/
370	#ifndef __rtems__
371	callwheelsize = 1 << fls(ncallout);
372	callwheelmask = callwheelsize - 1;
373	#endif /* __rtems__ */
374
375	#ifndef __rtems__
376	/*
377	* Fetch whether we're pinning the swi's or not.
378	*/
379	TUNABLE_INT_FETCH("kern.pin_default_swi", &pin_default_swi);
380	TUNABLE_INT_FETCH("kern.pin_pcpu_swi", &pin_pcpu_swi);
381	#endif /* __rtems__ */
382
383	/*
384	* Only cpu0 handles timeout(9) and receives a preallocation.
385	*
386	* XXX: Once all timeout(9) consumers are converted this can
387	* be removed.
388	*/
389	#ifndef __rtems__
390	timeout_cpu = PCPU_GET(cpuid);
391	#endif /* __rtems__ */
392	cc = CC_CPU(timeout_cpu);
393	#ifndef __rtems__
394	cc->cc_callout = malloc(ncallout * sizeof(struct callout),
395	M_CALLOUT, M_WAITOK);
396	#endif /* __rtems__ */
397	callout_cpu_init(cc, timeout_cpu);
398	}
399	#ifndef __rtems__
400	SYSINIT(callwheel_init, SI_SUB_CPU, SI_ORDER_ANY, callout_callwheel_init, NULL);
401	#endif /* __rtems__ */
402
403	/*
404	* Initialize the per-cpu callout structures.
405	*/
406	static void
407	callout_cpu_init(struct callout_cpu *cc, int cpu)
408	{
409	struct callout *c;
410	int i;
411
412	mtx_init(&cc->cc_lock, "callout", NULL, MTX_SPIN \| MTX_RECURSE);
413	SLIST_INIT(&cc->cc_callfree);
414	cc->cc_inited = 1;
415	#ifndef __rtems__
416	cc->cc_callwheel = malloc(sizeof(struct callout_list) * callwheelsize,
417	M_CALLOUT, M_WAITOK);
418	#endif /* __rtems__ */
419	for (i = 0; i < callwheelsize; i++)
420	LIST_INIT(&cc->cc_callwheel[i]);
421	#ifndef __rtems__
422	TAILQ_INIT(&cc->cc_expireq);
423	#endif /* __rtems__ */
424	cc->cc_firstevent = SBT_MAX;
425	for (i = 0; i < 2; i++)
426	cc_cce_cleanup(cc, i);
427	snprintf(cc->cc_ktr_event_name, sizeof(cc->cc_ktr_event_name),
428	"callwheel cpu %d", cpu);
429	#ifndef __rtems__
430	if (cc->cc_callout == NULL) /* Only cpu0 handles timeout(9) */
431	return;
432	#endif /* __rtems__ */
433	for (i = 0; i < ncallout; i++) {
434	c = &cc->cc_callout[i];
435	callout_init(c, 0);
436	c->c_iflags = CALLOUT_LOCAL_ALLOC;
437	SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle);
438	}
439	}
440
441	#ifdef SMP
442	/*
443	* Switches the cpu tied to a specific callout.
444	* The function expects a locked incoming callout cpu and returns with
445	* locked outcoming callout cpu.
446	*/
447	static struct callout_cpu *
448	callout_cpu_switch(struct callout c, struct callout_cpu cc, int new_cpu)
449	{
450	struct callout_cpu *new_cc;
451
452	MPASS(c != NULL && cc != NULL);
453	CC_LOCK_ASSERT(cc);
454
455	/*
456	* Avoid interrupts and preemption firing after the callout cpu
457	* is blocked in order to avoid deadlocks as the new thread
458	* may be willing to acquire the callout cpu lock.
459	*/
460	c->c_cpu = CPUBLOCK;
461	spinlock_enter();
462	CC_UNLOCK(cc);
463	new_cc = CC_CPU(new_cpu);
464	CC_LOCK(new_cc);
465	spinlock_exit();
466	c->c_cpu = new_cpu;
467	return (new_cc);
468	}
469	#endif
470
471	#ifndef __rtems__
472	/*
473	* Start standard softclock thread.
474	*/
475	static void
476	start_softclock(void *dummy)
477	{
478	struct callout_cpu *cc;
479	char name[MAXCOMLEN];
480	#ifdef SMP
481	int cpu;
482	struct intr_event *ie;
483	#endif
484
485	cc = CC_CPU(timeout_cpu);
486	snprintf(name, sizeof(name), "clock (%d)", timeout_cpu);
487	if (swi_add(&clk_intr_event, name, softclock, cc, SWI_CLOCK,
488	INTR_MPSAFE, &cc->cc_cookie))
489	panic("died while creating standard software ithreads");
490	if (pin_default_swi &&
491	(intr_event_bind(clk_intr_event, timeout_cpu) != 0)) {
492	printf("%s: timeout clock couldn't be pinned to cpu %d\n",
493	__func__,
494	timeout_cpu);
495	}
496
497	#ifdef SMP
498	CPU_FOREACH(cpu) {
499	if (cpu == timeout_cpu)
500	continue;
501	cc = CC_CPU(cpu);
502	cc->cc_callout = NULL; /* Only cpu0 handles timeout(9). */
503	callout_cpu_init(cc, cpu);
504	snprintf(name, sizeof(name), "clock (%d)", cpu);
505	ie = NULL;
506	if (swi_add(&ie, name, softclock, cc, SWI_CLOCK,
507	INTR_MPSAFE, &cc->cc_cookie))
508	panic("died while creating standard software ithreads");
509	if (pin_pcpu_swi && (intr_event_bind(ie, cpu) != 0)) {
510	printf("%s: per-cpu clock couldn't be pinned to "
511	"cpu %d\n",
512	__func__,
513	cpu);
514	}
515	}
516	#endif
517	}
518	SYSINIT(start_softclock, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softclock, NULL);
519	#endif /* __rtems__ */
520
521	#define CC_HASH_SHIFT 8
522
523	static inline u_int
524	callout_hash(sbintime_t sbt)
525	{
526
527	return (sbt >> (32 - CC_HASH_SHIFT));
528	}
529
530	static inline u_int
531	callout_get_bucket(sbintime_t sbt)
532	{
533
534	return (callout_hash(sbt) & callwheelmask);
535	}
536
537	void
538	callout_process(sbintime_t now)
539	{
540	#ifndef __rtems__
541	struct callout tmp, tmpn;
542	#else /* __rtems__ */
543	struct callout *tmp;
544	#endif /* __rtems__ */
545	struct callout_cpu *cc;
546	struct callout_list *sc;
547	sbintime_t first, last, max, tmp_max;
548	uint32_t lookahead;
549	u_int firstb, lastb, nowb;
550	#ifdef CALLOUT_PROFILING
551	int depth_dir = 0, mpcalls_dir = 0, lockcalls_dir = 0;
552	#endif
553
554	cc = CC_SELF();
555	mtx_lock_spin_flags(&cc->cc_lock, MTX_QUIET);
556
557	/* Compute the buckets of the last scan and present times. */
558	firstb = callout_hash(cc->cc_lastscan);
559	cc->cc_lastscan = now;
560	nowb = callout_hash(now);
561
562	/* Compute the last bucket and minimum time of the bucket after it. */
563	if (nowb == firstb)
564	lookahead = (SBT_1S / 16);
565	else if (nowb - firstb == 1)
566	lookahead = (SBT_1S / 8);
567	else
568	lookahead = (SBT_1S / 2);
569	first = last = now;
570	first += (lookahead / 2);
571	last += lookahead;
572	last &= (0xffffffffffffffffLLU << (32 - CC_HASH_SHIFT));
573	lastb = callout_hash(last) - 1;
574	max = last;
575
576	/*
577	* Check if we wrapped around the entire wheel from the last scan.
578	* In case, we need to scan entirely the wheel for pending callouts.
579	*/
580	if (lastb - firstb >= callwheelsize) {
581	lastb = firstb + callwheelsize - 1;
582	if (nowb - firstb >= callwheelsize)
583	nowb = lastb;
584	}
585
586	/* Iterate callwheel from firstb to nowb and then up to lastb. */
587	do {
588	sc = &cc->cc_callwheel[firstb & callwheelmask];
589	tmp = LIST_FIRST(sc);
590	while (tmp != NULL) {
591	/* Run the callout if present time within allowed. */
592	if (tmp->c_time <= now) {
593	#ifndef __rtems__
594	/*
595	* Consumer told us the callout may be run
596	* directly from hardware interrupt context.
597	*/
598	if (tmp->c_iflags & CALLOUT_DIRECT) {
599	#endif /* __rtems__ */
600	#ifdef CALLOUT_PROFILING
601	++depth_dir;
602	#endif
603	cc_exec_next(cc) =
604	LIST_NEXT(tmp, c_links.le);
605	cc->cc_bucket = firstb & callwheelmask;
606	LIST_REMOVE(tmp, c_links.le);
607	softclock_call_cc(tmp, cc,
608	#ifdef CALLOUT_PROFILING
609	&mpcalls_dir, &lockcalls_dir, NULL,
610	#endif
611	1);
612	tmp = cc_exec_next(cc);
613	cc_exec_next(cc) = NULL;
614	#ifndef __rtems__
615	} else {
616	tmpn = LIST_NEXT(tmp, c_links.le);
617	LIST_REMOVE(tmp, c_links.le);
618	TAILQ_INSERT_TAIL(&cc->cc_expireq,
619	tmp, c_links.tqe);
620	tmp->c_iflags \|= CALLOUT_PROCESSED;
621	tmp = tmpn;
622	}
623	#endif /* __rtems__ */
624	continue;
625	}
626	/* Skip events from distant future. */
627	if (tmp->c_time >= max)
628	goto next;
629	/*
630	* Event minimal time is bigger than present maximal
631	* time, so it cannot be aggregated.
632	*/
633	if (tmp->c_time > last) {
634	lastb = nowb;
635	goto next;
636	}
637	/* Update first and last time, respecting this event. */
638	if (tmp->c_time < first)
639	first = tmp->c_time;
640	tmp_max = tmp->c_time + tmp->c_precision;
641	if (tmp_max < last)
642	last = tmp_max;
643	next:
644	tmp = LIST_NEXT(tmp, c_links.le);
645	}
646	/* Proceed with the next bucket. */
647	firstb++;
648	/*
649	* Stop if we looked after present time and found
650	* some event we can't execute at now.
651	* Stop if we looked far enough into the future.
652	*/
653	} while (((int)(firstb - lastb)) <= 0);
654	cc->cc_firstevent = last;
655	#ifndef NO_EVENTTIMERS
656	cpu_new_callout(curcpu, last, first);
657	#endif
658	#ifdef CALLOUT_PROFILING
659	avg_depth_dir += (depth_dir * 1000 - avg_depth_dir) >> 8;
660	avg_mpcalls_dir += (mpcalls_dir * 1000 - avg_mpcalls_dir) >> 8;
661	avg_lockcalls_dir += (lockcalls_dir * 1000 - avg_lockcalls_dir) >> 8;
662	#endif
663	mtx_unlock_spin_flags(&cc->cc_lock, MTX_QUIET);
664	#ifndef __rtems__
665	/*
666	* swi_sched acquires the thread lock, so we don't want to call it
667	* with cc_lock held; incorrect locking order.
668	*/
669	if (!TAILQ_EMPTY(&cc->cc_expireq))
670	swi_sched(cc->cc_cookie, 0);
671	#endif /* __rtems__ */
672	}
673
674	static struct callout_cpu *
675	callout_lock(struct callout *c)
676	{
677	struct callout_cpu *cc;
678	#ifndef __rtems__
679	int cpu;
680
681	for (;;) {
682	cpu = c->c_cpu;
683	#ifdef SMP
684	if (cpu == CPUBLOCK) {
685	while (c->c_cpu == CPUBLOCK)
686	cpu_spinwait();
687	continue;
688	}
689	#endif
690	#endif /* __rtems__ */
691	cc = CC_CPU(cpu);
692	CC_LOCK(cc);
693	#ifndef __rtems__
694	if (cpu == c->c_cpu)
695	break;
696	CC_UNLOCK(cc);
697	}
698	#endif /* __rtems__ */
699	return (cc);
700	}
701
702	static void
703	callout_cc_add(struct callout c, struct callout_cpu cc,
704	sbintime_t sbt, sbintime_t precision, void (func)(void ),
705	void *arg, int cpu, int flags)
706	{
707	int bucket;
708
709	CC_LOCK_ASSERT(cc);
710	if (sbt < cc->cc_lastscan)
711	sbt = cc->cc_lastscan;
712	c->c_arg = arg;
713	c->c_iflags \|= CALLOUT_PENDING;
714	#ifndef __rtems__
715	c->c_iflags &= ~CALLOUT_PROCESSED;
716	#endif /* __rtems__ */
717	c->c_flags \|= CALLOUT_ACTIVE;
718	#ifndef __rtems__
719	if (flags & C_DIRECT_EXEC)
720	c->c_iflags \|= CALLOUT_DIRECT;
721	#endif /* __rtems__ */
722	c->c_func = func;
723	c->c_time = sbt;
724	c->c_precision = precision;
725	bucket = callout_get_bucket(c->c_time);
726	CTR3(KTR_CALLOUT, "precision set for %p: %d.%08x",
727	c, (int)(c->c_precision >> 32),
728	(u_int)(c->c_precision & 0xffffffff));
729	LIST_INSERT_HEAD(&cc->cc_callwheel[bucket], c, c_links.le);
730	if (cc->cc_bucket == bucket)
731	cc_exec_next(cc) = c;
732	#ifndef NO_EVENTTIMERS
733	/*
734	* Inform the eventtimers(4) subsystem there's a new callout
735	* that has been inserted, but only if really required.
736	*/
737	if (SBT_MAX - c->c_time < c->c_precision)
738	c->c_precision = SBT_MAX - c->c_time;
739	sbt = c->c_time + c->c_precision;
740	if (sbt < cc->cc_firstevent) {
741	cc->cc_firstevent = sbt;
742	cpu_new_callout(cpu, sbt, c->c_time);
743	}
744	#endif
745	}
746
747	static void
748	callout_cc_del(struct callout c, struct callout_cpu cc)
749	{
750
751	if ((c->c_iflags & CALLOUT_LOCAL_ALLOC) == 0)
752	return;
753	c->c_func = NULL;
754	SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle);
755	}
756
757	static void
758	softclock_call_cc(struct callout c, struct callout_cpu cc,
759	#ifdef CALLOUT_PROFILING
760	int mpcalls, int lockcalls, int *gcalls,
761	#endif
762	int direct)
763	{
764	#ifndef __rtems__
765	struct rm_priotracker tracker;
766	#endif /* __rtems__ */
767	void (c_func)(void );
768	void *c_arg;
769	struct lock_class *class;
770	struct lock_object *c_lock;
771	uintptr_t lock_status;
772	int c_iflags;
773	#ifdef SMP
774	struct callout_cpu *new_cc;
775	void (new_func)(void );
776	void *new_arg;
777	int flags, new_cpu;
778	sbintime_t new_prec, new_time;
779	#endif
780	#if defined(DIAGNOSTIC) \|\| defined(CALLOUT_PROFILING)
781	sbintime_t sbt1, sbt2;
782	struct timespec ts2;
783	static sbintime_t maxdt = 2 * SBT_1MS; /* 2 msec */
784	static timeout_t *lastfunc;
785	#endif
786
787	KASSERT((c->c_iflags & CALLOUT_PENDING) == CALLOUT_PENDING,
788	("softclock_call_cc: pend %p %x", c, c->c_iflags));
789	KASSERT((c->c_flags & CALLOUT_ACTIVE) == CALLOUT_ACTIVE,
790	("softclock_call_cc: act %p %x", c, c->c_flags));
791	class = (c->c_lock != NULL) ? LOCK_CLASS(c->c_lock) : NULL;
792	lock_status = 0;
793	if (c->c_flags & CALLOUT_SHAREDLOCK) {
794	#ifndef __rtems__
795	if (class == &lock_class_rm)
796	lock_status = (uintptr_t)&tracker;
797	else
798	#endif /* __rtems__ */
799	lock_status = 1;
800	}
801	c_lock = c->c_lock;
802	c_func = c->c_func;
803	c_arg = c->c_arg;
804	c_iflags = c->c_iflags;
805	if (c->c_iflags & CALLOUT_LOCAL_ALLOC)
806	c->c_iflags = CALLOUT_LOCAL_ALLOC;
807	else
808	c->c_iflags &= ~CALLOUT_PENDING;
809
810	cc_exec_curr(cc, direct) = c;
811	cc_exec_cancel(cc, direct) = false;
812	cc_exec_drain(cc, direct) = NULL;
813	CC_UNLOCK(cc);
814	if (c_lock != NULL) {
815	class->lc_lock(c_lock, lock_status);
816	/*
817	* The callout may have been cancelled
818	* while we switched locks.
819	*/
820	if (cc_exec_cancel(cc, direct)) {
821	class->lc_unlock(c_lock);
822	goto skip;
823	}
824	/* The callout cannot be stopped now. */
825	cc_exec_cancel(cc, direct) = true;
826	if (c_lock == &Giant.lock_object) {
827	#ifdef CALLOUT_PROFILING
828	(*gcalls)++;
829	#endif
830	CTR3(KTR_CALLOUT, "callout giant %p func %p arg %p",
831	c, c_func, c_arg);
832	} else {
833	#ifdef CALLOUT_PROFILING
834	(*lockcalls)++;
835	#endif
836	CTR3(KTR_CALLOUT, "callout lock %p func %p arg %p",
837	c, c_func, c_arg);
838	}
839	} else {
840	#ifdef CALLOUT_PROFILING
841	(*mpcalls)++;
842	#endif
843	CTR3(KTR_CALLOUT, "callout %p func %p arg %p",
844	c, c_func, c_arg);
845	}
846	KTR_STATE3(KTR_SCHED, "callout", cc->cc_ktr_event_name, "running",
847	"func:%p", c_func, "arg:%p", c_arg, "direct:%d", direct);
848	#if defined(DIAGNOSTIC) \|\| defined(CALLOUT_PROFILING)
849	sbt1 = sbinuptime();
850	#endif
851	#ifndef __rtems__
852	THREAD_NO_SLEEPING();
853	SDT_PROBE1(callout_execute, , , callout__start, c);
854	#endif /* __rtems__ */
855	c_func(c_arg);
856	#ifndef __rtems__
857	SDT_PROBE1(callout_execute, , , callout__end, c);
858	THREAD_SLEEPING_OK();
859	#endif /* __rtems__ */
860	#if defined(DIAGNOSTIC) \|\| defined(CALLOUT_PROFILING)
861	sbt2 = sbinuptime();
862	sbt2 -= sbt1;
863	if (sbt2 > maxdt) {
864	if (lastfunc != c_func \|\| sbt2 > maxdt * 2) {
865	ts2 = sbttots(sbt2);
866	printf(
867	"Expensive timeout(9) function: %p(%p) %jd.%09ld s\n",
868	c_func, c_arg, (intmax_t)ts2.tv_sec, ts2.tv_nsec);
869	}
870	maxdt = sbt2;
871	lastfunc = c_func;
872	}
873	#endif
874	KTR_STATE0(KTR_SCHED, "callout", cc->cc_ktr_event_name, "idle");
875	CTR1(KTR_CALLOUT, "callout %p finished", c);
876	if ((c_iflags & CALLOUT_RETURNUNLOCKED) == 0)
877	class->lc_unlock(c_lock);
878	skip:
879	CC_LOCK(cc);
880	KASSERT(cc_exec_curr(cc, direct) == c, ("mishandled cc_curr"));
881	cc_exec_curr(cc, direct) = NULL;
882	if (cc_exec_drain(cc, direct)) {
883	void (drain)(void );
884
885	drain = cc_exec_drain(cc, direct);
886	cc_exec_drain(cc, direct) = NULL;
887	CC_UNLOCK(cc);
888	drain(c_arg);
889	CC_LOCK(cc);
890	}
891	if (cc_exec_waiting(cc, direct)) {
892	#ifndef __rtems__
893	/*
894	* There is someone waiting for the
895	* callout to complete.
896	* If the callout was scheduled for
897	* migration just cancel it.
898	*/
899	if (cc_cce_migrating(cc, direct)) {
900	cc_cce_cleanup(cc, direct);
901
902	/*
903	* It should be assert here that the callout is not
904	* destroyed but that is not easy.
905	*/
906	c->c_iflags &= ~CALLOUT_DFRMIGRATION;
907	}
908	#endif /* __rtems__ */
909	cc_exec_waiting(cc, direct) = false;
910	CC_UNLOCK(cc);
911	wakeup(&cc_exec_waiting(cc, direct));
912	CC_LOCK(cc);
913	#ifndef __rtems__
914	} else if (cc_cce_migrating(cc, direct)) {
915	KASSERT((c_iflags & CALLOUT_LOCAL_ALLOC) == 0,
916	("Migrating legacy callout %p", c));
917	#ifdef SMP
918	/*
919	* If the callout was scheduled for
920	* migration just perform it now.
921	*/
922	new_cpu = cc_migration_cpu(cc, direct);
923	new_time = cc_migration_time(cc, direct);
924	new_prec = cc_migration_prec(cc, direct);
925	new_func = cc_migration_func(cc, direct);
926	new_arg = cc_migration_arg(cc, direct);
927	cc_cce_cleanup(cc, direct);
928
929	/*
930	* It should be assert here that the callout is not destroyed
931	* but that is not easy.
932	*
933	* As first thing, handle deferred callout stops.
934	*/
935	if (!callout_migrating(c)) {
936	CTR3(KTR_CALLOUT,
937	"deferred cancelled %p func %p arg %p",
938	c, new_func, new_arg);
939	callout_cc_del(c, cc);
940	return;
941	}
942	c->c_iflags &= ~CALLOUT_DFRMIGRATION;
943
944	new_cc = callout_cpu_switch(c, cc, new_cpu);
945	flags = (direct) ? C_DIRECT_EXEC : 0;
946	callout_cc_add(c, new_cc, new_time, new_prec, new_func,
947	new_arg, new_cpu, flags);
948	CC_UNLOCK(new_cc);
949	CC_LOCK(cc);
950	#else
951	panic("migration should not happen");
952	#endif
953	#endif /* __rtems__ */
954	}
955	/*
956	* If the current callout is locally allocated (from
957	* timeout(9)) then put it on the freelist.
958	*
959	* Note: we need to check the cached copy of c_iflags because
960	* if it was not local, then it's not safe to deref the
961	* callout pointer.
962	*/
963	KASSERT((c_iflags & CALLOUT_LOCAL_ALLOC) == 0 \|\|
964	c->c_iflags == CALLOUT_LOCAL_ALLOC,
965	("corrupted callout"));
966	if (c_iflags & CALLOUT_LOCAL_ALLOC)
967	callout_cc_del(c, cc);
968	}
969
970	/*
971	* The callout mechanism is based on the work of Adam M. Costello and
972	* George Varghese, published in a technical report entitled "Redesigning
973	* the BSD Callout and Timer Facilities" and modified slightly for inclusion
974	* in FreeBSD by Justin T. Gibbs. The original work on the data structures
975	* used in this implementation was published by G. Varghese and T. Lauck in
976	* the paper "Hashed and Hierarchical Timing Wheels: Data Structures for
977	* the Efficient Implementation of a Timer Facility" in the Proceedings of
978	* the 11th ACM Annual Symposium on Operating Systems Principles,
979	* Austin, Texas Nov 1987.
980	*/
981
982	#ifndef __rtems__
983	/*
984	* Software (low priority) clock interrupt.
985	* Run periodic events from timeout queue.
986	*/
987	void
988	softclock(void *arg)
989	{
990	struct callout_cpu *cc;
991	struct callout *c;
992	#ifdef CALLOUT_PROFILING
993	int depth = 0, gcalls = 0, lockcalls = 0, mpcalls = 0;
994	#endif
995
996	cc = (struct callout_cpu *)arg;
997	CC_LOCK(cc);
998	while ((c = TAILQ_FIRST(&cc->cc_expireq)) != NULL) {
999	TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe);
1000	softclock_call_cc(c, cc,
1001	#ifdef CALLOUT_PROFILING
1002	&mpcalls, &lockcalls, &gcalls,
1003	#endif
1004	0);
1005	#ifdef CALLOUT_PROFILING
1006	++depth;
1007	#endif
1008	}
1009	#ifdef CALLOUT_PROFILING
1010	avg_depth += (depth * 1000 - avg_depth) >> 8;
1011	avg_mpcalls += (mpcalls * 1000 - avg_mpcalls) >> 8;
1012	avg_lockcalls += (lockcalls * 1000 - avg_lockcalls) >> 8;
1013	avg_gcalls += (gcalls * 1000 - avg_gcalls) >> 8;
1014	#endif
1015	CC_UNLOCK(cc);
1016	}
1017	#endif /* __rtems__ */
1018
1019	/*
1020	* timeout --
1021	* Execute a function after a specified length of time.
1022	*
1023	* untimeout --
1024	* Cancel previous timeout function call.
1025	*
1026	* callout_handle_init --
1027	* Initialize a handle so that using it with untimeout is benign.
1028	*
1029	* See AT&T BCI Driver Reference Manual for specification. This
1030	* implementation differs from that one in that although an
1031	* identification value is returned from timeout, the original
1032	* arguments to timeout as well as the identifier are used to
1033	* identify entries for untimeout.
1034	*/
1035	struct callout_handle
1036	timeout(timeout_t ftn, void arg, int to_ticks)
1037	{
1038	struct callout_cpu *cc;
1039	struct callout *new;
1040	struct callout_handle handle;
1041
1042	cc = CC_CPU(timeout_cpu);
1043	CC_LOCK(cc);
1044	/* Fill in the next free callout structure. */
1045	new = SLIST_FIRST(&cc->cc_callfree);
1046	if (new == NULL)
1047	/* XXX Attempt to malloc first */
1048	panic("timeout table full");
1049	SLIST_REMOVE_HEAD(&cc->cc_callfree, c_links.sle);
1050	callout_reset(new, to_ticks, ftn, arg);
1051	handle.callout = new;
1052	CC_UNLOCK(cc);
1053
1054	return (handle);
1055	}
1056
1057	void
1058	untimeout(timeout_t ftn, void arg, struct callout_handle handle)
1059	{
1060	struct callout_cpu *cc;
1061
1062	/*
1063	* Check for a handle that was initialized
1064	* by callout_handle_init, but never used
1065	* for a real timeout.
1066	*/
1067	if (handle.callout == NULL)
1068	return;
1069
1070	cc = callout_lock(handle.callout);
1071	if (handle.callout->c_func == ftn && handle.callout->c_arg == arg)
1072	callout_stop(handle.callout);
1073	CC_UNLOCK(cc);
1074	}
1075
1076	void
1077	callout_handle_init(struct callout_handle *handle)
1078	{
1079	handle->callout = NULL;
1080	}
1081
1082	void
1083	callout_when(sbintime_t sbt, sbintime_t precision, int flags,
1084	sbintime_t res, sbintime_t prec_res)
1085	{
1086	sbintime_t to_sbt, to_pr;
1087
1088	if ((flags & (C_ABSOLUTE \| C_PRECALC)) != 0) {
1089	*res = sbt;
1090	*prec_res = precision;
1091	return;
1092	}
1093	if ((flags & C_HARDCLOCK) != 0 && sbt < tick_sbt)
1094	sbt = tick_sbt;
1095	if ((flags & C_HARDCLOCK) != 0 \|\|
1096	#ifdef NO_EVENTTIMERS
1097	sbt >= sbt_timethreshold) {
1098	to_sbt = getsbinuptime();
1099
1100	/* Add safety belt for the case of hz > 1000. */
1101	to_sbt += tc_tick_sbt - tick_sbt;
1102	#else
1103	sbt >= sbt_tickthreshold) {
1104	/*
1105	* Obtain the time of the last hardclock() call on
1106	* this CPU directly from the kern_clocksource.c.
1107	* This value is per-CPU, but it is equal for all
1108	* active ones.
1109	*/
1110	#ifdef __LP64__
1111	to_sbt = DPCPU_GET(hardclocktime);
1112	#else
1113	spinlock_enter();
1114	to_sbt = DPCPU_GET(hardclocktime);
1115	spinlock_exit();
1116	#endif
1117	#endif
1118	if (cold && to_sbt == 0)
1119	to_sbt = sbinuptime();
1120	if ((flags & C_HARDCLOCK) == 0)
1121	to_sbt += tick_sbt;
1122	} else
1123	to_sbt = sbinuptime();
1124	if (SBT_MAX - to_sbt < sbt)
1125	to_sbt = SBT_MAX;
1126	else
1127	to_sbt += sbt;
1128	*res = to_sbt;
1129	to_pr = ((C_PRELGET(flags) < 0) ? sbt >> tc_precexp :
1130	sbt >> C_PRELGET(flags));
1131	*prec_res = to_pr > precision ? to_pr : precision;
1132	}
1133
1134	/*
1135	* New interface; clients allocate their own callout structures.
1136	*
1137	* callout_reset() - establish or change a timeout
1138	* callout_stop() - disestablish a timeout
1139	* callout_init() - initialize a callout structure so that it can
1140	* safely be passed to callout_reset() and callout_stop()
1141	*
1142	* <sys/callout.h> defines three convenience macros:
1143	*
1144	* callout_active() - returns truth if callout has not been stopped,
1145	* drained, or deactivated since the last time the callout was
1146	* reset.
1147	* callout_pending() - returns truth if callout is still waiting for timeout
1148	* callout_deactivate() - marks the callout as having been serviced
1149	*/
1150	int
1151	callout_reset_sbt_on(struct callout *c, sbintime_t sbt, sbintime_t prec,
1152	void (ftn)(void ), void *arg, int cpu, int flags)
1153	{
1154	sbintime_t to_sbt, precision;
1155	struct callout_cpu *cc;
1156	#ifndef __rtems__
1157	int cancelled, direct;
1158	int ignore_cpu=0;
1159	#else /* __rtems__ */
1160	int cancelled;
1161	#endif /* __rtems__ */
1162
1163	cancelled = 0;
1164	#ifndef __rtems__
1165	if (cpu == -1) {
1166	ignore_cpu = 1;
1167	} else if ((cpu >= MAXCPU) \|\|
1168	((CC_CPU(cpu))->cc_inited == 0)) {
1169	/* Invalid CPU spec */
1170	panic("Invalid CPU in callout %d", cpu);
1171	}
1172	#endif /* __rtems__ */
1173	callout_when(sbt, prec, flags, &to_sbt, &precision);
1174
1175	#ifndef __rtems__
1176	/*
1177	* This flag used to be added by callout_cc_add, but the
1178	* first time you call this we could end up with the
1179	* wrong direct flag if we don't do it before we add.
1180	*/
1181	if (flags & C_DIRECT_EXEC) {
1182	direct = 1;
1183	} else {
1184	direct = 0;
1185	}
1186	KASSERT(!direct \|\| c->c_lock == NULL,
1187	("%s: direct callout %p has lock", __func__, c));
1188	#endif /* __rtems__ */
1189	cc = callout_lock(c);
1190	#ifndef __rtems__
1191	/*
1192	* Don't allow migration of pre-allocated callouts lest they
1193	* become unbalanced or handle the case where the user does
1194	* not care.
1195	*/
1196	if ((c->c_iflags & CALLOUT_LOCAL_ALLOC) \|\|
1197	ignore_cpu) {
1198	cpu = c->c_cpu;
1199	}
1200	#endif /* __rtems__ */
1201
1202	if (cc_exec_curr(cc, direct) == c) {
1203	/*
1204	* We're being asked to reschedule a callout which is
1205	* currently in progress. If there is a lock then we
1206	* can cancel the callout if it has not really started.
1207	*/
1208	if (c->c_lock != NULL && !cc_exec_cancel(cc, direct))
1209	cancelled = cc_exec_cancel(cc, direct) = true;
1210	if (cc_exec_waiting(cc, direct) \|\| cc_exec_drain(cc, direct)) {
1211	/*
1212	* Someone has called callout_drain to kill this
1213	* callout. Don't reschedule.
1214	*/
1215	CTR4(KTR_CALLOUT, "%s %p func %p arg %p",
1216	cancelled ? "cancelled" : "failed to cancel",
1217	c, c->c_func, c->c_arg);
1218	CC_UNLOCK(cc);
1219	return (cancelled);
1220	}
1221	#ifdef SMP
1222	if (callout_migrating(c)) {
1223	/*
1224	* This only occurs when a second callout_reset_sbt_on
1225	* is made after a previous one moved it into
1226	* deferred migration (below). Note we do not change
1227	* the prev_cpu even though the previous target may
1228	* be different.
1229	*/
1230	cc_migration_cpu(cc, direct) = cpu;
1231	cc_migration_time(cc, direct) = to_sbt;
1232	cc_migration_prec(cc, direct) = precision;
1233	cc_migration_func(cc, direct) = ftn;
1234	cc_migration_arg(cc, direct) = arg;
1235	cancelled = 1;
1236	CC_UNLOCK(cc);
1237	return (cancelled);
1238	}
1239	#endif
1240	}
1241	if (c->c_iflags & CALLOUT_PENDING) {
1242	#ifndef __rtems__
1243	if ((c->c_iflags & CALLOUT_PROCESSED) == 0) {
1244	#endif /* __rtems__ */
1245	if (cc_exec_next(cc) == c)
1246	cc_exec_next(cc) = LIST_NEXT(c, c_links.le);
1247	LIST_REMOVE(c, c_links.le);
1248	#ifndef __rtems__
1249	} else {
1250	TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe);
1251	}
1252	#endif /* __rtems__ */
1253	cancelled = 1;
1254	c->c_iflags &= ~ CALLOUT_PENDING;
1255	c->c_flags &= ~ CALLOUT_ACTIVE;
1256	}
1257
1258	#ifdef SMP
1259	/*
1260	* If the callout must migrate try to perform it immediately.
1261	* If the callout is currently running, just defer the migration
1262	* to a more appropriate moment.
1263	*/
1264	if (c->c_cpu != cpu) {
1265	if (cc_exec_curr(cc, direct) == c) {
1266	/*
1267	* Pending will have been removed since we are
1268	* actually executing the callout on another
1269	* CPU. That callout should be waiting on the
1270	* lock the caller holds. If we set both
1271	* active/and/pending after we return and the
1272	* lock on the executing callout proceeds, it
1273	* will then see pending is true and return.
1274	* At the return from the actual callout execution
1275	* the migration will occur in softclock_call_cc
1276	* and this new callout will be placed on the
1277	* new CPU via a call to callout_cpu_switch() which
1278	* will get the lock on the right CPU followed
1279	* by a call callout_cc_add() which will add it there.
1280	* (see above in softclock_call_cc()).
1281	*/
1282	cc_migration_cpu(cc, direct) = cpu;
1283	cc_migration_time(cc, direct) = to_sbt;
1284	cc_migration_prec(cc, direct) = precision;
1285	cc_migration_func(cc, direct) = ftn;
1286	cc_migration_arg(cc, direct) = arg;
1287	c->c_iflags \|= (CALLOUT_DFRMIGRATION \| CALLOUT_PENDING);
1288	c->c_flags \|= CALLOUT_ACTIVE;
1289	CTR6(KTR_CALLOUT,
1290	"migration of %p func %p arg %p in %d.%08x to %u deferred",
1291	c, c->c_func, c->c_arg, (int)(to_sbt >> 32),
1292	(u_int)(to_sbt & 0xffffffff), cpu);
1293	CC_UNLOCK(cc);
1294	return (cancelled);
1295	}
1296	cc = callout_cpu_switch(c, cc, cpu);
1297	}
1298	#endif
1299
1300	callout_cc_add(c, cc, to_sbt, precision, ftn, arg, cpu, flags);
1301	CTR6(KTR_CALLOUT, "%sscheduled %p func %p arg %p in %d.%08x",
1302	cancelled ? "re" : "", c, c->c_func, c->c_arg, (int)(to_sbt >> 32),
1303	(u_int)(to_sbt & 0xffffffff));
1304	CC_UNLOCK(cc);
1305
1306	return (cancelled);
1307	}
1308
1309	/*
1310	* Common idioms that can be optimized in the future.
1311	*/
1312	int
1313	callout_schedule_on(struct callout *c, int to_ticks, int cpu)
1314	{
1315	return callout_reset_on(c, to_ticks, c->c_func, c->c_arg, cpu);
1316	}
1317
1318	int
1319	callout_schedule(struct callout *c, int to_ticks)
1320	{
1321	#ifndef __rtems__
1322	return callout_reset_on(c, to_ticks, c->c_func, c->c_arg, c->c_cpu);
1323	#else /* __rtems__ */
1324	return callout_reset_on(c, to_ticks, c->c_func, c->c_arg, 0);
1325	#endif /* __rtems__ */
1326	}
1327
1328	int
1329	_callout_stop_safe(struct callout c, int flags, void (drain)(void *))
1330	{
1331	#ifndef __rtems__
1332	struct callout_cpu cc, old_cc;
1333	struct lock_class *class;
1334	int direct, sq_locked, use_lock;
1335	int cancelled, not_on_a_list;
1336	#else /* __rtems__ */
1337	struct callout_cpu *cc;
1338	struct lock_class *class;
1339	int use_lock;
1340	int cancelled;
1341	#endif /* __rtems__ */
1342
1343	if ((flags & CS_DRAIN) != 0)
1344	WITNESS_WARN(WARN_GIANTOK \| WARN_SLEEPOK, c->c_lock,
1345	"calling %s", __func__);
1346
1347	/*
1348	* Some old subsystems don't hold Giant while running a callout_stop(),
1349	* so just discard this check for the moment.
1350	*/
1351	if ((flags & CS_DRAIN) == 0 && c->c_lock != NULL) {
1352	if (c->c_lock == &Giant.lock_object)
1353	use_lock = mtx_owned(&Giant);
1354	else {
1355	use_lock = 1;
1356	class = LOCK_CLASS(c->c_lock);
1357	class->lc_assert(c->c_lock, LA_XLOCKED);
1358	}
1359	} else
1360	use_lock = 0;
1361	#ifndef __rtems__
1362	if (c->c_iflags & CALLOUT_DIRECT) {
1363	direct = 1;
1364	} else {
1365	direct = 0;
1366	}
1367
1368	sq_locked = 0;
1369	old_cc = NULL;
1370	again:
1371	#endif /* __rtems__ */
1372	cc = callout_lock(c);
1373
1374	#ifndef __rtems__
1375	if ((c->c_iflags & (CALLOUT_DFRMIGRATION \| CALLOUT_PENDING)) ==
1376	(CALLOUT_DFRMIGRATION \| CALLOUT_PENDING) &&
1377	((c->c_flags & CALLOUT_ACTIVE) == CALLOUT_ACTIVE)) {
1378	/*
1379	* Special case where this slipped in while we
1380	* were migrating as the callout is about to
1381	* execute. The caller probably holds the lock
1382	* the callout wants.
1383	*
1384	* Get rid of the migration first. Then set
1385	* the flag that tells this code not to
1386	* try to remove it from any lists (its not
1387	* on one yet). When the callout wheel runs,
1388	* it will ignore this callout.
1389	*/
1390	c->c_iflags &= ~CALLOUT_PENDING;
1391	c->c_flags &= ~CALLOUT_ACTIVE;
1392	not_on_a_list = 1;
1393	} else {
1394	not_on_a_list = 0;
1395	}
1396
1397	/*
1398	* If the callout was migrating while the callout cpu lock was
1399	* dropped, just drop the sleepqueue lock and check the states
1400	* again.
1401	*/
1402	if (sq_locked != 0 && cc != old_cc) {
1403	#ifdef SMP
1404	CC_UNLOCK(cc);
1405	sleepq_release(&cc_exec_waiting(old_cc, direct));
1406	sq_locked = 0;
1407	old_cc = NULL;
1408	goto again;
1409	#else
1410	panic("migration should not happen");
1411	#endif
1412	}
1413	#endif /* __rtems__ */
1414
1415	/*
1416	* If the callout is running, try to stop it or drain it.
1417	*/
1418	if (cc_exec_curr(cc, direct) == c) {
1419	/*
1420	* Succeed we to stop it or not, we must clear the
1421	* active flag - this is what API users expect. If we're
1422	* draining and the callout is currently executing, first wait
1423	* until it finishes.
1424	*/
1425	if ((flags & CS_DRAIN) == 0)
1426	c->c_flags &= ~CALLOUT_ACTIVE;
1427
1428	if ((flags & CS_DRAIN) != 0) {
1429	/*
1430	* The current callout is running (or just
1431	* about to run) and blocking is allowed, so
1432	* just wait for the current invocation to
1433	* finish.
1434	*/
1435	while (cc_exec_curr(cc, direct) == c) {
1436	#ifndef __rtems__
1437
1438	/*
1439	* Use direct calls to sleepqueue interface
1440	* instead of cv/msleep in order to avoid
1441	* a LOR between cc_lock and sleepqueue
1442	* chain spinlocks. This piece of code
1443	* emulates a msleep_spin() call actually.
1444	*
1445	* If we already have the sleepqueue chain
1446	* locked, then we can safely block. If we
1447	* don't already have it locked, however,
1448	* we have to drop the cc_lock to lock
1449	* it. This opens several races, so we
1450	* restart at the beginning once we have
1451	* both locks. If nothing has changed, then
1452	* we will end up back here with sq_locked
1453	* set.
1454	*/
1455	if (!sq_locked) {
1456	CC_UNLOCK(cc);
1457	sleepq_lock(
1458	&cc_exec_waiting(cc, direct));
1459	sq_locked = 1;
1460	old_cc = cc;
1461	goto again;
1462	}
1463
1464	/*
1465	* Migration could be cancelled here, but
1466	* as long as it is still not sure when it
1467	* will be packed up, just let softclock()
1468	* take care of it.
1469	*/
1470	cc_exec_waiting(cc, direct) = true;
1471	DROP_GIANT();
1472	CC_UNLOCK(cc);
1473	sleepq_add(
1474	&cc_exec_waiting(cc, direct),
1475	&cc->cc_lock.lock_object, "codrain",
1476	SLEEPQ_SLEEP, 0);
1477	sleepq_wait(
1478	&cc_exec_waiting(cc, direct),
1479	0);
1480	sq_locked = 0;
1481	old_cc = NULL;
1482
1483	/* Reacquire locks previously released. */
1484	PICKUP_GIANT();
1485	CC_LOCK(cc);
1486	#else /* __rtems__ */
1487	/*
1488	* On RTEMS the LOR problem above does not
1489	* exist since here we do not use
1490	* sleepq_set_timeout() and instead use the
1491	* RTEMS watchdog.
1492	*/
1493	cc_exec_waiting(cc, direct) = true;
1494	msleep_spin(&cc_exec_waiting(cc, direct),
1495	&cc->cc_lock, "codrain", 0);
1496	#endif /* __rtems__ */
1497	}
1498	c->c_flags &= ~CALLOUT_ACTIVE;
1499	} else if (use_lock &&
1500	!cc_exec_cancel(cc, direct) && (drain == NULL)) {
1501
1502	/*
1503	* The current callout is waiting for its
1504	* lock which we hold. Cancel the callout
1505	* and return. After our caller drops the
1506	* lock, the callout will be skipped in
1507	* softclock(). This only works with a
1508	* callout_stop() not callout_drain() or
1509	* callout_async_drain().
1510	*/
1511	cc_exec_cancel(cc, direct) = true;
1512	CTR3(KTR_CALLOUT, "cancelled %p func %p arg %p",
1513	c, c->c_func, c->c_arg);
1514	#ifndef __rtems__
1515	KASSERT(!cc_cce_migrating(cc, direct),
1516	("callout wrongly scheduled for migration"));
1517	if (callout_migrating(c)) {
1518	c->c_iflags &= ~CALLOUT_DFRMIGRATION;
1519	#ifdef SMP
1520	cc_migration_cpu(cc, direct) = CPUBLOCK;
1521	cc_migration_time(cc, direct) = 0;
1522	cc_migration_prec(cc, direct) = 0;
1523	cc_migration_func(cc, direct) = NULL;
1524	cc_migration_arg(cc, direct) = NULL;
1525	#endif
1526	}
1527	#endif /* __rtems__ */
1528	CC_UNLOCK(cc);
1529	#ifndef __rtems__
1530	KASSERT(!sq_locked, ("sleepqueue chain locked"));
1531	#endif /* __rtems__ */
1532	return (1);
1533	#ifndef __rtems__
1534	} else if (callout_migrating(c)) {
1535	/*
1536	* The callout is currently being serviced
1537	* and the "next" callout is scheduled at
1538	* its completion with a migration. We remove
1539	* the migration flag so it won't get rescheduled,
1540	* but we can't stop the one thats running so
1541	* we return 0.
1542	*/
1543	c->c_iflags &= ~CALLOUT_DFRMIGRATION;
1544	#ifdef SMP
1545	/*
1546	* We can't call cc_cce_cleanup here since
1547	* if we do it will remove .ce_curr and
1548	* its still running. This will prevent a
1549	* reschedule of the callout when the
1550	* execution completes.
1551	*/
1552	cc_migration_cpu(cc, direct) = CPUBLOCK;
1553	cc_migration_time(cc, direct) = 0;
1554	cc_migration_prec(cc, direct) = 0;
1555	cc_migration_func(cc, direct) = NULL;
1556	cc_migration_arg(cc, direct) = NULL;
1557	#endif
1558	CTR3(KTR_CALLOUT, "postponing stop %p func %p arg %p",
1559	c, c->c_func, c->c_arg);
1560	if (drain) {
1561	cc_exec_drain(cc, direct) = drain;
1562	}
1563	CC_UNLOCK(cc);
1564	return ((flags & CS_EXECUTING) != 0);
1565	#endif /* __rtems__ */
1566	}
1567	CTR3(KTR_CALLOUT, "failed to stop %p func %p arg %p",
1568	c, c->c_func, c->c_arg);
1569	if (drain) {
1570	cc_exec_drain(cc, direct) = drain;
1571	}
1572	#ifndef __rtems__
1573	KASSERT(!sq_locked, ("sleepqueue chain still locked"));
1574	#endif /* __rtems__ */
1575	cancelled = ((flags & CS_EXECUTING) != 0);
1576	} else
1577	cancelled = 1;
1578
1579	#ifndef __rtems__
1580	if (sq_locked)
1581	sleepq_release(&cc_exec_waiting(cc, direct));
1582	#endif /* __rtems__ */
1583
1584	if ((c->c_iflags & CALLOUT_PENDING) == 0) {
1585	CTR3(KTR_CALLOUT, "failed to stop %p func %p arg %p",
1586	c, c->c_func, c->c_arg);
1587	/*
1588	* For not scheduled and not executing callout return
1589	* negative value.
1590	*/
1591	if (cc_exec_curr(cc, direct) != c)
1592	cancelled = -1;
1593	CC_UNLOCK(cc);
1594	return (cancelled);
1595	}
1596
1597	c->c_iflags &= ~CALLOUT_PENDING;
1598	c->c_flags &= ~CALLOUT_ACTIVE;
1599
1600	CTR3(KTR_CALLOUT, "cancelled %p func %p arg %p",
1601	c, c->c_func, c->c_arg);
1602	#ifndef __rtems__
1603	if (not_on_a_list == 0) {
1604	if ((c->c_iflags & CALLOUT_PROCESSED) == 0) {
1605	#endif /* __rtems__ */
1606	if (cc_exec_next(cc) == c)
1607	cc_exec_next(cc) = LIST_NEXT(c, c_links.le);
1608	LIST_REMOVE(c, c_links.le);
1609	#ifndef __rtems__
1610	} else {
1611	TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe);
1612	}
1613	}
1614	#endif /* __rtems__ */
1615	callout_cc_del(c, cc);
1616	CC_UNLOCK(cc);
1617	return (cancelled);
1618	}
1619
1620	void
1621	callout_init(struct callout *c, int mpsafe)
1622	{
1623	bzero(c, sizeof *c);
1624	if (mpsafe) {
1625	c->c_lock = NULL;
1626	c->c_iflags = CALLOUT_RETURNUNLOCKED;
1627	} else {
1628	c->c_lock = &Giant.lock_object;
1629	c->c_iflags = 0;
1630	}
1631	#ifndef __rtems__
1632	c->c_cpu = timeout_cpu;
1633	#endif /* __rtems__ */
1634	}
1635
1636	void
1637	_callout_init_lock(struct callout c, struct lock_object lock, int flags)
1638	{
1639	bzero(c, sizeof *c);
1640	c->c_lock = lock;
1641	KASSERT((flags & ~(CALLOUT_RETURNUNLOCKED \| CALLOUT_SHAREDLOCK)) == 0,
1642	("callout_init_lock: bad flags %d", flags));
1643	KASSERT(lock != NULL \|\| (flags & CALLOUT_RETURNUNLOCKED) == 0,
1644	("callout_init_lock: CALLOUT_RETURNUNLOCKED with no lock"));
1645	KASSERT(lock == NULL \|\| !(LOCK_CLASS(lock)->lc_flags &
1646	(LC_SPINLOCK \| LC_SLEEPABLE)), ("%s: invalid lock class",
1647	__func__));
1648	c->c_iflags = flags & (CALLOUT_RETURNUNLOCKED \| CALLOUT_SHAREDLOCK);
1649	#ifndef __rtems__
1650	c->c_cpu = timeout_cpu;
1651	#endif /* __rtems__ */
1652	}
1653
1654	#ifdef APM_FIXUP_CALLTODO
1655	/*
1656	* Adjust the kernel calltodo timeout list. This routine is used after
1657	* an APM resume to recalculate the calltodo timer list values with the
1658	* number of hz's we have been sleeping. The next hardclock() will detect
1659	* that there are fired timers and run softclock() to execute them.
1660	*
1661	* Please note, I have not done an exhaustive analysis of what code this
1662	* might break. I am motivated to have my select()'s and alarm()'s that
1663	* have expired during suspend firing upon resume so that the applications
1664	* which set the timer can do the maintanence the timer was for as close
1665	* as possible to the originally intended time. Testing this code for a
1666	* week showed that resuming from a suspend resulted in 22 to 25 timers
1667	* firing, which seemed independent on whether the suspend was 2 hours or
1668	* 2 days. Your milage may vary. - Ken Key <key@cs.utk.edu>
1669	*/
1670	void
1671	adjust_timeout_calltodo(struct timeval *time_change)
1672	{
1673	register struct callout *p;
1674	unsigned long delta_ticks;
1675
1676	/*
1677	* How many ticks were we asleep?
1678	* (stolen from tvtohz()).
1679	*/
1680
1681	/* Don't do anything */
1682	if (time_change->tv_sec < 0)
1683	return;
1684	else if (time_change->tv_sec <= LONG_MAX / 1000000)
1685	delta_ticks = howmany(time_change->tv_sec * 1000000 +
1686	time_change->tv_usec, tick) + 1;
1687	else if (time_change->tv_sec <= LONG_MAX / hz)
1688	delta_ticks = time_change->tv_sec * hz +
1689	howmany(time_change->tv_usec, tick) + 1;
1690	else
1691	delta_ticks = LONG_MAX;
1692
1693	if (delta_ticks > INT_MAX)
1694	delta_ticks = INT_MAX;
1695
1696	/*
1697	* Now rip through the timer calltodo list looking for timers
1698	* to expire.
1699	*/
1700
1701	/* don't collide with softclock() */
1702	CC_LOCK(cc);
1703	for (p = calltodo.c_next; p != NULL; p = p->c_next) {
1704	p->c_time -= delta_ticks;
1705
1706	/* Break if the timer had more time on it than delta_ticks */
1707	if (p->c_time > 0)
1708	break;
1709
1710	/* take back the ticks the timer didn't use (p->c_time <= 0) */
1711	delta_ticks = -p->c_time;
1712	}
1713	CC_UNLOCK(cc);
1714
1715	return;
1716	}
1717	#endif /* APM_FIXUP_CALLTODO */
1718
1719	static int
1720	flssbt(sbintime_t sbt)
1721	{
1722
1723	sbt += (uint64_t)sbt >> 1;
1724	if (sizeof(long) >= sizeof(sbintime_t))
1725	return (flsl(sbt));
1726	if (sbt >= SBT_1S)
1727	return (flsl(((uint64_t)sbt) >> 32) + 32);
1728	return (flsl(sbt));
1729	}
1730
1731	/*
1732	* Dump immediate statistic snapshot of the scheduled callouts.
1733	*/
1734	static int
1735	sysctl_kern_callout_stat(SYSCTL_HANDLER_ARGS)
1736	{
1737	struct callout *tmp;
1738	struct callout_cpu *cc;
1739	struct callout_list *sc;
1740	sbintime_t maxpr, maxt, medpr, medt, now, spr, st, t;
1741	int ct[64], cpr[64], ccpbk[32];
1742	int error, val, i, count, tcum, pcum, maxc, c, medc;
1743	#ifdef SMP
1744	int cpu;
1745	#endif
1746
1747	val = 0;
1748	error = sysctl_handle_int(oidp, &val, 0, req);
1749	if (error != 0 \|\| req->newptr == NULL)
1750	return (error);
1751	count = maxc = 0;
1752	st = spr = maxt = maxpr = 0;
1753	bzero(ccpbk, sizeof(ccpbk));
1754	bzero(ct, sizeof(ct));
1755	bzero(cpr, sizeof(cpr));
1756	now = sbinuptime();
1757	#ifdef SMP
1758	CPU_FOREACH(cpu) {
1759	cc = CC_CPU(cpu);
1760	#else
1761	cc = CC_CPU(timeout_cpu);
1762	#endif
1763	CC_LOCK(cc);
1764	for (i = 0; i < callwheelsize; i++) {
1765	sc = &cc->cc_callwheel[i];
1766	c = 0;
1767	LIST_FOREACH(tmp, sc, c_links.le) {
1768	c++;
1769	t = tmp->c_time - now;
1770	if (t < 0)
1771	t = 0;
1772	st += t / SBT_1US;
1773	spr += tmp->c_precision / SBT_1US;
1774	if (t > maxt)
1775	maxt = t;
1776	if (tmp->c_precision > maxpr)
1777	maxpr = tmp->c_precision;
1778	ct[flssbt(t)]++;
1779	cpr[flssbt(tmp->c_precision)]++;
1780	}
1781	if (c > maxc)
1782	maxc = c;
1783	ccpbk[fls(c + c / 2)]++;
1784	count += c;
1785	}
1786	CC_UNLOCK(cc);
1787	#ifdef SMP
1788	}
1789	#endif
1790
1791	for (i = 0, tcum = 0; i < 64 && tcum < count / 2; i++)
1792	tcum += ct[i];
1793	medt = (i >= 2) ? (((sbintime_t)1) << (i - 2)) : 0;
1794	for (i = 0, pcum = 0; i < 64 && pcum < count / 2; i++)
1795	pcum += cpr[i];
1796	medpr = (i >= 2) ? (((sbintime_t)1) << (i - 2)) : 0;
1797	for (i = 0, c = 0; i < 32 && c < count / 2; i++)
1798	c += ccpbk[i];
1799	medc = (i >= 2) ? (1 << (i - 2)) : 0;
1800
1801	printf("Scheduled callouts statistic snapshot:\n");
1802	printf(" Callouts: %6d Buckets: %6d*%-3d Bucket size: 0.%06ds\n",
1803	count, callwheelsize, mp_ncpus, 1000000 >> CC_HASH_SHIFT);
1804	printf(" C/Bk: med %5d avg %6d.%06jd max %6d\n",
1805	medc,
1806	count / callwheelsize / mp_ncpus,
1807	(uint64_t)count * 1000000 / callwheelsize / mp_ncpus % 1000000,
1808	maxc);
1809	printf(" Time: med %5jd.%06jds avg %6jd.%06jds max %6jd.%06jds\n",
1810	medt / SBT_1S, (medt & 0xffffffff) * 1000000 >> 32,
1811	(st / count) / 1000000, (st / count) % 1000000,
1812	maxt / SBT_1S, (maxt & 0xffffffff) * 1000000 >> 32);
1813	printf(" Prec: med %5jd.%06jds avg %6jd.%06jds max %6jd.%06jds\n",
1814	medpr / SBT_1S, (medpr & 0xffffffff) * 1000000 >> 32,
1815	(spr / count) / 1000000, (spr / count) % 1000000,
1816	maxpr / SBT_1S, (maxpr & 0xffffffff) * 1000000 >> 32);
1817	printf(" Distribution: \tbuckets\t time\t tcum\t"
1818	" prec\t pcum\n");
1819	for (i = 0, tcum = pcum = 0; i < 64; i++) {
1820	if (ct[i] == 0 && cpr[i] == 0)
1821	continue;
1822	t = (i != 0) ? (((sbintime_t)1) << (i - 1)) : 0;
1823	tcum += ct[i];
1824	pcum += cpr[i];
1825	printf(" %10jd.%06jds\t 2**%d\t%7d\t%7d\t%7d\t%7d\n",
1826	t / SBT_1S, (t & 0xffffffff) * 1000000 >> 32,
1827	i - 1 - (32 - CC_HASH_SHIFT),
1828	ct[i], tcum, cpr[i], pcum);
1829	}
1830	return (error);
1831	}
1832	SYSCTL_PROC(_kern, OID_AUTO, callout_stat,
1833	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_MPSAFE,
1834	0, 0, sysctl_kern_callout_stat, "I",
1835	"Dump immediate statistic snapshot of the scheduled callouts");
1836
1837	#ifdef DDB
1838	static void
1839	_show_callout(struct callout *c)
1840	{
1841
1842	db_printf("callout %p\n", c);
1843	#define C_DB_PRINTF(f, e) db_printf(" %s = " f "\n", #e, c->e);
1844	db_printf(" &c_links = %p\n", &(c->c_links));
1845	C_DB_PRINTF("%" PRId64, c_time);
1846	C_DB_PRINTF("%" PRId64, c_precision);
1847	C_DB_PRINTF("%p", c_arg);
1848	C_DB_PRINTF("%p", c_func);
1849	C_DB_PRINTF("%p", c_lock);
1850	C_DB_PRINTF("%#x", c_flags);
1851	C_DB_PRINTF("%#x", c_iflags);
1852	C_DB_PRINTF("%d", c_cpu);
1853	#undef C_DB_PRINTF
1854	}
1855
1856	DB_SHOW_COMMAND(callout, db_show_callout)
1857	{
1858
1859	if (!have_addr) {
1860	db_printf("usage: show callout <struct callout *>\n");
1861	return;
1862	}
1863
1864	_show_callout((struct callout *)addr);
1865	}
1866	#endif /* DDB */

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: rtems-libbsd/freebsd/sys/kern/kern_timeout.c @ a85d900

Download in other formats: