Context Navigation

kern_timeout.c @ bb80d9d

55-freebsd-126-freebsd-12

Last change on this file since bb80d9d was bb80d9d, checked in by Sebastian Huber <sebastian.huber@…>, on 08/09/18 at 12:02:09

Update to FreeBSD head 2017-12-01

Git mirror commit e724f51f811a4b2bd29447f8b85ab5c2f9b88266.

Update #3472.

Property mode set to 100644

File size: 51.7 KB

Line
1	#include <machine/rtems-bsd-kernel-space.h>
2
3	/*-
4	* SPDX-License-Identifier: BSD-3-Clause
5	*
6	* Copyright (c) 1982, 1986, 1991, 1993
7	* The Regents of the University of California. All rights reserved.
8	* (c) UNIX System Laboratories, Inc.
9	* All or some portions of this file are derived from material licensed
10	* to the University of California by American Telephone and Telegraph
11	* Co. or Unix System Laboratories, Inc. and are reproduced herein with
12	* the permission of UNIX System Laboratories, Inc.
13	*
14	* Redistribution and use in source and binary forms, with or without
15	* modification, are permitted provided that the following conditions
16	* are met:
17	* 1. Redistributions of source code must retain the above copyright
18	* notice, this list of conditions and the following disclaimer.
19	* 2. Redistributions in binary form must reproduce the above copyright
20	* notice, this list of conditions and the following disclaimer in the
21	* documentation and/or other materials provided with the distribution.
22	* 3. Neither the name of the University nor the names of its contributors
23	* may be used to endorse or promote products derived from this software
24	* without specific prior written permission.
25	*
26	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
27	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
29	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
30	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36	* SUCH DAMAGE.
37	*
38	* From: @(#)kern_clock.c 8.5 (Berkeley) 1/21/94
39	*/
40
41	#include <sys/cdefs.h>
42	__FBSDID("$FreeBSD$");
43
44	#include <rtems/bsd/local/opt_callout_profiling.h>
45	#include <rtems/bsd/local/opt_ddb.h>
46	#if defined(__arm__) \|\| defined(__rtems__)
47	#include <rtems/bsd/local/opt_timer.h>
48	#endif
49	#include <rtems/bsd/local/opt_rss.h>
50
51	#include <sys/param.h>
52	#include <sys/systm.h>
53	#include <sys/bus.h>
54	#include <sys/callout.h>
55	#include <sys/file.h>
56	#include <sys/interrupt.h>
57	#include <sys/kernel.h>
58	#include <sys/ktr.h>
59	#include <sys/lock.h>
60	#include <sys/malloc.h>
61	#include <sys/mutex.h>
62	#include <sys/proc.h>
63	#include <sys/sdt.h>
64	#include <sys/sleepqueue.h>
65	#include <sys/sysctl.h>
66	#include <sys/smp.h>
67
68	#ifdef DDB
69	#include <ddb/ddb.h>
70	#include <machine/_inttypes.h>
71	#endif
72
73	#ifdef SMP
74	#include <machine/cpu.h>
75	#endif
76
77	#ifndef NO_EVENTTIMERS
78	DPCPU_DECLARE(sbintime_t, hardclocktime);
79	#endif
80
81	SDT_PROVIDER_DEFINE(callout_execute);
82	SDT_PROBE_DEFINE1(callout_execute, , , callout__start, "struct callout *");
83	SDT_PROBE_DEFINE1(callout_execute, , , callout__end, "struct callout *");
84
85	#ifdef CALLOUT_PROFILING
86	static int avg_depth;
87	SYSCTL_INT(_debug, OID_AUTO, to_avg_depth, CTLFLAG_RD, &avg_depth, 0,
88	"Average number of items examined per softclock call. Units = 1/1000");
89	static int avg_gcalls;
90	SYSCTL_INT(_debug, OID_AUTO, to_avg_gcalls, CTLFLAG_RD, &avg_gcalls, 0,
91	"Average number of Giant callouts made per softclock call. Units = 1/1000");
92	static int avg_lockcalls;
93	SYSCTL_INT(_debug, OID_AUTO, to_avg_lockcalls, CTLFLAG_RD, &avg_lockcalls, 0,
94	"Average number of lock callouts made per softclock call. Units = 1/1000");
95	static int avg_mpcalls;
96	SYSCTL_INT(_debug, OID_AUTO, to_avg_mpcalls, CTLFLAG_RD, &avg_mpcalls, 0,
97	"Average number of MP callouts made per softclock call. Units = 1/1000");
98	static int avg_depth_dir;
99	SYSCTL_INT(_debug, OID_AUTO, to_avg_depth_dir, CTLFLAG_RD, &avg_depth_dir, 0,
100	"Average number of direct callouts examined per callout_process call. "
101	"Units = 1/1000");
102	static int avg_lockcalls_dir;
103	SYSCTL_INT(_debug, OID_AUTO, to_avg_lockcalls_dir, CTLFLAG_RD,
104	&avg_lockcalls_dir, 0, "Average number of lock direct callouts made per "
105	"callout_process call. Units = 1/1000");
106	static int avg_mpcalls_dir;
107	SYSCTL_INT(_debug, OID_AUTO, to_avg_mpcalls_dir, CTLFLAG_RD, &avg_mpcalls_dir,
108	0, "Average number of MP direct callouts made per callout_process call. "
109	"Units = 1/1000");
110	#endif
111
112	#ifndef __rtems__
113	static int ncallout;
114	SYSCTL_INT(_kern, OID_AUTO, ncallout, CTLFLAG_RDTUN \| CTLFLAG_NOFETCH, &ncallout, 0,
115	"Number of entries in callwheel and size of timeout() preallocation");
116	#else /* __rtems__ */
117	#define ncallout 16
118	#endif /* __rtems__ */
119
120	#ifdef RSS
121	static int pin_default_swi = 1;
122	static int pin_pcpu_swi = 1;
123	#else
124	static int pin_default_swi = 0;
125	static int pin_pcpu_swi = 0;
126	#endif
127
128	SYSCTL_INT(_kern, OID_AUTO, pin_default_swi, CTLFLAG_RDTUN \| CTLFLAG_NOFETCH, &pin_default_swi,
129	0, "Pin the default (non-per-cpu) swi (shared with PCPU 0 swi)");
130	SYSCTL_INT(_kern, OID_AUTO, pin_pcpu_swi, CTLFLAG_RDTUN \| CTLFLAG_NOFETCH, &pin_pcpu_swi,
131	0, "Pin the per-CPU swis (except PCPU 0, which is also default");
132
133	#ifndef __rtems__
134	/*
135	* TODO:
136	* allocate more timeout table slots when table overflows.
137	*/
138	u_int callwheelsize, callwheelmask;
139	#else /* __rtems__ */
140	#define callwheelsize (2 * ncallout)
141	#define callwheelmask (callwheelsize - 1)
142	#endif /* __rtems__ */
143
144	/*
145	* The callout cpu exec entities represent informations necessary for
146	* describing the state of callouts currently running on the CPU and the ones
147	* necessary for migrating callouts to the new callout cpu. In particular,
148	* the first entry of the array cc_exec_entity holds informations for callout
149	* running in SWI thread context, while the second one holds informations
150	* for callout running directly from hardware interrupt context.
151	* The cached informations are very important for deferring migration when
152	* the migrating callout is already running.
153	*/
154	struct cc_exec {
155	struct callout *cc_curr;
156	void (cc_drain)(void );
157	#ifdef SMP
158	void (ce_migration_func)(void );
159	void *ce_migration_arg;
160	int ce_migration_cpu;
161	sbintime_t ce_migration_time;
162	sbintime_t ce_migration_prec;
163	#endif
164	bool cc_cancel;
165	bool cc_waiting;
166	};
167
168	/*
169	* There is one struct callout_cpu per cpu, holding all relevant
170	* state for the callout processing thread on the individual CPU.
171	*/
172	struct callout_cpu {
173	struct mtx_padalign cc_lock;
174	#ifndef __rtems__
175	struct cc_exec cc_exec_entity[2];
176	#else /* __rtems__ */
177	struct cc_exec cc_exec_entity;
178	#endif /* __rtems__ */
179	struct callout *cc_next;
180	#ifndef __rtems__
181	struct callout *cc_callout;
182	struct callout_list *cc_callwheel;
183	struct callout_tailq cc_expireq;
184	#else /* __rtems__ */
185	struct callout cc_callout[ncallout];
186	struct callout_list cc_callwheel[callwheelsize];
187	#endif /* __rtems__ */
188	struct callout_slist cc_callfree;
189	sbintime_t cc_firstevent;
190	sbintime_t cc_lastscan;
191	void *cc_cookie;
192	u_int cc_bucket;
193	u_int cc_inited;
194	char cc_ktr_event_name[20];
195	};
196
197	#ifndef __rtems__
198	#define callout_migrating(c) ((c)->c_iflags & CALLOUT_DFRMIGRATION)
199	#endif /* __rtems__ */
200
201	#ifndef __rtems__
202	#define cc_exec_curr(cc, dir) cc->cc_exec_entity[dir].cc_curr
203	#define cc_exec_drain(cc, dir) cc->cc_exec_entity[dir].cc_drain
204	#else /* __rtems__ */
205	#define cc_exec_curr(cc, dir) cc->cc_exec_entity.cc_curr
206	#define cc_exec_drain(cc, dir) cc->cc_exec_entity.cc_drain
207	#endif /* __rtems__ */
208	#define cc_exec_next(cc) cc->cc_next
209	#ifndef __rtems__
210	#define cc_exec_cancel(cc, dir) cc->cc_exec_entity[dir].cc_cancel
211	#define cc_exec_waiting(cc, dir) cc->cc_exec_entity[dir].cc_waiting
212	#else /* __rtems__ */
213	#define cc_exec_cancel(cc, dir) cc->cc_exec_entity.cc_cancel
214	#define cc_exec_waiting(cc, dir) cc->cc_exec_entity.cc_waiting
215	#endif /* __rtems__ */
216	#ifdef SMP
217	#define cc_migration_func(cc, dir) cc->cc_exec_entity[dir].ce_migration_func
218	#define cc_migration_arg(cc, dir) cc->cc_exec_entity[dir].ce_migration_arg
219	#define cc_migration_cpu(cc, dir) cc->cc_exec_entity[dir].ce_migration_cpu
220	#define cc_migration_time(cc, dir) cc->cc_exec_entity[dir].ce_migration_time
221	#define cc_migration_prec(cc, dir) cc->cc_exec_entity[dir].ce_migration_prec
222
223	struct callout_cpu cc_cpu[MAXCPU];
224	#define CPUBLOCK MAXCPU
225	#define CC_CPU(cpu) (&cc_cpu[(cpu)])
226	#define CC_SELF() CC_CPU(PCPU_GET(cpuid))
227	#else
228	struct callout_cpu cc_cpu;
229	#define CC_CPU(cpu) &cc_cpu
230	#define CC_SELF() &cc_cpu
231	#endif
232	#define CC_LOCK(cc) mtx_lock_spin(&(cc)->cc_lock)
233	#define CC_UNLOCK(cc) mtx_unlock_spin(&(cc)->cc_lock)
234	#define CC_LOCK_ASSERT(cc) mtx_assert(&(cc)->cc_lock, MA_OWNED)
235
236	#ifndef __rtems__
237	static int timeout_cpu;
238	#else /* __rtems__ */
239	#define timeout_cpu 0
240	#endif /* __rtems__ */
241
242	static void callout_cpu_init(struct callout_cpu *cc, int cpu);
243	static void softclock_call_cc(struct callout c, struct callout_cpu cc,
244	#ifdef CALLOUT_PROFILING
245	int mpcalls, int lockcalls, int *gcalls,
246	#endif
247	int direct);
248
249	static MALLOC_DEFINE(M_CALLOUT, "callout", "Callout datastructures");
250
251	/**
252	* Locked by cc_lock:
253	* cc_curr - If a callout is in progress, it is cc_curr.
254	* If cc_curr is non-NULL, threads waiting in
255	* callout_drain() will be woken up as soon as the
256	* relevant callout completes.
257	* cc_cancel - Changing to 1 with both callout_lock and cc_lock held
258	* guarantees that the current callout will not run.
259	* The softclock() function sets this to 0 before it
260	* drops callout_lock to acquire c_lock, and it calls
261	* the handler only if curr_cancelled is still 0 after
262	* cc_lock is successfully acquired.
263	* cc_waiting - If a thread is waiting in callout_drain(), then
264	* callout_wait is nonzero. Set only when
265	* cc_curr is non-NULL.
266	*/
267
268	/*
269	* Resets the execution entity tied to a specific callout cpu.
270	*/
271	static void
272	cc_cce_cleanup(struct callout_cpu *cc, int direct)
273	{
274
275	cc_exec_curr(cc, direct) = NULL;
276	cc_exec_cancel(cc, direct) = false;
277	cc_exec_waiting(cc, direct) = false;
278	#ifdef SMP
279	cc_migration_cpu(cc, direct) = CPUBLOCK;
280	cc_migration_time(cc, direct) = 0;
281	cc_migration_prec(cc, direct) = 0;
282	cc_migration_func(cc, direct) = NULL;
283	cc_migration_arg(cc, direct) = NULL;
284	#endif
285	}
286
287	#ifndef __rtems__
288	/*
289	* Checks if migration is requested by a specific callout cpu.
290	*/
291	static int
292	cc_cce_migrating(struct callout_cpu *cc, int direct)
293	{
294
295	#ifdef SMP
296	return (cc_migration_cpu(cc, direct) != CPUBLOCK);
297	#else
298	return (0);
299	#endif
300	}
301	#endif /* __rtems__ */
302
303	/*
304	* Kernel low level callwheel initialization
305	* called on the BSP during kernel startup.
306	*/
307	#ifdef __rtems__
308	static void rtems_bsd_timeout_init_early(void *);
309
310	static void
311	rtems_bsd_callout_timer(rtems_id id, void *arg)
312	{
313	rtems_status_code sc;
314
315	(void) arg;
316
317	sc = rtems_timer_reset(id);
318	BSD_ASSERT(sc == RTEMS_SUCCESSFUL);
319
320	callout_process(sbinuptime());
321	}
322
323	static void
324	rtems_bsd_timeout_init_late(void *unused)
325	{
326	rtems_status_code sc;
327	rtems_id id;
328
329	(void) unused;
330
331	sc = rtems_timer_create(rtems_build_name('_', 'C', 'L', 'O'), &id);
332	BSD_ASSERT(sc == RTEMS_SUCCESSFUL);
333
334	sc = rtems_timer_server_fire_after(id, 1, rtems_bsd_callout_timer, NULL);
335	BSD_ASSERT(sc == RTEMS_SUCCESSFUL);
336	}
337
338	SYSINIT(rtems_bsd_timeout_early, SI_SUB_VM, SI_ORDER_FIRST,
339	rtems_bsd_timeout_init_early, NULL);
340
341	SYSINIT(rtems_bsd_timeout_late, SI_SUB_LAST, SI_ORDER_FIRST,
342	rtems_bsd_timeout_init_late, NULL);
343
344	static void
345	rtems_bsd_timeout_init_early(void *dummy)
346	#else /* __rtems__ */
347	static void
348	callout_callwheel_init(void *dummy)
349	#endif /* __rtems__ */
350	{
351	struct callout_cpu *cc;
352	#ifdef __rtems__
353	(void) dummy;
354	#endif /* __rtems__ */
355
356	/*
357	* Calculate the size of the callout wheel and the preallocated
358	* timeout() structures.
359	* XXX: Clip callout to result of previous function of maxusers
360	* maximum 384. This is still huge, but acceptable.
361	*/
362	memset(CC_CPU(curcpu), 0, sizeof(cc_cpu));
363	#ifndef __rtems__
364	ncallout = imin(16 + maxproc + maxfiles, 18508);
365	TUNABLE_INT_FETCH("kern.ncallout", &ncallout);
366	#endif /* __rtems__ */
367
368	/*
369	* Calculate callout wheel size, should be next power of two higher
370	* than 'ncallout'.
371	*/
372	#ifndef __rtems__
373	callwheelsize = 1 << fls(ncallout);
374	callwheelmask = callwheelsize - 1;
375	#endif /* __rtems__ */
376
377	#ifndef __rtems__
378	/*
379	* Fetch whether we're pinning the swi's or not.
380	*/
381	TUNABLE_INT_FETCH("kern.pin_default_swi", &pin_default_swi);
382	TUNABLE_INT_FETCH("kern.pin_pcpu_swi", &pin_pcpu_swi);
383	#endif /* __rtems__ */
384
385	/*
386	* Only BSP handles timeout(9) and receives a preallocation.
387	*
388	* XXX: Once all timeout(9) consumers are converted this can
389	* be removed.
390	*/
391	#ifndef __rtems__
392	timeout_cpu = PCPU_GET(cpuid);
393	#endif /* __rtems__ */
394	cc = CC_CPU(timeout_cpu);
395	#ifndef __rtems__
396	cc->cc_callout = malloc(ncallout * sizeof(struct callout),
397	M_CALLOUT, M_WAITOK);
398	#endif /* __rtems__ */
399	callout_cpu_init(cc, timeout_cpu);
400	}
401	#ifndef __rtems__
402	SYSINIT(callwheel_init, SI_SUB_CPU, SI_ORDER_ANY, callout_callwheel_init, NULL);
403	#endif /* __rtems__ */
404
405	/*
406	* Initialize the per-cpu callout structures.
407	*/
408	static void
409	callout_cpu_init(struct callout_cpu *cc, int cpu)
410	{
411	struct callout *c;
412	int i;
413
414	mtx_init(&cc->cc_lock, "callout", NULL, MTX_SPIN \| MTX_RECURSE);
415	SLIST_INIT(&cc->cc_callfree);
416	cc->cc_inited = 1;
417	#ifndef __rtems__
418	cc->cc_callwheel = malloc(sizeof(struct callout_list) * callwheelsize,
419	M_CALLOUT, M_WAITOK);
420	#endif /* __rtems__ */
421	for (i = 0; i < callwheelsize; i++)
422	LIST_INIT(&cc->cc_callwheel[i]);
423	#ifndef __rtems__
424	TAILQ_INIT(&cc->cc_expireq);
425	#endif /* __rtems__ */
426	cc->cc_firstevent = SBT_MAX;
427	for (i = 0; i < 2; i++)
428	cc_cce_cleanup(cc, i);
429	snprintf(cc->cc_ktr_event_name, sizeof(cc->cc_ktr_event_name),
430	"callwheel cpu %d", cpu);
431	#ifndef __rtems__
432	if (cc->cc_callout == NULL) /* Only BSP handles timeout(9) */
433	return;
434	#endif /* __rtems__ */
435	for (i = 0; i < ncallout; i++) {
436	c = &cc->cc_callout[i];
437	callout_init(c, 0);
438	c->c_iflags = CALLOUT_LOCAL_ALLOC;
439	SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle);
440	}
441	}
442
443	#ifdef SMP
444	/*
445	* Switches the cpu tied to a specific callout.
446	* The function expects a locked incoming callout cpu and returns with
447	* locked outcoming callout cpu.
448	*/
449	static struct callout_cpu *
450	callout_cpu_switch(struct callout c, struct callout_cpu cc, int new_cpu)
451	{
452	struct callout_cpu *new_cc;
453
454	MPASS(c != NULL && cc != NULL);
455	CC_LOCK_ASSERT(cc);
456
457	/*
458	* Avoid interrupts and preemption firing after the callout cpu
459	* is blocked in order to avoid deadlocks as the new thread
460	* may be willing to acquire the callout cpu lock.
461	*/
462	c->c_cpu = CPUBLOCK;
463	spinlock_enter();
464	CC_UNLOCK(cc);
465	new_cc = CC_CPU(new_cpu);
466	CC_LOCK(new_cc);
467	spinlock_exit();
468	c->c_cpu = new_cpu;
469	return (new_cc);
470	}
471	#endif
472
473	#ifndef __rtems__
474	/*
475	* Start standard softclock thread.
476	*/
477	static void
478	start_softclock(void *dummy)
479	{
480	struct callout_cpu *cc;
481	char name[MAXCOMLEN];
482	#ifdef SMP
483	int cpu;
484	struct intr_event *ie;
485	#endif
486
487	cc = CC_CPU(timeout_cpu);
488	snprintf(name, sizeof(name), "clock (%d)", timeout_cpu);
489	if (swi_add(&clk_intr_event, name, softclock, cc, SWI_CLOCK,
490	INTR_MPSAFE, &cc->cc_cookie))
491	panic("died while creating standard software ithreads");
492	if (pin_default_swi &&
493	(intr_event_bind(clk_intr_event, timeout_cpu) != 0)) {
494	printf("%s: timeout clock couldn't be pinned to cpu %d\n",
495	__func__,
496	timeout_cpu);
497	}
498
499	#ifdef SMP
500	CPU_FOREACH(cpu) {
501	if (cpu == timeout_cpu)
502	continue;
503	cc = CC_CPU(cpu);
504	cc->cc_callout = NULL; /* Only BSP handles timeout(9). */
505	callout_cpu_init(cc, cpu);
506	snprintf(name, sizeof(name), "clock (%d)", cpu);
507	ie = NULL;
508	if (swi_add(&ie, name, softclock, cc, SWI_CLOCK,
509	INTR_MPSAFE, &cc->cc_cookie))
510	panic("died while creating standard software ithreads");
511	if (pin_pcpu_swi && (intr_event_bind(ie, cpu) != 0)) {
512	printf("%s: per-cpu clock couldn't be pinned to "
513	"cpu %d\n",
514	__func__,
515	cpu);
516	}
517	}
518	#endif
519	}
520	SYSINIT(start_softclock, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softclock, NULL);
521	#endif /* __rtems__ */
522
523	#define CC_HASH_SHIFT 8
524
525	static inline u_int
526	callout_hash(sbintime_t sbt)
527	{
528
529	return (sbt >> (32 - CC_HASH_SHIFT));
530	}
531
532	static inline u_int
533	callout_get_bucket(sbintime_t sbt)
534	{
535
536	return (callout_hash(sbt) & callwheelmask);
537	}
538
539	void
540	callout_process(sbintime_t now)
541	{
542	#ifndef __rtems__
543	struct callout tmp, tmpn;
544	#else /* __rtems__ */
545	struct callout *tmp;
546	#endif /* __rtems__ */
547	struct callout_cpu *cc;
548	struct callout_list *sc;
549	sbintime_t first, last, max, tmp_max;
550	uint32_t lookahead;
551	u_int firstb, lastb, nowb;
552	#ifdef CALLOUT_PROFILING
553	int depth_dir = 0, mpcalls_dir = 0, lockcalls_dir = 0;
554	#endif
555
556	cc = CC_SELF();
557	mtx_lock_spin_flags(&cc->cc_lock, MTX_QUIET);
558
559	/* Compute the buckets of the last scan and present times. */
560	firstb = callout_hash(cc->cc_lastscan);
561	cc->cc_lastscan = now;
562	nowb = callout_hash(now);
563
564	/* Compute the last bucket and minimum time of the bucket after it. */
565	if (nowb == firstb)
566	lookahead = (SBT_1S / 16);
567	else if (nowb - firstb == 1)
568	lookahead = (SBT_1S / 8);
569	else
570	lookahead = (SBT_1S / 2);
571	first = last = now;
572	first += (lookahead / 2);
573	last += lookahead;
574	last &= (0xffffffffffffffffLLU << (32 - CC_HASH_SHIFT));
575	lastb = callout_hash(last) - 1;
576	max = last;
577
578	/*
579	* Check if we wrapped around the entire wheel from the last scan.
580	* In case, we need to scan entirely the wheel for pending callouts.
581	*/
582	if (lastb - firstb >= callwheelsize) {
583	lastb = firstb + callwheelsize - 1;
584	if (nowb - firstb >= callwheelsize)
585	nowb = lastb;
586	}
587
588	/* Iterate callwheel from firstb to nowb and then up to lastb. */
589	do {
590	sc = &cc->cc_callwheel[firstb & callwheelmask];
591	tmp = LIST_FIRST(sc);
592	while (tmp != NULL) {
593	/* Run the callout if present time within allowed. */
594	if (tmp->c_time <= now) {
595	#ifndef __rtems__
596	/*
597	* Consumer told us the callout may be run
598	* directly from hardware interrupt context.
599	*/
600	if (tmp->c_iflags & CALLOUT_DIRECT) {
601	#endif /* __rtems__ */
602	#ifdef CALLOUT_PROFILING
603	++depth_dir;
604	#endif
605	cc_exec_next(cc) =
606	LIST_NEXT(tmp, c_links.le);
607	cc->cc_bucket = firstb & callwheelmask;
608	LIST_REMOVE(tmp, c_links.le);
609	softclock_call_cc(tmp, cc,
610	#ifdef CALLOUT_PROFILING
611	&mpcalls_dir, &lockcalls_dir, NULL,
612	#endif
613	1);
614	tmp = cc_exec_next(cc);
615	cc_exec_next(cc) = NULL;
616	#ifndef __rtems__
617	} else {
618	tmpn = LIST_NEXT(tmp, c_links.le);
619	LIST_REMOVE(tmp, c_links.le);
620	TAILQ_INSERT_TAIL(&cc->cc_expireq,
621	tmp, c_links.tqe);
622	tmp->c_iflags \|= CALLOUT_PROCESSED;
623	tmp = tmpn;
624	}
625	#endif /* __rtems__ */
626	continue;
627	}
628	/* Skip events from distant future. */
629	if (tmp->c_time >= max)
630	goto next;
631	/*
632	* Event minimal time is bigger than present maximal
633	* time, so it cannot be aggregated.
634	*/
635	if (tmp->c_time > last) {
636	lastb = nowb;
637	goto next;
638	}
639	/* Update first and last time, respecting this event. */
640	if (tmp->c_time < first)
641	first = tmp->c_time;
642	tmp_max = tmp->c_time + tmp->c_precision;
643	if (tmp_max < last)
644	last = tmp_max;
645	next:
646	tmp = LIST_NEXT(tmp, c_links.le);
647	}
648	/* Proceed with the next bucket. */
649	firstb++;
650	/*
651	* Stop if we looked after present time and found
652	* some event we can't execute at now.
653	* Stop if we looked far enough into the future.
654	*/
655	} while (((int)(firstb - lastb)) <= 0);
656	cc->cc_firstevent = last;
657	#ifndef NO_EVENTTIMERS
658	cpu_new_callout(curcpu, last, first);
659	#endif
660	#ifdef CALLOUT_PROFILING
661	avg_depth_dir += (depth_dir * 1000 - avg_depth_dir) >> 8;
662	avg_mpcalls_dir += (mpcalls_dir * 1000 - avg_mpcalls_dir) >> 8;
663	avg_lockcalls_dir += (lockcalls_dir * 1000 - avg_lockcalls_dir) >> 8;
664	#endif
665	mtx_unlock_spin_flags(&cc->cc_lock, MTX_QUIET);
666	#ifndef __rtems__
667	/*
668	* swi_sched acquires the thread lock, so we don't want to call it
669	* with cc_lock held; incorrect locking order.
670	*/
671	if (!TAILQ_EMPTY(&cc->cc_expireq))
672	swi_sched(cc->cc_cookie, 0);
673	#endif /* __rtems__ */
674	}
675
676	static struct callout_cpu *
677	callout_lock(struct callout *c)
678	{
679	struct callout_cpu *cc;
680	#ifndef __rtems__
681	int cpu;
682
683	for (;;) {
684	cpu = c->c_cpu;
685	#ifdef SMP
686	if (cpu == CPUBLOCK) {
687	while (c->c_cpu == CPUBLOCK)
688	cpu_spinwait();
689	continue;
690	}
691	#endif
692	#endif /* __rtems__ */
693	cc = CC_CPU(cpu);
694	CC_LOCK(cc);
695	#ifndef __rtems__
696	if (cpu == c->c_cpu)
697	break;
698	CC_UNLOCK(cc);
699	}
700	#endif /* __rtems__ */
701	return (cc);
702	}
703
704	static void
705	callout_cc_add(struct callout c, struct callout_cpu cc,
706	sbintime_t sbt, sbintime_t precision, void (func)(void ),
707	void *arg, int cpu, int flags)
708	{
709	int bucket;
710
711	CC_LOCK_ASSERT(cc);
712	if (sbt < cc->cc_lastscan)
713	sbt = cc->cc_lastscan;
714	c->c_arg = arg;
715	c->c_iflags \|= CALLOUT_PENDING;
716	#ifndef __rtems__
717	c->c_iflags &= ~CALLOUT_PROCESSED;
718	#endif /* __rtems__ */
719	c->c_flags \|= CALLOUT_ACTIVE;
720	#ifndef __rtems__
721	if (flags & C_DIRECT_EXEC)
722	c->c_iflags \|= CALLOUT_DIRECT;
723	#endif /* __rtems__ */
724	c->c_func = func;
725	c->c_time = sbt;
726	c->c_precision = precision;
727	bucket = callout_get_bucket(c->c_time);
728	CTR3(KTR_CALLOUT, "precision set for %p: %d.%08x",
729	c, (int)(c->c_precision >> 32),
730	(u_int)(c->c_precision & 0xffffffff));
731	LIST_INSERT_HEAD(&cc->cc_callwheel[bucket], c, c_links.le);
732	if (cc->cc_bucket == bucket)
733	cc_exec_next(cc) = c;
734	#ifndef NO_EVENTTIMERS
735	/*
736	* Inform the eventtimers(4) subsystem there's a new callout
737	* that has been inserted, but only if really required.
738	*/
739	if (SBT_MAX - c->c_time < c->c_precision)
740	c->c_precision = SBT_MAX - c->c_time;
741	sbt = c->c_time + c->c_precision;
742	if (sbt < cc->cc_firstevent) {
743	cc->cc_firstevent = sbt;
744	cpu_new_callout(cpu, sbt, c->c_time);
745	}
746	#endif
747	}
748
749	static void
750	callout_cc_del(struct callout c, struct callout_cpu cc)
751	{
752
753	if ((c->c_iflags & CALLOUT_LOCAL_ALLOC) == 0)
754	return;
755	c->c_func = NULL;
756	SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle);
757	}
758
759	static void
760	softclock_call_cc(struct callout c, struct callout_cpu cc,
761	#ifdef CALLOUT_PROFILING
762	int mpcalls, int lockcalls, int *gcalls,
763	#endif
764	int direct)
765	{
766	#ifndef __rtems__
767	struct rm_priotracker tracker;
768	#endif /* __rtems__ */
769	void (c_func)(void );
770	void *c_arg;
771	struct lock_class *class;
772	struct lock_object *c_lock;
773	uintptr_t lock_status;
774	int c_iflags;
775	#ifdef SMP
776	struct callout_cpu *new_cc;
777	void (new_func)(void );
778	void *new_arg;
779	int flags, new_cpu;
780	sbintime_t new_prec, new_time;
781	#endif
782	#if defined(DIAGNOSTIC) \|\| defined(CALLOUT_PROFILING)
783	sbintime_t sbt1, sbt2;
784	struct timespec ts2;
785	static sbintime_t maxdt = 2 * SBT_1MS; /* 2 msec */
786	static timeout_t *lastfunc;
787	#endif
788
789	KASSERT((c->c_iflags & CALLOUT_PENDING) == CALLOUT_PENDING,
790	("softclock_call_cc: pend %p %x", c, c->c_iflags));
791	KASSERT((c->c_flags & CALLOUT_ACTIVE) == CALLOUT_ACTIVE,
792	("softclock_call_cc: act %p %x", c, c->c_flags));
793	class = (c->c_lock != NULL) ? LOCK_CLASS(c->c_lock) : NULL;
794	lock_status = 0;
795	if (c->c_flags & CALLOUT_SHAREDLOCK) {
796	#ifndef __rtems__
797	if (class == &lock_class_rm)
798	lock_status = (uintptr_t)&tracker;
799	else
800	#endif /* __rtems__ */
801	lock_status = 1;
802	}
803	c_lock = c->c_lock;
804	c_func = c->c_func;
805	c_arg = c->c_arg;
806	c_iflags = c->c_iflags;
807	if (c->c_iflags & CALLOUT_LOCAL_ALLOC)
808	c->c_iflags = CALLOUT_LOCAL_ALLOC;
809	else
810	c->c_iflags &= ~CALLOUT_PENDING;
811
812	cc_exec_curr(cc, direct) = c;
813	cc_exec_cancel(cc, direct) = false;
814	cc_exec_drain(cc, direct) = NULL;
815	CC_UNLOCK(cc);
816	if (c_lock != NULL) {
817	class->lc_lock(c_lock, lock_status);
818	/*
819	* The callout may have been cancelled
820	* while we switched locks.
821	*/
822	if (cc_exec_cancel(cc, direct)) {
823	class->lc_unlock(c_lock);
824	goto skip;
825	}
826	/* The callout cannot be stopped now. */
827	cc_exec_cancel(cc, direct) = true;
828	if (c_lock == &Giant.lock_object) {
829	#ifdef CALLOUT_PROFILING
830	(*gcalls)++;
831	#endif
832	CTR3(KTR_CALLOUT, "callout giant %p func %p arg %p",
833	c, c_func, c_arg);
834	} else {
835	#ifdef CALLOUT_PROFILING
836	(*lockcalls)++;
837	#endif
838	CTR3(KTR_CALLOUT, "callout lock %p func %p arg %p",
839	c, c_func, c_arg);
840	}
841	} else {
842	#ifdef CALLOUT_PROFILING
843	(*mpcalls)++;
844	#endif
845	CTR3(KTR_CALLOUT, "callout %p func %p arg %p",
846	c, c_func, c_arg);
847	}
848	KTR_STATE3(KTR_SCHED, "callout", cc->cc_ktr_event_name, "running",
849	"func:%p", c_func, "arg:%p", c_arg, "direct:%d", direct);
850	#if defined(DIAGNOSTIC) \|\| defined(CALLOUT_PROFILING)
851	sbt1 = sbinuptime();
852	#endif
853	#ifndef __rtems__
854	THREAD_NO_SLEEPING();
855	SDT_PROBE1(callout_execute, , , callout__start, c);
856	#endif /* __rtems__ */
857	c_func(c_arg);
858	#ifndef __rtems__
859	SDT_PROBE1(callout_execute, , , callout__end, c);
860	THREAD_SLEEPING_OK();
861	#endif /* __rtems__ */
862	#if defined(DIAGNOSTIC) \|\| defined(CALLOUT_PROFILING)
863	sbt2 = sbinuptime();
864	sbt2 -= sbt1;
865	if (sbt2 > maxdt) {
866	if (lastfunc != c_func \|\| sbt2 > maxdt * 2) {
867	ts2 = sbttots(sbt2);
868	printf(
869	"Expensive timeout(9) function: %p(%p) %jd.%09ld s\n",
870	c_func, c_arg, (intmax_t)ts2.tv_sec, ts2.tv_nsec);
871	}
872	maxdt = sbt2;
873	lastfunc = c_func;
874	}
875	#endif
876	KTR_STATE0(KTR_SCHED, "callout", cc->cc_ktr_event_name, "idle");
877	CTR1(KTR_CALLOUT, "callout %p finished", c);
878	if ((c_iflags & CALLOUT_RETURNUNLOCKED) == 0)
879	class->lc_unlock(c_lock);
880	skip:
881	CC_LOCK(cc);
882	KASSERT(cc_exec_curr(cc, direct) == c, ("mishandled cc_curr"));
883	cc_exec_curr(cc, direct) = NULL;
884	if (cc_exec_drain(cc, direct)) {
885	void (drain)(void );
886
887	drain = cc_exec_drain(cc, direct);
888	cc_exec_drain(cc, direct) = NULL;
889	CC_UNLOCK(cc);
890	drain(c_arg);
891	CC_LOCK(cc);
892	}
893	if (cc_exec_waiting(cc, direct)) {
894	#ifndef __rtems__
895	/*
896	* There is someone waiting for the
897	* callout to complete.
898	* If the callout was scheduled for
899	* migration just cancel it.
900	*/
901	if (cc_cce_migrating(cc, direct)) {
902	cc_cce_cleanup(cc, direct);
903
904	/*
905	* It should be assert here that the callout is not
906	* destroyed but that is not easy.
907	*/
908	c->c_iflags &= ~CALLOUT_DFRMIGRATION;
909	}
910	#endif /* __rtems__ */
911	cc_exec_waiting(cc, direct) = false;
912	CC_UNLOCK(cc);
913	wakeup(&cc_exec_waiting(cc, direct));
914	CC_LOCK(cc);
915	#ifndef __rtems__
916	} else if (cc_cce_migrating(cc, direct)) {
917	KASSERT((c_iflags & CALLOUT_LOCAL_ALLOC) == 0,
918	("Migrating legacy callout %p", c));
919	#ifdef SMP
920	/*
921	* If the callout was scheduled for
922	* migration just perform it now.
923	*/
924	new_cpu = cc_migration_cpu(cc, direct);
925	new_time = cc_migration_time(cc, direct);
926	new_prec = cc_migration_prec(cc, direct);
927	new_func = cc_migration_func(cc, direct);
928	new_arg = cc_migration_arg(cc, direct);
929	cc_cce_cleanup(cc, direct);
930
931	/*
932	* It should be assert here that the callout is not destroyed
933	* but that is not easy.
934	*
935	* As first thing, handle deferred callout stops.
936	*/
937	if (!callout_migrating(c)) {
938	CTR3(KTR_CALLOUT,
939	"deferred cancelled %p func %p arg %p",
940	c, new_func, new_arg);
941	callout_cc_del(c, cc);
942	return;
943	}
944	c->c_iflags &= ~CALLOUT_DFRMIGRATION;
945
946	new_cc = callout_cpu_switch(c, cc, new_cpu);
947	flags = (direct) ? C_DIRECT_EXEC : 0;
948	callout_cc_add(c, new_cc, new_time, new_prec, new_func,
949	new_arg, new_cpu, flags);
950	CC_UNLOCK(new_cc);
951	CC_LOCK(cc);
952	#else
953	panic("migration should not happen");
954	#endif
955	#endif /* __rtems__ */
956	}
957	/*
958	* If the current callout is locally allocated (from
959	* timeout(9)) then put it on the freelist.
960	*
961	* Note: we need to check the cached copy of c_iflags because
962	* if it was not local, then it's not safe to deref the
963	* callout pointer.
964	*/
965	KASSERT((c_iflags & CALLOUT_LOCAL_ALLOC) == 0 \|\|
966	c->c_iflags == CALLOUT_LOCAL_ALLOC,
967	("corrupted callout"));
968	if (c_iflags & CALLOUT_LOCAL_ALLOC)
969	callout_cc_del(c, cc);
970	}
971
972	/*
973	* The callout mechanism is based on the work of Adam M. Costello and
974	* George Varghese, published in a technical report entitled "Redesigning
975	* the BSD Callout and Timer Facilities" and modified slightly for inclusion
976	* in FreeBSD by Justin T. Gibbs. The original work on the data structures
977	* used in this implementation was published by G. Varghese and T. Lauck in
978	* the paper "Hashed and Hierarchical Timing Wheels: Data Structures for
979	* the Efficient Implementation of a Timer Facility" in the Proceedings of
980	* the 11th ACM Annual Symposium on Operating Systems Principles,
981	* Austin, Texas Nov 1987.
982	*/
983
984	#ifndef __rtems__
985	/*
986	* Software (low priority) clock interrupt.
987	* Run periodic events from timeout queue.
988	*/
989	void
990	softclock(void *arg)
991	{
992	struct callout_cpu *cc;
993	struct callout *c;
994	#ifdef CALLOUT_PROFILING
995	int depth = 0, gcalls = 0, lockcalls = 0, mpcalls = 0;
996	#endif
997
998	cc = (struct callout_cpu *)arg;
999	CC_LOCK(cc);
1000	while ((c = TAILQ_FIRST(&cc->cc_expireq)) != NULL) {
1001	TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe);
1002	softclock_call_cc(c, cc,
1003	#ifdef CALLOUT_PROFILING
1004	&mpcalls, &lockcalls, &gcalls,
1005	#endif
1006	0);
1007	#ifdef CALLOUT_PROFILING
1008	++depth;
1009	#endif
1010	}
1011	#ifdef CALLOUT_PROFILING
1012	avg_depth += (depth * 1000 - avg_depth) >> 8;
1013	avg_mpcalls += (mpcalls * 1000 - avg_mpcalls) >> 8;
1014	avg_lockcalls += (lockcalls * 1000 - avg_lockcalls) >> 8;
1015	avg_gcalls += (gcalls * 1000 - avg_gcalls) >> 8;
1016	#endif
1017	CC_UNLOCK(cc);
1018	}
1019	#endif /* __rtems__ */
1020
1021	/*
1022	* timeout --
1023	* Execute a function after a specified length of time.
1024	*
1025	* untimeout --
1026	* Cancel previous timeout function call.
1027	*
1028	* callout_handle_init --
1029	* Initialize a handle so that using it with untimeout is benign.
1030	*
1031	* See AT&T BCI Driver Reference Manual for specification. This
1032	* implementation differs from that one in that although an
1033	* identification value is returned from timeout, the original
1034	* arguments to timeout as well as the identifier are used to
1035	* identify entries for untimeout.
1036	*/
1037	struct callout_handle
1038	timeout(timeout_t ftn, void arg, int to_ticks)
1039	{
1040	struct callout_cpu *cc;
1041	struct callout *new;
1042	struct callout_handle handle;
1043
1044	cc = CC_CPU(timeout_cpu);
1045	CC_LOCK(cc);
1046	/* Fill in the next free callout structure. */
1047	new = SLIST_FIRST(&cc->cc_callfree);
1048	if (new == NULL)
1049	/* XXX Attempt to malloc first */
1050	panic("timeout table full");
1051	SLIST_REMOVE_HEAD(&cc->cc_callfree, c_links.sle);
1052	callout_reset(new, to_ticks, ftn, arg);
1053	handle.callout = new;
1054	CC_UNLOCK(cc);
1055
1056	return (handle);
1057	}
1058
1059	void
1060	untimeout(timeout_t ftn, void arg, struct callout_handle handle)
1061	{
1062	struct callout_cpu *cc;
1063
1064	/*
1065	* Check for a handle that was initialized
1066	* by callout_handle_init, but never used
1067	* for a real timeout.
1068	*/
1069	if (handle.callout == NULL)
1070	return;
1071
1072	cc = callout_lock(handle.callout);
1073	if (handle.callout->c_func == ftn && handle.callout->c_arg == arg)
1074	callout_stop(handle.callout);
1075	CC_UNLOCK(cc);
1076	}
1077
1078	void
1079	callout_handle_init(struct callout_handle *handle)
1080	{
1081	handle->callout = NULL;
1082	}
1083
1084	void
1085	callout_when(sbintime_t sbt, sbintime_t precision, int flags,
1086	sbintime_t res, sbintime_t prec_res)
1087	{
1088	sbintime_t to_sbt, to_pr;
1089
1090	if ((flags & (C_ABSOLUTE \| C_PRECALC)) != 0) {
1091	*res = sbt;
1092	*prec_res = precision;
1093	return;
1094	}
1095	if ((flags & C_HARDCLOCK) != 0 && sbt < tick_sbt)
1096	sbt = tick_sbt;
1097	if ((flags & C_HARDCLOCK) != 0 \|\|
1098	#ifdef NO_EVENTTIMERS
1099	sbt >= sbt_timethreshold) {
1100	to_sbt = getsbinuptime();
1101
1102	/* Add safety belt for the case of hz > 1000. */
1103	to_sbt += tc_tick_sbt - tick_sbt;
1104	#else
1105	sbt >= sbt_tickthreshold) {
1106	/*
1107	* Obtain the time of the last hardclock() call on
1108	* this CPU directly from the kern_clocksource.c.
1109	* This value is per-CPU, but it is equal for all
1110	* active ones.
1111	*/
1112	#ifdef __LP64__
1113	to_sbt = DPCPU_GET(hardclocktime);
1114	#else
1115	spinlock_enter();
1116	to_sbt = DPCPU_GET(hardclocktime);
1117	spinlock_exit();
1118	#endif
1119	#endif
1120	if (cold && to_sbt == 0)
1121	to_sbt = sbinuptime();
1122	if ((flags & C_HARDCLOCK) == 0)
1123	to_sbt += tick_sbt;
1124	} else
1125	to_sbt = sbinuptime();
1126	if (SBT_MAX - to_sbt < sbt)
1127	to_sbt = SBT_MAX;
1128	else
1129	to_sbt += sbt;
1130	*res = to_sbt;
1131	to_pr = ((C_PRELGET(flags) < 0) ? sbt >> tc_precexp :
1132	sbt >> C_PRELGET(flags));
1133	*prec_res = to_pr > precision ? to_pr : precision;
1134	}
1135
1136	/*
1137	* New interface; clients allocate their own callout structures.
1138	*
1139	* callout_reset() - establish or change a timeout
1140	* callout_stop() - disestablish a timeout
1141	* callout_init() - initialize a callout structure so that it can
1142	* safely be passed to callout_reset() and callout_stop()
1143	*
1144	* <sys/callout.h> defines three convenience macros:
1145	*
1146	* callout_active() - returns truth if callout has not been stopped,
1147	* drained, or deactivated since the last time the callout was
1148	* reset.
1149	* callout_pending() - returns truth if callout is still waiting for timeout
1150	* callout_deactivate() - marks the callout as having been serviced
1151	*/
1152	int
1153	callout_reset_sbt_on(struct callout *c, sbintime_t sbt, sbintime_t prec,
1154	void (ftn)(void ), void *arg, int cpu, int flags)
1155	{
1156	sbintime_t to_sbt, precision;
1157	struct callout_cpu *cc;
1158	#ifndef __rtems__
1159	int cancelled, direct;
1160	int ignore_cpu=0;
1161	#else /* __rtems__ */
1162	int cancelled;
1163	#endif /* __rtems__ */
1164
1165	cancelled = 0;
1166	#ifndef __rtems__
1167	if (cpu == -1) {
1168	ignore_cpu = 1;
1169	} else if ((cpu >= MAXCPU) \|\|
1170	((CC_CPU(cpu))->cc_inited == 0)) {
1171	/* Invalid CPU spec */
1172	panic("Invalid CPU in callout %d", cpu);
1173	}
1174	#endif /* __rtems__ */
1175	callout_when(sbt, prec, flags, &to_sbt, &precision);
1176
1177	#ifndef __rtems__
1178	/*
1179	* This flag used to be added by callout_cc_add, but the
1180	* first time you call this we could end up with the
1181	* wrong direct flag if we don't do it before we add.
1182	*/
1183	if (flags & C_DIRECT_EXEC) {
1184	direct = 1;
1185	} else {
1186	direct = 0;
1187	}
1188	KASSERT(!direct \|\| c->c_lock == NULL,
1189	("%s: direct callout %p has lock", __func__, c));
1190	#endif /* __rtems__ */
1191	cc = callout_lock(c);
1192	#ifndef __rtems__
1193	/*
1194	* Don't allow migration of pre-allocated callouts lest they
1195	* become unbalanced or handle the case where the user does
1196	* not care.
1197	*/
1198	if ((c->c_iflags & CALLOUT_LOCAL_ALLOC) \|\|
1199	ignore_cpu) {
1200	cpu = c->c_cpu;
1201	}
1202	#endif /* __rtems__ */
1203
1204	if (cc_exec_curr(cc, direct) == c) {
1205	/*
1206	* We're being asked to reschedule a callout which is
1207	* currently in progress. If there is a lock then we
1208	* can cancel the callout if it has not really started.
1209	*/
1210	if (c->c_lock != NULL && !cc_exec_cancel(cc, direct))
1211	cancelled = cc_exec_cancel(cc, direct) = true;
1212	if (cc_exec_waiting(cc, direct) \|\| cc_exec_drain(cc, direct)) {
1213	/*
1214	* Someone has called callout_drain to kill this
1215	* callout. Don't reschedule.
1216	*/
1217	CTR4(KTR_CALLOUT, "%s %p func %p arg %p",
1218	cancelled ? "cancelled" : "failed to cancel",
1219	c, c->c_func, c->c_arg);
1220	CC_UNLOCK(cc);
1221	return (cancelled);
1222	}
1223	#ifdef SMP
1224	if (callout_migrating(c)) {
1225	/*
1226	* This only occurs when a second callout_reset_sbt_on
1227	* is made after a previous one moved it into
1228	* deferred migration (below). Note we do not change
1229	* the prev_cpu even though the previous target may
1230	* be different.
1231	*/
1232	cc_migration_cpu(cc, direct) = cpu;
1233	cc_migration_time(cc, direct) = to_sbt;
1234	cc_migration_prec(cc, direct) = precision;
1235	cc_migration_func(cc, direct) = ftn;
1236	cc_migration_arg(cc, direct) = arg;
1237	cancelled = 1;
1238	CC_UNLOCK(cc);
1239	return (cancelled);
1240	}
1241	#endif
1242	}
1243	if (c->c_iflags & CALLOUT_PENDING) {
1244	#ifndef __rtems__
1245	if ((c->c_iflags & CALLOUT_PROCESSED) == 0) {
1246	#endif /* __rtems__ */
1247	if (cc_exec_next(cc) == c)
1248	cc_exec_next(cc) = LIST_NEXT(c, c_links.le);
1249	LIST_REMOVE(c, c_links.le);
1250	#ifndef __rtems__
1251	} else {
1252	TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe);
1253	}
1254	#endif /* __rtems__ */
1255	cancelled = 1;
1256	c->c_iflags &= ~ CALLOUT_PENDING;
1257	c->c_flags &= ~ CALLOUT_ACTIVE;
1258	}
1259
1260	#ifdef SMP
1261	/*
1262	* If the callout must migrate try to perform it immediately.
1263	* If the callout is currently running, just defer the migration
1264	* to a more appropriate moment.
1265	*/
1266	if (c->c_cpu != cpu) {
1267	if (cc_exec_curr(cc, direct) == c) {
1268	/*
1269	* Pending will have been removed since we are
1270	* actually executing the callout on another
1271	* CPU. That callout should be waiting on the
1272	* lock the caller holds. If we set both
1273	* active/and/pending after we return and the
1274	* lock on the executing callout proceeds, it
1275	* will then see pending is true and return.
1276	* At the return from the actual callout execution
1277	* the migration will occur in softclock_call_cc
1278	* and this new callout will be placed on the
1279	* new CPU via a call to callout_cpu_switch() which
1280	* will get the lock on the right CPU followed
1281	* by a call callout_cc_add() which will add it there.
1282	* (see above in softclock_call_cc()).
1283	*/
1284	cc_migration_cpu(cc, direct) = cpu;
1285	cc_migration_time(cc, direct) = to_sbt;
1286	cc_migration_prec(cc, direct) = precision;
1287	cc_migration_func(cc, direct) = ftn;
1288	cc_migration_arg(cc, direct) = arg;
1289	c->c_iflags \|= (CALLOUT_DFRMIGRATION \| CALLOUT_PENDING);
1290	c->c_flags \|= CALLOUT_ACTIVE;
1291	CTR6(KTR_CALLOUT,
1292	"migration of %p func %p arg %p in %d.%08x to %u deferred",
1293	c, c->c_func, c->c_arg, (int)(to_sbt >> 32),
1294	(u_int)(to_sbt & 0xffffffff), cpu);
1295	CC_UNLOCK(cc);
1296	return (cancelled);
1297	}
1298	cc = callout_cpu_switch(c, cc, cpu);
1299	}
1300	#endif
1301
1302	callout_cc_add(c, cc, to_sbt, precision, ftn, arg, cpu, flags);
1303	CTR6(KTR_CALLOUT, "%sscheduled %p func %p arg %p in %d.%08x",
1304	cancelled ? "re" : "", c, c->c_func, c->c_arg, (int)(to_sbt >> 32),
1305	(u_int)(to_sbt & 0xffffffff));
1306	CC_UNLOCK(cc);
1307
1308	return (cancelled);
1309	}
1310
1311	/*
1312	* Common idioms that can be optimized in the future.
1313	*/
1314	int
1315	callout_schedule_on(struct callout *c, int to_ticks, int cpu)
1316	{
1317	return callout_reset_on(c, to_ticks, c->c_func, c->c_arg, cpu);
1318	}
1319
1320	int
1321	callout_schedule(struct callout *c, int to_ticks)
1322	{
1323	#ifndef __rtems__
1324	return callout_reset_on(c, to_ticks, c->c_func, c->c_arg, c->c_cpu);
1325	#else /* __rtems__ */
1326	return callout_reset_on(c, to_ticks, c->c_func, c->c_arg, 0);
1327	#endif /* __rtems__ */
1328	}
1329
1330	int
1331	_callout_stop_safe(struct callout c, int flags, void (drain)(void *))
1332	{
1333	#ifndef __rtems__
1334	struct callout_cpu cc, old_cc;
1335	struct lock_class *class;
1336	int direct, sq_locked, use_lock;
1337	int cancelled, not_on_a_list;
1338	#else /* __rtems__ */
1339	struct callout_cpu *cc;
1340	struct lock_class *class;
1341	int use_lock;
1342	int cancelled;
1343	#endif /* __rtems__ */
1344
1345	if ((flags & CS_DRAIN) != 0)
1346	WITNESS_WARN(WARN_GIANTOK \| WARN_SLEEPOK, c->c_lock,
1347	"calling %s", __func__);
1348
1349	/*
1350	* Some old subsystems don't hold Giant while running a callout_stop(),
1351	* so just discard this check for the moment.
1352	*/
1353	if ((flags & CS_DRAIN) == 0 && c->c_lock != NULL) {
1354	if (c->c_lock == &Giant.lock_object)
1355	use_lock = mtx_owned(&Giant);
1356	else {
1357	use_lock = 1;
1358	class = LOCK_CLASS(c->c_lock);
1359	class->lc_assert(c->c_lock, LA_XLOCKED);
1360	}
1361	} else
1362	use_lock = 0;
1363	#ifndef __rtems__
1364	if (c->c_iflags & CALLOUT_DIRECT) {
1365	direct = 1;
1366	} else {
1367	direct = 0;
1368	}
1369
1370	sq_locked = 0;
1371	old_cc = NULL;
1372	again:
1373	#endif /* __rtems__ */
1374	cc = callout_lock(c);
1375
1376	#ifndef __rtems__
1377	if ((c->c_iflags & (CALLOUT_DFRMIGRATION \| CALLOUT_PENDING)) ==
1378	(CALLOUT_DFRMIGRATION \| CALLOUT_PENDING) &&
1379	((c->c_flags & CALLOUT_ACTIVE) == CALLOUT_ACTIVE)) {
1380	/*
1381	* Special case where this slipped in while we
1382	* were migrating as the callout is about to
1383	* execute. The caller probably holds the lock
1384	* the callout wants.
1385	*
1386	* Get rid of the migration first. Then set
1387	* the flag that tells this code not to
1388	* try to remove it from any lists (its not
1389	* on one yet). When the callout wheel runs,
1390	* it will ignore this callout.
1391	*/
1392	c->c_iflags &= ~CALLOUT_PENDING;
1393	c->c_flags &= ~CALLOUT_ACTIVE;
1394	not_on_a_list = 1;
1395	} else {
1396	not_on_a_list = 0;
1397	}
1398
1399	/*
1400	* If the callout was migrating while the callout cpu lock was
1401	* dropped, just drop the sleepqueue lock and check the states
1402	* again.
1403	*/
1404	if (sq_locked != 0 && cc != old_cc) {
1405	#ifdef SMP
1406	CC_UNLOCK(cc);
1407	sleepq_release(&cc_exec_waiting(old_cc, direct));
1408	sq_locked = 0;
1409	old_cc = NULL;
1410	goto again;
1411	#else
1412	panic("migration should not happen");
1413	#endif
1414	}
1415	#endif /* __rtems__ */
1416
1417	/*
1418	* If the callout is running, try to stop it or drain it.
1419	*/
1420	if (cc_exec_curr(cc, direct) == c) {
1421	/*
1422	* Succeed we to stop it or not, we must clear the
1423	* active flag - this is what API users expect. If we're
1424	* draining and the callout is currently executing, first wait
1425	* until it finishes.
1426	*/
1427	if ((flags & CS_DRAIN) == 0)
1428	c->c_flags &= ~CALLOUT_ACTIVE;
1429
1430	if ((flags & CS_DRAIN) != 0) {
1431	/*
1432	* The current callout is running (or just
1433	* about to run) and blocking is allowed, so
1434	* just wait for the current invocation to
1435	* finish.
1436	*/
1437	while (cc_exec_curr(cc, direct) == c) {
1438	#ifndef __rtems__
1439
1440	/*
1441	* Use direct calls to sleepqueue interface
1442	* instead of cv/msleep in order to avoid
1443	* a LOR between cc_lock and sleepqueue
1444	* chain spinlocks. This piece of code
1445	* emulates a msleep_spin() call actually.
1446	*
1447	* If we already have the sleepqueue chain
1448	* locked, then we can safely block. If we
1449	* don't already have it locked, however,
1450	* we have to drop the cc_lock to lock
1451	* it. This opens several races, so we
1452	* restart at the beginning once we have
1453	* both locks. If nothing has changed, then
1454	* we will end up back here with sq_locked
1455	* set.
1456	*/
1457	if (!sq_locked) {
1458	CC_UNLOCK(cc);
1459	sleepq_lock(
1460	&cc_exec_waiting(cc, direct));
1461	sq_locked = 1;
1462	old_cc = cc;
1463	goto again;
1464	}
1465
1466	/*
1467	* Migration could be cancelled here, but
1468	* as long as it is still not sure when it
1469	* will be packed up, just let softclock()
1470	* take care of it.
1471	*/
1472	cc_exec_waiting(cc, direct) = true;
1473	DROP_GIANT();
1474	CC_UNLOCK(cc);
1475	sleepq_add(
1476	&cc_exec_waiting(cc, direct),
1477	&cc->cc_lock.lock_object, "codrain",
1478	SLEEPQ_SLEEP, 0);
1479	sleepq_wait(
1480	&cc_exec_waiting(cc, direct),
1481	0);
1482	sq_locked = 0;
1483	old_cc = NULL;
1484
1485	/* Reacquire locks previously released. */
1486	PICKUP_GIANT();
1487	CC_LOCK(cc);
1488	#else /* __rtems__ */
1489	/*
1490	* On RTEMS the LOR problem above does not
1491	* exist since here we do not use
1492	* sleepq_set_timeout() and instead use the
1493	* RTEMS watchdog.
1494	*/
1495	cc_exec_waiting(cc, direct) = true;
1496	msleep_spin(&cc_exec_waiting(cc, direct),
1497	&cc->cc_lock, "codrain", 0);
1498	#endif /* __rtems__ */
1499	}
1500	c->c_flags &= ~CALLOUT_ACTIVE;
1501	} else if (use_lock &&
1502	!cc_exec_cancel(cc, direct) && (drain == NULL)) {
1503
1504	/*
1505	* The current callout is waiting for its
1506	* lock which we hold. Cancel the callout
1507	* and return. After our caller drops the
1508	* lock, the callout will be skipped in
1509	* softclock(). This only works with a
1510	* callout_stop() not callout_drain() or
1511	* callout_async_drain().
1512	*/
1513	cc_exec_cancel(cc, direct) = true;
1514	CTR3(KTR_CALLOUT, "cancelled %p func %p arg %p",
1515	c, c->c_func, c->c_arg);
1516	#ifndef __rtems__
1517	KASSERT(!cc_cce_migrating(cc, direct),
1518	("callout wrongly scheduled for migration"));
1519	if (callout_migrating(c)) {
1520	c->c_iflags &= ~CALLOUT_DFRMIGRATION;
1521	#ifdef SMP
1522	cc_migration_cpu(cc, direct) = CPUBLOCK;
1523	cc_migration_time(cc, direct) = 0;
1524	cc_migration_prec(cc, direct) = 0;
1525	cc_migration_func(cc, direct) = NULL;
1526	cc_migration_arg(cc, direct) = NULL;
1527	#endif
1528	}
1529	#endif /* __rtems__ */
1530	CC_UNLOCK(cc);
1531	#ifndef __rtems__
1532	KASSERT(!sq_locked, ("sleepqueue chain locked"));
1533	#endif /* __rtems__ */
1534	return (1);
1535	#ifndef __rtems__
1536	} else if (callout_migrating(c)) {
1537	/*
1538	* The callout is currently being serviced
1539	* and the "next" callout is scheduled at
1540	* its completion with a migration. We remove
1541	* the migration flag so it won't get rescheduled,
1542	* but we can't stop the one thats running so
1543	* we return 0.
1544	*/
1545	c->c_iflags &= ~CALLOUT_DFRMIGRATION;
1546	#ifdef SMP
1547	/*
1548	* We can't call cc_cce_cleanup here since
1549	* if we do it will remove .ce_curr and
1550	* its still running. This will prevent a
1551	* reschedule of the callout when the
1552	* execution completes.
1553	*/
1554	cc_migration_cpu(cc, direct) = CPUBLOCK;
1555	cc_migration_time(cc, direct) = 0;
1556	cc_migration_prec(cc, direct) = 0;
1557	cc_migration_func(cc, direct) = NULL;
1558	cc_migration_arg(cc, direct) = NULL;
1559	#endif
1560	CTR3(KTR_CALLOUT, "postponing stop %p func %p arg %p",
1561	c, c->c_func, c->c_arg);
1562	if (drain) {
1563	cc_exec_drain(cc, direct) = drain;
1564	}
1565	CC_UNLOCK(cc);
1566	return ((flags & CS_EXECUTING) != 0);
1567	#endif /* __rtems__ */
1568	}
1569	CTR3(KTR_CALLOUT, "failed to stop %p func %p arg %p",
1570	c, c->c_func, c->c_arg);
1571	if (drain) {
1572	cc_exec_drain(cc, direct) = drain;
1573	}
1574	#ifndef __rtems__
1575	KASSERT(!sq_locked, ("sleepqueue chain still locked"));
1576	#endif /* __rtems__ */
1577	cancelled = ((flags & CS_EXECUTING) != 0);
1578	} else
1579	cancelled = 1;
1580
1581	#ifndef __rtems__
1582	if (sq_locked)
1583	sleepq_release(&cc_exec_waiting(cc, direct));
1584	#endif /* __rtems__ */
1585
1586	if ((c->c_iflags & CALLOUT_PENDING) == 0) {
1587	CTR3(KTR_CALLOUT, "failed to stop %p func %p arg %p",
1588	c, c->c_func, c->c_arg);
1589	/*
1590	* For not scheduled and not executing callout return
1591	* negative value.
1592	*/
1593	if (cc_exec_curr(cc, direct) != c)
1594	cancelled = -1;
1595	CC_UNLOCK(cc);
1596	return (cancelled);
1597	}
1598
1599	c->c_iflags &= ~CALLOUT_PENDING;
1600	c->c_flags &= ~CALLOUT_ACTIVE;
1601
1602	CTR3(KTR_CALLOUT, "cancelled %p func %p arg %p",
1603	c, c->c_func, c->c_arg);
1604	#ifndef __rtems__
1605	if (not_on_a_list == 0) {
1606	if ((c->c_iflags & CALLOUT_PROCESSED) == 0) {
1607	#endif /* __rtems__ */
1608	if (cc_exec_next(cc) == c)
1609	cc_exec_next(cc) = LIST_NEXT(c, c_links.le);
1610	LIST_REMOVE(c, c_links.le);
1611	#ifndef __rtems__
1612	} else {
1613	TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe);
1614	}
1615	}
1616	#endif /* __rtems__ */
1617	callout_cc_del(c, cc);
1618	CC_UNLOCK(cc);
1619	return (cancelled);
1620	}
1621
1622	void
1623	callout_init(struct callout *c, int mpsafe)
1624	{
1625	bzero(c, sizeof *c);
1626	if (mpsafe) {
1627	c->c_lock = NULL;
1628	c->c_iflags = CALLOUT_RETURNUNLOCKED;
1629	} else {
1630	c->c_lock = &Giant.lock_object;
1631	c->c_iflags = 0;
1632	}
1633	#ifndef __rtems__
1634	c->c_cpu = timeout_cpu;
1635	#endif /* __rtems__ */
1636	}
1637
1638	void
1639	_callout_init_lock(struct callout c, struct lock_object lock, int flags)
1640	{
1641	bzero(c, sizeof *c);
1642	c->c_lock = lock;
1643	KASSERT((flags & ~(CALLOUT_RETURNUNLOCKED \| CALLOUT_SHAREDLOCK)) == 0,
1644	("callout_init_lock: bad flags %d", flags));
1645	KASSERT(lock != NULL \|\| (flags & CALLOUT_RETURNUNLOCKED) == 0,
1646	("callout_init_lock: CALLOUT_RETURNUNLOCKED with no lock"));
1647	KASSERT(lock == NULL \|\| !(LOCK_CLASS(lock)->lc_flags &
1648	(LC_SPINLOCK \| LC_SLEEPABLE)), ("%s: invalid lock class",
1649	__func__));
1650	c->c_iflags = flags & (CALLOUT_RETURNUNLOCKED \| CALLOUT_SHAREDLOCK);
1651	#ifndef __rtems__
1652	c->c_cpu = timeout_cpu;
1653	#endif /* __rtems__ */
1654	}
1655
1656	#ifdef APM_FIXUP_CALLTODO
1657	/*
1658	* Adjust the kernel calltodo timeout list. This routine is used after
1659	* an APM resume to recalculate the calltodo timer list values with the
1660	* number of hz's we have been sleeping. The next hardclock() will detect
1661	* that there are fired timers and run softclock() to execute them.
1662	*
1663	* Please note, I have not done an exhaustive analysis of what code this
1664	* might break. I am motivated to have my select()'s and alarm()'s that
1665	* have expired during suspend firing upon resume so that the applications
1666	* which set the timer can do the maintanence the timer was for as close
1667	* as possible to the originally intended time. Testing this code for a
1668	* week showed that resuming from a suspend resulted in 22 to 25 timers
1669	* firing, which seemed independent on whether the suspend was 2 hours or
1670	* 2 days. Your milage may vary. - Ken Key <key@cs.utk.edu>
1671	*/
1672	void
1673	adjust_timeout_calltodo(struct timeval *time_change)
1674	{
1675	struct callout *p;
1676	unsigned long delta_ticks;
1677
1678	/*
1679	* How many ticks were we asleep?
1680	* (stolen from tvtohz()).
1681	*/
1682
1683	/* Don't do anything */
1684	if (time_change->tv_sec < 0)
1685	return;
1686	else if (time_change->tv_sec <= LONG_MAX / 1000000)
1687	delta_ticks = howmany(time_change->tv_sec * 1000000 +
1688	time_change->tv_usec, tick) + 1;
1689	else if (time_change->tv_sec <= LONG_MAX / hz)
1690	delta_ticks = time_change->tv_sec * hz +
1691	howmany(time_change->tv_usec, tick) + 1;
1692	else
1693	delta_ticks = LONG_MAX;
1694
1695	if (delta_ticks > INT_MAX)
1696	delta_ticks = INT_MAX;
1697
1698	/*
1699	* Now rip through the timer calltodo list looking for timers
1700	* to expire.
1701	*/
1702
1703	/* don't collide with softclock() */
1704	CC_LOCK(cc);
1705	for (p = calltodo.c_next; p != NULL; p = p->c_next) {
1706	p->c_time -= delta_ticks;
1707
1708	/* Break if the timer had more time on it than delta_ticks */
1709	if (p->c_time > 0)
1710	break;
1711
1712	/* take back the ticks the timer didn't use (p->c_time <= 0) */
1713	delta_ticks = -p->c_time;
1714	}
1715	CC_UNLOCK(cc);
1716
1717	return;
1718	}
1719	#endif /* APM_FIXUP_CALLTODO */
1720
1721	static int
1722	flssbt(sbintime_t sbt)
1723	{
1724
1725	sbt += (uint64_t)sbt >> 1;
1726	if (sizeof(long) >= sizeof(sbintime_t))
1727	return (flsl(sbt));
1728	if (sbt >= SBT_1S)
1729	return (flsl(((uint64_t)sbt) >> 32) + 32);
1730	return (flsl(sbt));
1731	}
1732
1733	/*
1734	* Dump immediate statistic snapshot of the scheduled callouts.
1735	*/
1736	static int
1737	sysctl_kern_callout_stat(SYSCTL_HANDLER_ARGS)
1738	{
1739	struct callout *tmp;
1740	struct callout_cpu *cc;
1741	struct callout_list *sc;
1742	sbintime_t maxpr, maxt, medpr, medt, now, spr, st, t;
1743	int ct[64], cpr[64], ccpbk[32];
1744	int error, val, i, count, tcum, pcum, maxc, c, medc;
1745	#ifdef SMP
1746	int cpu;
1747	#endif
1748
1749	val = 0;
1750	error = sysctl_handle_int(oidp, &val, 0, req);
1751	if (error != 0 \|\| req->newptr == NULL)
1752	return (error);
1753	count = maxc = 0;
1754	st = spr = maxt = maxpr = 0;
1755	bzero(ccpbk, sizeof(ccpbk));
1756	bzero(ct, sizeof(ct));
1757	bzero(cpr, sizeof(cpr));
1758	now = sbinuptime();
1759	#ifdef SMP
1760	CPU_FOREACH(cpu) {
1761	cc = CC_CPU(cpu);
1762	#else
1763	cc = CC_CPU(timeout_cpu);
1764	#endif
1765	CC_LOCK(cc);
1766	for (i = 0; i < callwheelsize; i++) {
1767	sc = &cc->cc_callwheel[i];
1768	c = 0;
1769	LIST_FOREACH(tmp, sc, c_links.le) {
1770	c++;
1771	t = tmp->c_time - now;
1772	if (t < 0)
1773	t = 0;
1774	st += t / SBT_1US;
1775	spr += tmp->c_precision / SBT_1US;
1776	if (t > maxt)
1777	maxt = t;
1778	if (tmp->c_precision > maxpr)
1779	maxpr = tmp->c_precision;
1780	ct[flssbt(t)]++;
1781	cpr[flssbt(tmp->c_precision)]++;
1782	}
1783	if (c > maxc)
1784	maxc = c;
1785	ccpbk[fls(c + c / 2)]++;
1786	count += c;
1787	}
1788	CC_UNLOCK(cc);
1789	#ifdef SMP
1790	}
1791	#endif
1792
1793	for (i = 0, tcum = 0; i < 64 && tcum < count / 2; i++)
1794	tcum += ct[i];
1795	medt = (i >= 2) ? (((sbintime_t)1) << (i - 2)) : 0;
1796	for (i = 0, pcum = 0; i < 64 && pcum < count / 2; i++)
1797	pcum += cpr[i];
1798	medpr = (i >= 2) ? (((sbintime_t)1) << (i - 2)) : 0;
1799	for (i = 0, c = 0; i < 32 && c < count / 2; i++)
1800	c += ccpbk[i];
1801	medc = (i >= 2) ? (1 << (i - 2)) : 0;
1802
1803	printf("Scheduled callouts statistic snapshot:\n");
1804	printf(" Callouts: %6d Buckets: %6d*%-3d Bucket size: 0.%06ds\n",
1805	count, callwheelsize, mp_ncpus, 1000000 >> CC_HASH_SHIFT);
1806	printf(" C/Bk: med %5d avg %6d.%06jd max %6d\n",
1807	medc,
1808	count / callwheelsize / mp_ncpus,
1809	(uint64_t)count * 1000000 / callwheelsize / mp_ncpus % 1000000,
1810	maxc);
1811	printf(" Time: med %5jd.%06jds avg %6jd.%06jds max %6jd.%06jds\n",
1812	medt / SBT_1S, (medt & 0xffffffff) * 1000000 >> 32,
1813	(st / count) / 1000000, (st / count) % 1000000,
1814	maxt / SBT_1S, (maxt & 0xffffffff) * 1000000 >> 32);
1815	printf(" Prec: med %5jd.%06jds avg %6jd.%06jds max %6jd.%06jds\n",
1816	medpr / SBT_1S, (medpr & 0xffffffff) * 1000000 >> 32,
1817	(spr / count) / 1000000, (spr / count) % 1000000,
1818	maxpr / SBT_1S, (maxpr & 0xffffffff) * 1000000 >> 32);
1819	printf(" Distribution: \tbuckets\t time\t tcum\t"
1820	" prec\t pcum\n");
1821	for (i = 0, tcum = pcum = 0; i < 64; i++) {
1822	if (ct[i] == 0 && cpr[i] == 0)
1823	continue;
1824	t = (i != 0) ? (((sbintime_t)1) << (i - 1)) : 0;
1825	tcum += ct[i];
1826	pcum += cpr[i];
1827	printf(" %10jd.%06jds\t 2**%d\t%7d\t%7d\t%7d\t%7d\n",
1828	t / SBT_1S, (t & 0xffffffff) * 1000000 >> 32,
1829	i - 1 - (32 - CC_HASH_SHIFT),
1830	ct[i], tcum, cpr[i], pcum);
1831	}
1832	return (error);
1833	}
1834	SYSCTL_PROC(_kern, OID_AUTO, callout_stat,
1835	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_MPSAFE,
1836	0, 0, sysctl_kern_callout_stat, "I",
1837	"Dump immediate statistic snapshot of the scheduled callouts");
1838
1839	#ifdef DDB
1840	static void
1841	_show_callout(struct callout *c)
1842	{
1843
1844	db_printf("callout %p\n", c);
1845	#define C_DB_PRINTF(f, e) db_printf(" %s = " f "\n", #e, c->e);
1846	db_printf(" &c_links = %p\n", &(c->c_links));
1847	C_DB_PRINTF("%" PRId64, c_time);
1848	C_DB_PRINTF("%" PRId64, c_precision);
1849	C_DB_PRINTF("%p", c_arg);
1850	C_DB_PRINTF("%p", c_func);
1851	C_DB_PRINTF("%p", c_lock);
1852	C_DB_PRINTF("%#x", c_flags);
1853	C_DB_PRINTF("%#x", c_iflags);
1854	C_DB_PRINTF("%d", c_cpu);
1855	#undef C_DB_PRINTF
1856	}
1857
1858	DB_SHOW_COMMAND(callout, db_show_callout)
1859	{
1860
1861	if (!have_addr) {
1862	db_printf("usage: show callout <struct callout *>\n");
1863	return;
1864	}
1865
1866	_show_callout((struct callout *)addr);
1867	}
1868	#endif /* DDB */

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: rtems-libbsd/freebsd/sys/kern/kern_timeout.c @ bb80d9d

Download in other formats: