source: rtems/cpukit/score/src/kern_tc.c @ 469dc47

5
Last change on this file since 469dc47 was 1ef8e4a8, checked in by Sebastian Huber <sebastian.huber@…>, on 04/27/16 at 20:07:56

score: Avoid Giant lock for set time of day

Update #2555.
Update #2630.

  • Property mode set to 100644
File size: 57.3 KB
Line 
1/*-
2 * ----------------------------------------------------------------------------
3 * "THE BEER-WARE LICENSE" (Revision 42):
4 * <phk@FreeBSD.ORG> wrote this file.  As long as you retain this notice you
5 * can do whatever you want with this stuff. If we meet some day, and you think
6 * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
7 * ----------------------------------------------------------------------------
8 *
9 * Copyright (c) 2011 The FreeBSD Foundation
10 * All rights reserved.
11 *
12 * Portions of this software were developed by Julien Ridoux at the University
13 * of Melbourne under sponsorship from the FreeBSD Foundation.
14 */
15
16#ifdef __rtems__
17#define _KERNEL
18#define binuptime(_bt) _Timecounter_Binuptime(_bt)
19#define nanouptime(_tsp) _Timecounter_Nanouptime(_tsp)
20#define microuptime(_tvp) _Timecounter_Microuptime(_tvp)
21#define bintime(_bt) _Timecounter_Bintime(_bt)
22#define nanotime(_tsp) _Timecounter_Nanotime(_tsp)
23#define microtime(_tvp) _Timecounter_Microtime(_tvp)
24#define getbinuptime(_bt) _Timecounter_Getbinuptime(_bt)
25#define getnanouptime(_tsp) _Timecounter_Getnanouptime(_tsp)
26#define getmicrouptime(_tvp) _Timecounter_Getmicrouptime(_tvp)
27#define getbintime(_bt) _Timecounter_Getbintime(_bt)
28#define getnanotime(_tsp) _Timecounter_Getnanotime(_tsp)
29#define getmicrotime(_tvp) _Timecounter_Getmicrotime(_tvp)
30#define tc_init _Timecounter_Install
31#define timecounter _Timecounter
32#define time_second _Timecounter_Time_second
33#define time_uptime _Timecounter_Time_uptime
34#define boottimebin _Timecounter_Boottimebin
35#include <rtems/score/timecounterimpl.h>
36#include <rtems/score/smp.h>
37#include <rtems/score/todimpl.h>
38#include <rtems/score/watchdogimpl.h>
39#endif /* __rtems__ */
40#include <sys/cdefs.h>
41__FBSDID("$FreeBSD r284178 2015-06-09T11:49:56Z$");
42
43#include "opt_compat.h"
44#include "opt_ntp.h"
45#include "opt_ffclock.h"
46
47#include <sys/param.h>
48#ifndef __rtems__
49#include <sys/kernel.h>
50#include <sys/limits.h>
51#include <sys/lock.h>
52#include <sys/mutex.h>
53#include <sys/sbuf.h>
54#include <sys/sysctl.h>
55#include <sys/syslog.h>
56#include <sys/systm.h>
57#endif /* __rtems__ */
58#include <sys/timeffc.h>
59#include <sys/timepps.h>
60#include <sys/timetc.h>
61#include <sys/timex.h>
62#ifndef __rtems__
63#include <sys/vdso.h>
64#include <machine/atomic.h>
65#endif /* __rtems__ */
66#ifdef __rtems__
67#include <limits.h>
68#include <rtems.h>
69ISR_LOCK_DEFINE(, _Timecounter_Lock, "Timecounter")
70#define _Timecounter_Release(lock_context) \
71  _ISR_lock_Release_and_ISR_enable(&_Timecounter_Lock, lock_context)
72#define hz rtems_clock_get_ticks_per_second()
73#define printf(...)
74#define bcopy(x, y, z) memcpy(y, x, z);
75#define log(...)
76static inline int
77fls(int x)
78{
79        return x ? sizeof(x) * 8 - __builtin_clz(x) : 0;
80}
81/* FIXME: https://devel.rtems.org/ticket/2348 */
82#define ntp_update_second(a, b) do { (void) a; (void) b; } while (0)
83#endif /* __rtems__ */
84
85/*
86 * A large step happens on boot.  This constant detects such steps.
87 * It is relatively small so that ntp_update_second gets called enough
88 * in the typical 'missed a couple of seconds' case, but doesn't loop
89 * forever when the time step is large.
90 */
91#define LARGE_STEP      200
92
93/*
94 * Implement a dummy timecounter which we can use until we get a real one
95 * in the air.  This allows the console and other early stuff to use
96 * time services.
97 */
98
99static uint32_t
100dummy_get_timecount(struct timecounter *tc)
101{
102#ifndef __rtems__
103        static uint32_t now;
104
105        return (++now);
106#else /* __rtems__ */
107        return 0;
108#endif /* __rtems__ */
109}
110
111static struct timecounter dummy_timecounter = {
112        dummy_get_timecount, 0, ~0u, 1000000, "dummy", -1000000
113};
114
115struct timehands {
116        /* These fields must be initialized by the driver. */
117        struct timecounter      *th_counter;
118        int64_t                 th_adjustment;
119        uint64_t                th_scale;
120        uint32_t                th_offset_count;
121        struct bintime          th_offset;
122        struct timeval          th_microtime;
123        struct timespec         th_nanotime;
124        /* Fields not to be copied in tc_windup start with th_generation. */
125#ifndef __rtems__
126        u_int                   th_generation;
127#else /* __rtems__ */
128        Atomic_Ulong            th_generation;
129#endif /* __rtems__ */
130        struct timehands        *th_next;
131};
132
133#if defined(RTEMS_SMP)
134static struct timehands th0;
135static struct timehands th9 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th0};
136static struct timehands th8 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th9};
137static struct timehands th7 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th8};
138static struct timehands th6 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th7};
139static struct timehands th5 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th6};
140static struct timehands th4 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th5};
141static struct timehands th3 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th4};
142static struct timehands th2 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th3};
143static struct timehands th1 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th2};
144#endif
145static struct timehands th0 = {
146        &dummy_timecounter,
147        0,
148        (uint64_t)-1 / 1000000,
149        0,
150        {1, 0},
151#ifndef __rtems__
152        {0, 0},
153        {0, 0},
154#else /* __rtems__ */
155        {TOD_SECONDS_1970_THROUGH_1988, 0},
156        {TOD_SECONDS_1970_THROUGH_1988, 0},
157#endif /* __rtems__ */
158        1,
159#if defined(RTEMS_SMP)
160        &th1
161#else
162        &th0
163#endif
164};
165
166static struct timehands *volatile timehands = &th0;
167struct timecounter *timecounter = &dummy_timecounter;
168static struct timecounter *timecounters = &dummy_timecounter;
169
170#ifndef __rtems__
171int tc_min_ticktock_freq = 1;
172#endif /* __rtems__ */
173
174#ifndef __rtems__
175volatile time_t time_second = 1;
176#else /* __rtems__ */
177volatile time_t time_second = TOD_SECONDS_1970_THROUGH_1988;
178#endif /* __rtems__ */
179volatile time_t time_uptime = 1;
180
181#ifndef __rtems__
182struct bintime boottimebin;
183#else /* __rtems__ */
184struct bintime boottimebin = {
185  .sec = TOD_SECONDS_1970_THROUGH_1988 - 1
186};
187#endif /* __rtems__ */
188#ifndef __rtems__
189struct timeval boottime;
190static int sysctl_kern_boottime(SYSCTL_HANDLER_ARGS);
191SYSCTL_PROC(_kern, KERN_BOOTTIME, boottime, CTLTYPE_STRUCT|CTLFLAG_RD,
192    NULL, 0, sysctl_kern_boottime, "S,timeval", "System boottime");
193
194SYSCTL_NODE(_kern, OID_AUTO, timecounter, CTLFLAG_RW, 0, "");
195static SYSCTL_NODE(_kern_timecounter, OID_AUTO, tc, CTLFLAG_RW, 0, "");
196
197static int timestepwarnings;
198SYSCTL_INT(_kern_timecounter, OID_AUTO, stepwarnings, CTLFLAG_RW,
199    &timestepwarnings, 0, "Log time steps");
200
201struct bintime bt_timethreshold;
202struct bintime bt_tickthreshold;
203sbintime_t sbt_timethreshold;
204sbintime_t sbt_tickthreshold;
205struct bintime tc_tick_bt;
206sbintime_t tc_tick_sbt;
207int tc_precexp;
208int tc_timepercentage = TC_DEFAULTPERC;
209static int sysctl_kern_timecounter_adjprecision(SYSCTL_HANDLER_ARGS);
210SYSCTL_PROC(_kern_timecounter, OID_AUTO, alloweddeviation,
211    CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, 0, 0,
212    sysctl_kern_timecounter_adjprecision, "I",
213    "Allowed time interval deviation in percents");
214#endif /* __rtems__ */
215
216static void tc_windup(void);
217#ifndef __rtems__
218static void cpu_tick_calibrate(int);
219#else /* __rtems__ */
220static void _Timecounter_Windup(ISR_lock_Context *lock_context);
221#endif /* __rtems__ */
222
223void dtrace_getnanotime(struct timespec *tsp);
224
225#ifndef __rtems__
226static int
227sysctl_kern_boottime(SYSCTL_HANDLER_ARGS)
228{
229#ifndef __mips__
230#ifdef SCTL_MASK32
231        int tv[2];
232
233        if (req->flags & SCTL_MASK32) {
234                tv[0] = boottime.tv_sec;
235                tv[1] = boottime.tv_usec;
236                return SYSCTL_OUT(req, tv, sizeof(tv));
237        } else
238#endif
239#endif
240                return SYSCTL_OUT(req, &boottime, sizeof(boottime));
241}
242
243static int
244sysctl_kern_timecounter_get(SYSCTL_HANDLER_ARGS)
245{
246        uint32_t ncount;
247        struct timecounter *tc = arg1;
248
249        ncount = tc->tc_get_timecount(tc);
250        return sysctl_handle_int(oidp, &ncount, 0, req);
251}
252
253static int
254sysctl_kern_timecounter_freq(SYSCTL_HANDLER_ARGS)
255{
256        uint64_t freq;
257        struct timecounter *tc = arg1;
258
259        freq = tc->tc_frequency;
260        return sysctl_handle_64(oidp, &freq, 0, req);
261}
262#endif /* __rtems__ */
263
264/*
265 * Return the difference between the timehands' counter value now and what
266 * was when we copied it to the timehands' offset_count.
267 */
268static __inline uint32_t
269tc_delta(struct timehands *th)
270{
271        struct timecounter *tc;
272
273        tc = th->th_counter;
274        return ((tc->tc_get_timecount(tc) - th->th_offset_count) &
275            tc->tc_counter_mask);
276}
277
278static u_int
279tc_getgen(struct timehands *th)
280{
281
282#ifndef __rtems__
283#ifdef SMP
284        return (atomic_load_acq_int(&th->th_generation));
285#else
286        u_int gen;
287
288        gen = th->th_generation;
289        __compiler_membar();
290        return (gen);
291#endif
292#else /* __rtems__ */
293        return (_Atomic_Load_ulong(&th->th_generation, ATOMIC_ORDER_ACQUIRE));
294#endif /* __rtems__ */
295}
296
297static void
298tc_setgen(struct timehands *th, u_int newgen)
299{
300
301#ifndef __rtems__
302#ifdef SMP
303        atomic_store_rel_int(&th->th_generation, newgen);
304#else
305        __compiler_membar();
306        th->th_generation = newgen;
307#endif
308#else /* __rtems__ */
309        _Atomic_Store_ulong(&th->th_generation, newgen, ATOMIC_ORDER_RELEASE);
310#endif /* __rtems__ */
311}
312
313/*
314 * Functions for reading the time.  We have to loop until we are sure that
315 * the timehands that we operated on was not updated under our feet.  See
316 * the comment in <sys/time.h> for a description of these 12 functions.
317 */
318
319#ifdef FFCLOCK
320void
321fbclock_binuptime(struct bintime *bt)
322{
323        struct timehands *th;
324        unsigned int gen;
325
326        do {
327                th = timehands;
328                gen = tc_getgen(th);
329                *bt = th->th_offset;
330                bintime_addx(bt, th->th_scale * tc_delta(th));
331        } while (gen == 0 || gen != tc_getgen(th));
332}
333
334void
335fbclock_nanouptime(struct timespec *tsp)
336{
337        struct bintime bt;
338
339        fbclock_binuptime(&bt);
340        bintime2timespec(&bt, tsp);
341}
342
343void
344fbclock_microuptime(struct timeval *tvp)
345{
346        struct bintime bt;
347
348        fbclock_binuptime(&bt);
349        bintime2timeval(&bt, tvp);
350}
351
352void
353fbclock_bintime(struct bintime *bt)
354{
355
356        fbclock_binuptime(bt);
357        bintime_add(bt, &boottimebin);
358}
359
360void
361fbclock_nanotime(struct timespec *tsp)
362{
363        struct bintime bt;
364
365        fbclock_bintime(&bt);
366        bintime2timespec(&bt, tsp);
367}
368
369void
370fbclock_microtime(struct timeval *tvp)
371{
372        struct bintime bt;
373
374        fbclock_bintime(&bt);
375        bintime2timeval(&bt, tvp);
376}
377
378void
379fbclock_getbinuptime(struct bintime *bt)
380{
381        struct timehands *th;
382        unsigned int gen;
383
384        do {
385                th = timehands;
386                gen = tc_getgen(th);
387                *bt = th->th_offset;
388        } while (gen == 0 || gen != tc_getgen(th));
389}
390
391void
392fbclock_getnanouptime(struct timespec *tsp)
393{
394        struct timehands *th;
395        unsigned int gen;
396
397        do {
398                th = timehands;
399                gen = tc_getgen(th);
400                bintime2timespec(&th->th_offset, tsp);
401        } while (gen == 0 || gen != tc_getgen(th));
402}
403
404void
405fbclock_getmicrouptime(struct timeval *tvp)
406{
407        struct timehands *th;
408        unsigned int gen;
409
410        do {
411                th = timehands;
412                gen = tc_getgen(th);
413                bintime2timeval(&th->th_offset, tvp);
414        } while (gen == 0 || gen != tc_getgen(th));
415}
416
417void
418fbclock_getbintime(struct bintime *bt)
419{
420        struct timehands *th;
421        unsigned int gen;
422
423        do {
424                th = timehands;
425                gen = tc_getgen(th);
426                *bt = th->th_offset;
427        } while (gen == 0 || gen != tc_getgen(th));
428        bintime_add(bt, &boottimebin);
429}
430
431void
432fbclock_getnanotime(struct timespec *tsp)
433{
434        struct timehands *th;
435        unsigned int gen;
436
437        do {
438                th = timehands;
439                gen = tc_getgen(th);
440                *tsp = th->th_nanotime;
441        } while (gen == 0 || gen != tc_getgen(th));
442}
443
444void
445fbclock_getmicrotime(struct timeval *tvp)
446{
447        struct timehands *th;
448        unsigned int gen;
449
450        do {
451                th = timehands;
452                gen = tc_getgen(th);
453                *tvp = th->th_microtime;
454        } while (gen == 0 || gen != tc_getgen(th));
455}
456#else /* !FFCLOCK */
457void
458binuptime(struct bintime *bt)
459{
460        struct timehands *th;
461        uint32_t gen;
462
463        do {
464                th = timehands;
465                gen = tc_getgen(th);
466                *bt = th->th_offset;
467                bintime_addx(bt, th->th_scale * tc_delta(th));
468        } while (gen == 0 || gen != tc_getgen(th));
469}
470
471void
472nanouptime(struct timespec *tsp)
473{
474        struct bintime bt;
475
476        binuptime(&bt);
477        bintime2timespec(&bt, tsp);
478}
479
480void
481microuptime(struct timeval *tvp)
482{
483        struct bintime bt;
484
485        binuptime(&bt);
486        bintime2timeval(&bt, tvp);
487}
488
489void
490bintime(struct bintime *bt)
491{
492
493        binuptime(bt);
494        bintime_add(bt, &boottimebin);
495}
496
497void
498nanotime(struct timespec *tsp)
499{
500        struct bintime bt;
501
502        bintime(&bt);
503        bintime2timespec(&bt, tsp);
504}
505
506void
507microtime(struct timeval *tvp)
508{
509        struct bintime bt;
510
511        bintime(&bt);
512        bintime2timeval(&bt, tvp);
513}
514
515void
516getbinuptime(struct bintime *bt)
517{
518        struct timehands *th;
519        uint32_t gen;
520
521        do {
522                th = timehands;
523                gen = tc_getgen(th);
524                *bt = th->th_offset;
525        } while (gen == 0 || gen != tc_getgen(th));
526}
527
528void
529getnanouptime(struct timespec *tsp)
530{
531        struct timehands *th;
532        uint32_t gen;
533
534        do {
535                th = timehands;
536                gen = tc_getgen(th);
537                bintime2timespec(&th->th_offset, tsp);
538        } while (gen == 0 || gen != tc_getgen(th));
539}
540
541void
542getmicrouptime(struct timeval *tvp)
543{
544        struct timehands *th;
545        uint32_t gen;
546
547        do {
548                th = timehands;
549                gen = tc_getgen(th);
550                bintime2timeval(&th->th_offset, tvp);
551        } while (gen == 0 || gen != tc_getgen(th));
552}
553
554void
555getbintime(struct bintime *bt)
556{
557        struct timehands *th;
558        uint32_t gen;
559
560        do {
561                th = timehands;
562                gen = tc_getgen(th);
563                *bt = th->th_offset;
564        } while (gen == 0 || gen != tc_getgen(th));
565        bintime_add(bt, &boottimebin);
566}
567
568void
569getnanotime(struct timespec *tsp)
570{
571        struct timehands *th;
572        uint32_t gen;
573
574        do {
575                th = timehands;
576                gen = tc_getgen(th);
577                *tsp = th->th_nanotime;
578        } while (gen == 0 || gen != tc_getgen(th));
579}
580
581void
582getmicrotime(struct timeval *tvp)
583{
584        struct timehands *th;
585        uint32_t gen;
586
587        do {
588                th = timehands;
589                gen = tc_getgen(th);
590                *tvp = th->th_microtime;
591        } while (gen == 0 || gen != tc_getgen(th));
592}
593#endif /* FFCLOCK */
594
595#ifdef FFCLOCK
596/*
597 * Support for feed-forward synchronization algorithms. This is heavily inspired
598 * by the timehands mechanism but kept independent from it. *_windup() functions
599 * have some connection to avoid accessing the timecounter hardware more than
600 * necessary.
601 */
602
603/* Feed-forward clock estimates kept updated by the synchronization daemon. */
604struct ffclock_estimate ffclock_estimate;
605struct bintime ffclock_boottime;        /* Feed-forward boot time estimate. */
606uint32_t ffclock_status;                /* Feed-forward clock status. */
607int8_t ffclock_updated;                 /* New estimates are available. */
608struct mtx ffclock_mtx;                 /* Mutex on ffclock_estimate. */
609
610struct fftimehands {
611        struct ffclock_estimate cest;
612        struct bintime          tick_time;
613        struct bintime          tick_time_lerp;
614        ffcounter               tick_ffcount;
615        uint64_t                period_lerp;
616        volatile uint8_t        gen;
617        struct fftimehands      *next;
618};
619
620#define NUM_ELEMENTS(x) (sizeof(x) / sizeof(*x))
621
622static struct fftimehands ffth[10];
623static struct fftimehands *volatile fftimehands = ffth;
624
625static void
626ffclock_init(void)
627{
628        struct fftimehands *cur;
629        struct fftimehands *last;
630
631        memset(ffth, 0, sizeof(ffth));
632
633        last = ffth + NUM_ELEMENTS(ffth) - 1;
634        for (cur = ffth; cur < last; cur++)
635                cur->next = cur + 1;
636        last->next = ffth;
637
638        ffclock_updated = 0;
639        ffclock_status = FFCLOCK_STA_UNSYNC;
640        mtx_init(&ffclock_mtx, "ffclock lock", NULL, MTX_DEF);
641}
642
643/*
644 * Reset the feed-forward clock estimates. Called from inittodr() to get things
645 * kick started and uses the timecounter nominal frequency as a first period
646 * estimate. Note: this function may be called several time just after boot.
647 * Note: this is the only function that sets the value of boot time for the
648 * monotonic (i.e. uptime) version of the feed-forward clock.
649 */
650void
651ffclock_reset_clock(struct timespec *ts)
652{
653        struct timecounter *tc;
654        struct ffclock_estimate cest;
655
656        tc = timehands->th_counter;
657        memset(&cest, 0, sizeof(struct ffclock_estimate));
658
659        timespec2bintime(ts, &ffclock_boottime);
660        timespec2bintime(ts, &(cest.update_time));
661        ffclock_read_counter(&cest.update_ffcount);
662        cest.leapsec_next = 0;
663        cest.period = ((1ULL << 63) / tc->tc_frequency) << 1;
664        cest.errb_abs = 0;
665        cest.errb_rate = 0;
666        cest.status = FFCLOCK_STA_UNSYNC;
667        cest.leapsec_total = 0;
668        cest.leapsec = 0;
669
670        mtx_lock(&ffclock_mtx);
671        bcopy(&cest, &ffclock_estimate, sizeof(struct ffclock_estimate));
672        ffclock_updated = INT8_MAX;
673        mtx_unlock(&ffclock_mtx);
674
675        printf("ffclock reset: %s (%llu Hz), time = %ld.%09lu\n", tc->tc_name,
676            (unsigned long long)tc->tc_frequency, (long)ts->tv_sec,
677            (unsigned long)ts->tv_nsec);
678}
679
680/*
681 * Sub-routine to convert a time interval measured in RAW counter units to time
682 * in seconds stored in bintime format.
683 * NOTE: bintime_mul requires u_int, but the value of the ffcounter may be
684 * larger than the max value of u_int (on 32 bit architecture). Loop to consume
685 * extra cycles.
686 */
687static void
688ffclock_convert_delta(ffcounter ffdelta, uint64_t period, struct bintime *bt)
689{
690        struct bintime bt2;
691        ffcounter delta, delta_max;
692
693        delta_max = (1ULL << (8 * sizeof(unsigned int))) - 1;
694        bintime_clear(bt);
695        do {
696                if (ffdelta > delta_max)
697                        delta = delta_max;
698                else
699                        delta = ffdelta;
700                bt2.sec = 0;
701                bt2.frac = period;
702                bintime_mul(&bt2, (unsigned int)delta);
703                bintime_add(bt, &bt2);
704                ffdelta -= delta;
705        } while (ffdelta > 0);
706}
707
708/*
709 * Update the fftimehands.
710 * Push the tick ffcount and time(s) forward based on current clock estimate.
711 * The conversion from ffcounter to bintime relies on the difference clock
712 * principle, whose accuracy relies on computing small time intervals. If a new
713 * clock estimate has been passed by the synchronisation daemon, make it
714 * current, and compute the linear interpolation for monotonic time if needed.
715 */
716static void
717ffclock_windup(unsigned int delta)
718{
719        struct ffclock_estimate *cest;
720        struct fftimehands *ffth;
721        struct bintime bt, gap_lerp;
722        ffcounter ffdelta;
723        uint64_t frac;
724        unsigned int polling;
725        uint8_t forward_jump, ogen;
726
727        /*
728         * Pick the next timehand, copy current ffclock estimates and move tick
729         * times and counter forward.
730         */
731        forward_jump = 0;
732        ffth = fftimehands->next;
733        ogen = ffth->gen;
734        ffth->gen = 0;
735        cest = &ffth->cest;
736        bcopy(&fftimehands->cest, cest, sizeof(struct ffclock_estimate));
737        ffdelta = (ffcounter)delta;
738        ffth->period_lerp = fftimehands->period_lerp;
739
740        ffth->tick_time = fftimehands->tick_time;
741        ffclock_convert_delta(ffdelta, cest->period, &bt);
742        bintime_add(&ffth->tick_time, &bt);
743
744        ffth->tick_time_lerp = fftimehands->tick_time_lerp;
745        ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt);
746        bintime_add(&ffth->tick_time_lerp, &bt);
747
748        ffth->tick_ffcount = fftimehands->tick_ffcount + ffdelta;
749
750        /*
751         * Assess the status of the clock, if the last update is too old, it is
752         * likely the synchronisation daemon is dead and the clock is free
753         * running.
754         */
755        if (ffclock_updated == 0) {
756                ffdelta = ffth->tick_ffcount - cest->update_ffcount;
757                ffclock_convert_delta(ffdelta, cest->period, &bt);
758                if (bt.sec > 2 * FFCLOCK_SKM_SCALE)
759                        ffclock_status |= FFCLOCK_STA_UNSYNC;
760        }
761
762        /*
763         * If available, grab updated clock estimates and make them current.
764         * Recompute time at this tick using the updated estimates. The clock
765         * estimates passed the feed-forward synchronisation daemon may result
766         * in time conversion that is not monotonically increasing (just after
767         * the update). time_lerp is a particular linear interpolation over the
768         * synchronisation algo polling period that ensures monotonicity for the
769         * clock ids requesting it.
770         */
771        if (ffclock_updated > 0) {
772                bcopy(&ffclock_estimate, cest, sizeof(struct ffclock_estimate));
773                ffdelta = ffth->tick_ffcount - cest->update_ffcount;
774                ffth->tick_time = cest->update_time;
775                ffclock_convert_delta(ffdelta, cest->period, &bt);
776                bintime_add(&ffth->tick_time, &bt);
777
778                /* ffclock_reset sets ffclock_updated to INT8_MAX */
779                if (ffclock_updated == INT8_MAX)
780                        ffth->tick_time_lerp = ffth->tick_time;
781
782                if (bintime_cmp(&ffth->tick_time, &ffth->tick_time_lerp, >))
783                        forward_jump = 1;
784                else
785                        forward_jump = 0;
786
787                bintime_clear(&gap_lerp);
788                if (forward_jump) {
789                        gap_lerp = ffth->tick_time;
790                        bintime_sub(&gap_lerp, &ffth->tick_time_lerp);
791                } else {
792                        gap_lerp = ffth->tick_time_lerp;
793                        bintime_sub(&gap_lerp, &ffth->tick_time);
794                }
795
796                /*
797                 * The reset from the RTC clock may be far from accurate, and
798                 * reducing the gap between real time and interpolated time
799                 * could take a very long time if the interpolated clock insists
800                 * on strict monotonicity. The clock is reset under very strict
801                 * conditions (kernel time is known to be wrong and
802                 * synchronization daemon has been restarted recently.
803                 * ffclock_boottime absorbs the jump to ensure boot time is
804                 * correct and uptime functions stay consistent.
805                 */
806                if (((ffclock_status & FFCLOCK_STA_UNSYNC) == FFCLOCK_STA_UNSYNC) &&
807                    ((cest->status & FFCLOCK_STA_UNSYNC) == 0) &&
808                    ((cest->status & FFCLOCK_STA_WARMUP) == FFCLOCK_STA_WARMUP)) {
809                        if (forward_jump)
810                                bintime_add(&ffclock_boottime, &gap_lerp);
811                        else
812                                bintime_sub(&ffclock_boottime, &gap_lerp);
813                        ffth->tick_time_lerp = ffth->tick_time;
814                        bintime_clear(&gap_lerp);
815                }
816
817                ffclock_status = cest->status;
818                ffth->period_lerp = cest->period;
819
820                /*
821                 * Compute corrected period used for the linear interpolation of
822                 * time. The rate of linear interpolation is capped to 5000PPM
823                 * (5ms/s).
824                 */
825                if (bintime_isset(&gap_lerp)) {
826                        ffdelta = cest->update_ffcount;
827                        ffdelta -= fftimehands->cest.update_ffcount;
828                        ffclock_convert_delta(ffdelta, cest->period, &bt);
829                        polling = bt.sec;
830                        bt.sec = 0;
831                        bt.frac = 5000000 * (uint64_t)18446744073LL;
832                        bintime_mul(&bt, polling);
833                        if (bintime_cmp(&gap_lerp, &bt, >))
834                                gap_lerp = bt;
835
836                        /* Approximate 1 sec by 1-(1/2^64) to ease arithmetic */
837                        frac = 0;
838                        if (gap_lerp.sec > 0) {
839                                frac -= 1;
840                                frac /= ffdelta / gap_lerp.sec;
841                        }
842                        frac += gap_lerp.frac / ffdelta;
843
844                        if (forward_jump)
845                                ffth->period_lerp += frac;
846                        else
847                                ffth->period_lerp -= frac;
848                }
849
850                ffclock_updated = 0;
851        }
852        if (++ogen == 0)
853                ogen = 1;
854        ffth->gen = ogen;
855        fftimehands = ffth;
856}
857
858/*
859 * Adjust the fftimehands when the timecounter is changed. Stating the obvious,
860 * the old and new hardware counter cannot be read simultaneously. tc_windup()
861 * does read the two counters 'back to back', but a few cycles are effectively
862 * lost, and not accumulated in tick_ffcount. This is a fairly radical
863 * operation for a feed-forward synchronization daemon, and it is its job to not
864 * pushing irrelevant data to the kernel. Because there is no locking here,
865 * simply force to ignore pending or next update to give daemon a chance to
866 * realize the counter has changed.
867 */
868static void
869ffclock_change_tc(struct timehands *th)
870{
871        struct fftimehands *ffth;
872        struct ffclock_estimate *cest;
873        struct timecounter *tc;
874        uint8_t ogen;
875
876        tc = th->th_counter;
877        ffth = fftimehands->next;
878        ogen = ffth->gen;
879        ffth->gen = 0;
880
881        cest = &ffth->cest;
882        bcopy(&(fftimehands->cest), cest, sizeof(struct ffclock_estimate));
883        cest->period = ((1ULL << 63) / tc->tc_frequency ) << 1;
884        cest->errb_abs = 0;
885        cest->errb_rate = 0;
886        cest->status |= FFCLOCK_STA_UNSYNC;
887
888        ffth->tick_ffcount = fftimehands->tick_ffcount;
889        ffth->tick_time_lerp = fftimehands->tick_time_lerp;
890        ffth->tick_time = fftimehands->tick_time;
891        ffth->period_lerp = cest->period;
892
893        /* Do not lock but ignore next update from synchronization daemon. */
894        ffclock_updated--;
895
896        if (++ogen == 0)
897                ogen = 1;
898        ffth->gen = ogen;
899        fftimehands = ffth;
900}
901
902/*
903 * Retrieve feed-forward counter and time of last kernel tick.
904 */
905void
906ffclock_last_tick(ffcounter *ffcount, struct bintime *bt, uint32_t flags)
907{
908        struct fftimehands *ffth;
909        uint8_t gen;
910
911        /*
912         * No locking but check generation has not changed. Also need to make
913         * sure ffdelta is positive, i.e. ffcount > tick_ffcount.
914         */
915        do {
916                ffth = fftimehands;
917                gen = ffth->gen;
918                if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP)
919                        *bt = ffth->tick_time_lerp;
920                else
921                        *bt = ffth->tick_time;
922                *ffcount = ffth->tick_ffcount;
923        } while (gen == 0 || gen != ffth->gen);
924}
925
926/*
927 * Absolute clock conversion. Low level function to convert ffcounter to
928 * bintime. The ffcounter is converted using the current ffclock period estimate
929 * or the "interpolated period" to ensure monotonicity.
930 * NOTE: this conversion may have been deferred, and the clock updated since the
931 * hardware counter has been read.
932 */
933void
934ffclock_convert_abs(ffcounter ffcount, struct bintime *bt, uint32_t flags)
935{
936        struct fftimehands *ffth;
937        struct bintime bt2;
938        ffcounter ffdelta;
939        uint8_t gen;
940
941        /*
942         * No locking but check generation has not changed. Also need to make
943         * sure ffdelta is positive, i.e. ffcount > tick_ffcount.
944         */
945        do {
946                ffth = fftimehands;
947                gen = ffth->gen;
948                if (ffcount > ffth->tick_ffcount)
949                        ffdelta = ffcount - ffth->tick_ffcount;
950                else
951                        ffdelta = ffth->tick_ffcount - ffcount;
952
953                if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP) {
954                        *bt = ffth->tick_time_lerp;
955                        ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt2);
956                } else {
957                        *bt = ffth->tick_time;
958                        ffclock_convert_delta(ffdelta, ffth->cest.period, &bt2);
959                }
960
961                if (ffcount > ffth->tick_ffcount)
962                        bintime_add(bt, &bt2);
963                else
964                        bintime_sub(bt, &bt2);
965        } while (gen == 0 || gen != ffth->gen);
966}
967
968/*
969 * Difference clock conversion.
970 * Low level function to Convert a time interval measured in RAW counter units
971 * into bintime. The difference clock allows measuring small intervals much more
972 * reliably than the absolute clock.
973 */
974void
975ffclock_convert_diff(ffcounter ffdelta, struct bintime *bt)
976{
977        struct fftimehands *ffth;
978        uint8_t gen;
979
980        /* No locking but check generation has not changed. */
981        do {
982                ffth = fftimehands;
983                gen = ffth->gen;
984                ffclock_convert_delta(ffdelta, ffth->cest.period, bt);
985        } while (gen == 0 || gen != ffth->gen);
986}
987
988/*
989 * Access to current ffcounter value.
990 */
991void
992ffclock_read_counter(ffcounter *ffcount)
993{
994        struct timehands *th;
995        struct fftimehands *ffth;
996        unsigned int gen, delta;
997
998        /*
999         * ffclock_windup() called from tc_windup(), safe to rely on
1000         * th->th_generation only, for correct delta and ffcounter.
1001         */
1002        do {
1003                th = timehands;
1004                gen = tc_getgen(th);
1005                ffth = fftimehands;
1006                delta = tc_delta(th);
1007                *ffcount = ffth->tick_ffcount;
1008        } while (gen == 0 || gen != tc_getgen(th));
1009
1010        *ffcount += delta;
1011}
1012
1013void
1014binuptime(struct bintime *bt)
1015{
1016
1017        binuptime_fromclock(bt, sysclock_active);
1018}
1019
1020void
1021nanouptime(struct timespec *tsp)
1022{
1023
1024        nanouptime_fromclock(tsp, sysclock_active);
1025}
1026
1027void
1028microuptime(struct timeval *tvp)
1029{
1030
1031        microuptime_fromclock(tvp, sysclock_active);
1032}
1033
1034void
1035bintime(struct bintime *bt)
1036{
1037
1038        bintime_fromclock(bt, sysclock_active);
1039}
1040
1041void
1042nanotime(struct timespec *tsp)
1043{
1044
1045        nanotime_fromclock(tsp, sysclock_active);
1046}
1047
1048void
1049microtime(struct timeval *tvp)
1050{
1051
1052        microtime_fromclock(tvp, sysclock_active);
1053}
1054
1055void
1056getbinuptime(struct bintime *bt)
1057{
1058
1059        getbinuptime_fromclock(bt, sysclock_active);
1060}
1061
1062void
1063getnanouptime(struct timespec *tsp)
1064{
1065
1066        getnanouptime_fromclock(tsp, sysclock_active);
1067}
1068
1069void
1070getmicrouptime(struct timeval *tvp)
1071{
1072
1073        getmicrouptime_fromclock(tvp, sysclock_active);
1074}
1075
1076void
1077getbintime(struct bintime *bt)
1078{
1079
1080        getbintime_fromclock(bt, sysclock_active);
1081}
1082
1083void
1084getnanotime(struct timespec *tsp)
1085{
1086
1087        getnanotime_fromclock(tsp, sysclock_active);
1088}
1089
1090void
1091getmicrotime(struct timeval *tvp)
1092{
1093
1094        getmicrouptime_fromclock(tvp, sysclock_active);
1095}
1096
1097#endif /* FFCLOCK */
1098
1099#ifndef __rtems__
1100/*
1101 * This is a clone of getnanotime and used for walltimestamps.
1102 * The dtrace_ prefix prevents fbt from creating probes for
1103 * it so walltimestamp can be safely used in all fbt probes.
1104 */
1105void
1106dtrace_getnanotime(struct timespec *tsp)
1107{
1108        struct timehands *th;
1109        uint32_t gen;
1110
1111        do {
1112                th = timehands;
1113                gen = tc_getgen(th);
1114                *tsp = th->th_nanotime;
1115        } while (gen == 0 || gen != tc_getgen(th));
1116}
1117#endif /* __rtems__ */
1118
1119#ifdef FFCLOCK
1120/*
1121 * System clock currently providing time to the system. Modifiable via sysctl
1122 * when the FFCLOCK option is defined.
1123 */
1124int sysclock_active = SYSCLOCK_FBCK;
1125#endif
1126
1127/* Internal NTP status and error estimates. */
1128extern int time_status;
1129extern long time_esterror;
1130
1131#ifndef __rtems__
1132/*
1133 * Take a snapshot of sysclock data which can be used to compare system clocks
1134 * and generate timestamps after the fact.
1135 */
1136void
1137sysclock_getsnapshot(struct sysclock_snap *clock_snap, int fast)
1138{
1139        struct fbclock_info *fbi;
1140        struct timehands *th;
1141        struct bintime bt;
1142        unsigned int delta, gen;
1143#ifdef FFCLOCK
1144        ffcounter ffcount;
1145        struct fftimehands *ffth;
1146        struct ffclock_info *ffi;
1147        struct ffclock_estimate cest;
1148
1149        ffi = &clock_snap->ff_info;
1150#endif
1151
1152        fbi = &clock_snap->fb_info;
1153        delta = 0;
1154
1155        do {
1156                th = timehands;
1157                gen = tc_getgen(th);
1158                fbi->th_scale = th->th_scale;
1159                fbi->tick_time = th->th_offset;
1160#ifdef FFCLOCK
1161                ffth = fftimehands;
1162                ffi->tick_time = ffth->tick_time_lerp;
1163                ffi->tick_time_lerp = ffth->tick_time_lerp;
1164                ffi->period = ffth->cest.period;
1165                ffi->period_lerp = ffth->period_lerp;
1166                clock_snap->ffcount = ffth->tick_ffcount;
1167                cest = ffth->cest;
1168#endif
1169                if (!fast)
1170                        delta = tc_delta(th);
1171        } while (gen == 0 || gen != tc_getgen(th));
1172
1173        clock_snap->delta = delta;
1174#ifdef FFCLOCK
1175        clock_snap->sysclock_active = sysclock_active;
1176#endif
1177
1178        /* Record feedback clock status and error. */
1179        clock_snap->fb_info.status = time_status;
1180        /* XXX: Very crude estimate of feedback clock error. */
1181        bt.sec = time_esterror / 1000000;
1182        bt.frac = ((time_esterror - bt.sec) * 1000000) *
1183            (uint64_t)18446744073709ULL;
1184        clock_snap->fb_info.error = bt;
1185
1186#ifdef FFCLOCK
1187        if (!fast)
1188                clock_snap->ffcount += delta;
1189
1190        /* Record feed-forward clock leap second adjustment. */
1191        ffi->leapsec_adjustment = cest.leapsec_total;
1192        if (clock_snap->ffcount > cest.leapsec_next)
1193                ffi->leapsec_adjustment -= cest.leapsec;
1194
1195        /* Record feed-forward clock status and error. */
1196        clock_snap->ff_info.status = cest.status;
1197        ffcount = clock_snap->ffcount - cest.update_ffcount;
1198        ffclock_convert_delta(ffcount, cest.period, &bt);
1199        /* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s]. */
1200        bintime_mul(&bt, cest.errb_rate * (uint64_t)18446744073709ULL);
1201        /* 18446744073 = int(2^64 / 1e9), since err_abs in [ns]. */
1202        bintime_addx(&bt, cest.errb_abs * (uint64_t)18446744073ULL);
1203        clock_snap->ff_info.error = bt;
1204#endif
1205}
1206
1207/*
1208 * Convert a sysclock snapshot into a struct bintime based on the specified
1209 * clock source and flags.
1210 */
1211int
1212sysclock_snap2bintime(struct sysclock_snap *cs, struct bintime *bt,
1213    int whichclock, uint32_t flags)
1214{
1215#ifdef FFCLOCK
1216        struct bintime bt2;
1217        uint64_t period;
1218#endif
1219
1220        switch (whichclock) {
1221        case SYSCLOCK_FBCK:
1222                *bt = cs->fb_info.tick_time;
1223
1224                /* If snapshot was created with !fast, delta will be >0. */
1225                if (cs->delta > 0)
1226                        bintime_addx(bt, cs->fb_info.th_scale * cs->delta);
1227
1228                if ((flags & FBCLOCK_UPTIME) == 0)
1229                        bintime_add(bt, &boottimebin);
1230                break;
1231#ifdef FFCLOCK
1232        case SYSCLOCK_FFWD:
1233                if (flags & FFCLOCK_LERP) {
1234                        *bt = cs->ff_info.tick_time_lerp;
1235                        period = cs->ff_info.period_lerp;
1236                } else {
1237                        *bt = cs->ff_info.tick_time;
1238                        period = cs->ff_info.period;
1239                }
1240
1241                /* If snapshot was created with !fast, delta will be >0. */
1242                if (cs->delta > 0) {
1243                        ffclock_convert_delta(cs->delta, period, &bt2);
1244                        bintime_add(bt, &bt2);
1245                }
1246
1247                /* Leap second adjustment. */
1248                if (flags & FFCLOCK_LEAPSEC)
1249                        bt->sec -= cs->ff_info.leapsec_adjustment;
1250
1251                /* Boot time adjustment, for uptime/monotonic clocks. */
1252                if (flags & FFCLOCK_UPTIME)
1253                        bintime_sub(bt, &ffclock_boottime);
1254                break;
1255#endif
1256        default:
1257                return (EINVAL);
1258                break;
1259        }
1260
1261        return (0);
1262}
1263#endif /* __rtems__ */
1264
1265/*
1266 * Initialize a new timecounter and possibly use it.
1267 */
1268void
1269tc_init(struct timecounter *tc)
1270{
1271#ifndef __rtems__
1272        uint32_t u;
1273        struct sysctl_oid *tc_root;
1274
1275        u = tc->tc_frequency / tc->tc_counter_mask;
1276        /* XXX: We need some margin here, 10% is a guess */
1277        u *= 11;
1278        u /= 10;
1279        if (u > hz && tc->tc_quality >= 0) {
1280                tc->tc_quality = -2000;
1281                if (bootverbose) {
1282                        printf("Timecounter \"%s\" frequency %ju Hz",
1283                            tc->tc_name, (uintmax_t)tc->tc_frequency);
1284                        printf(" -- Insufficient hz, needs at least %u\n", u);
1285                }
1286        } else if (tc->tc_quality >= 0 || bootverbose) {
1287                printf("Timecounter \"%s\" frequency %ju Hz quality %d\n",
1288                    tc->tc_name, (uintmax_t)tc->tc_frequency,
1289                    tc->tc_quality);
1290        }
1291#endif /* __rtems__ */
1292
1293        tc->tc_next = timecounters;
1294        timecounters = tc;
1295#ifndef __rtems__
1296        /*
1297         * Set up sysctl tree for this counter.
1298         */
1299        tc_root = SYSCTL_ADD_NODE(NULL,
1300            SYSCTL_STATIC_CHILDREN(_kern_timecounter_tc), OID_AUTO, tc->tc_name,
1301            CTLFLAG_RW, 0, "timecounter description");
1302        SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
1303            "mask", CTLFLAG_RD, &(tc->tc_counter_mask), 0,
1304            "mask for implemented bits");
1305        SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
1306            "counter", CTLTYPE_UINT | CTLFLAG_RD, tc, sizeof(*tc),
1307            sysctl_kern_timecounter_get, "IU", "current timecounter value");
1308        SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
1309            "frequency", CTLTYPE_U64 | CTLFLAG_RD, tc, sizeof(*tc),
1310             sysctl_kern_timecounter_freq, "QU", "timecounter frequency");
1311        SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
1312            "quality", CTLFLAG_RD, &(tc->tc_quality), 0,
1313            "goodness of time counter");
1314        /*
1315         * Never automatically use a timecounter with negative quality.
1316         * Even though we run on the dummy counter, switching here may be
1317         * worse since this timecounter may not be monotonous.
1318         */
1319        if (tc->tc_quality < 0)
1320                return;
1321        if (tc->tc_quality < timecounter->tc_quality)
1322                return;
1323        if (tc->tc_quality == timecounter->tc_quality &&
1324            tc->tc_frequency < timecounter->tc_frequency)
1325                return;
1326#endif /* __rtems__ */
1327        (void)tc->tc_get_timecount(tc);
1328        (void)tc->tc_get_timecount(tc);
1329        timecounter = tc;
1330#ifdef __rtems__
1331        tc_windup();
1332#endif /* __rtems__ */
1333}
1334
1335#ifndef __rtems__
1336/* Report the frequency of the current timecounter. */
1337uint64_t
1338tc_getfrequency(void)
1339{
1340
1341        return (timehands->th_counter->tc_frequency);
1342}
1343#endif /* __rtems__ */
1344
1345/*
1346 * Step our concept of UTC.  This is done by modifying our estimate of
1347 * when we booted.
1348 * XXX: not locked.
1349 */
1350void
1351#ifndef __rtems__
1352tc_setclock(struct timespec *ts)
1353#else /* __rtems__ */
1354_Timecounter_Set_clock(const struct bintime *_bt,
1355    ISR_lock_Context *lock_context)
1356#endif /* __rtems__ */
1357{
1358#ifndef __rtems__
1359        struct timespec tbef, taft;
1360#endif /* __rtems__ */
1361        struct bintime bt, bt2;
1362
1363#ifndef __rtems__
1364        cpu_tick_calibrate(1);
1365        nanotime(&tbef);
1366        timespec2bintime(ts, &bt);
1367#else /* __rtems__ */
1368        bt = *_bt;
1369#endif /* __rtems__ */
1370        binuptime(&bt2);
1371        bintime_sub(&bt, &bt2);
1372        bintime_add(&bt2, &boottimebin);
1373        boottimebin = bt;
1374#ifndef __rtems__
1375        bintime2timeval(&bt, &boottime);
1376
1377        /* XXX fiddle all the little crinkly bits around the fiords... */
1378        tc_windup();
1379        nanotime(&taft);
1380        if (timestepwarnings) {
1381                log(LOG_INFO,
1382                    "Time stepped from %jd.%09ld to %jd.%09ld (%jd.%09ld)\n",
1383                    (intmax_t)tbef.tv_sec, tbef.tv_nsec,
1384                    (intmax_t)taft.tv_sec, taft.tv_nsec,
1385                    (intmax_t)ts->tv_sec, ts->tv_nsec);
1386        }
1387        cpu_tick_calibrate(1);
1388#else /* __rtems__ */
1389        _Timecounter_Windup(lock_context);
1390#endif /* __rtems__ */
1391}
1392
1393/*
1394 * Initialize the next struct timehands in the ring and make
1395 * it the active timehands.  Along the way we might switch to a different
1396 * timecounter and/or do seconds processing in NTP.  Slightly magic.
1397 */
1398static void
1399tc_windup(void)
1400#ifdef __rtems__
1401{
1402        ISR_lock_Context lock_context;
1403
1404        _Timecounter_Acquire(&lock_context);
1405        _Timecounter_Windup(&lock_context);
1406}
1407
1408static void
1409_Timecounter_Windup(ISR_lock_Context *lock_context)
1410#endif /* __rtems__ */
1411{
1412        struct bintime bt;
1413        struct timehands *th, *tho;
1414        uint64_t scale;
1415        uint32_t delta, ncount, ogen;
1416        int i;
1417        time_t t;
1418
1419        /*
1420         * Make the next timehands a copy of the current one, but do not
1421         * overwrite the generation or next pointer.  While we update
1422         * the contents, the generation must be zero.
1423         */
1424        tho = timehands;
1425#if defined(RTEMS_SMP)
1426        th = tho->th_next;
1427#else
1428        th = tho;
1429#endif
1430        ogen = th->th_generation;
1431        tc_setgen(th, 0);
1432#if defined(RTEMS_SMP)
1433        bcopy(tho, th, offsetof(struct timehands, th_generation));
1434#endif
1435
1436        /*
1437         * Capture a timecounter delta on the current timecounter and if
1438         * changing timecounters, a counter value from the new timecounter.
1439         * Update the offset fields accordingly.
1440         */
1441        delta = tc_delta(th);
1442        if (th->th_counter != timecounter)
1443                ncount = timecounter->tc_get_timecount(timecounter);
1444        else
1445                ncount = 0;
1446#ifdef FFCLOCK
1447        ffclock_windup(delta);
1448#endif
1449        th->th_offset_count += delta;
1450        th->th_offset_count &= th->th_counter->tc_counter_mask;
1451        while (delta > th->th_counter->tc_frequency) {
1452                /* Eat complete unadjusted seconds. */
1453                delta -= th->th_counter->tc_frequency;
1454                th->th_offset.sec++;
1455        }
1456        if ((delta > th->th_counter->tc_frequency / 2) &&
1457            (th->th_scale * delta < ((uint64_t)1 << 63))) {
1458                /* The product th_scale * delta just barely overflows. */
1459                th->th_offset.sec++;
1460        }
1461        bintime_addx(&th->th_offset, th->th_scale * delta);
1462
1463        /*
1464         * Hardware latching timecounters may not generate interrupts on
1465         * PPS events, so instead we poll them.  There is a finite risk that
1466         * the hardware might capture a count which is later than the one we
1467         * got above, and therefore possibly in the next NTP second which might
1468         * have a different rate than the current NTP second.  It doesn't
1469         * matter in practice.
1470         */
1471        if (tho->th_counter->tc_poll_pps)
1472                tho->th_counter->tc_poll_pps(tho->th_counter);
1473
1474        /*
1475         * Deal with NTP second processing.  The for loop normally
1476         * iterates at most once, but in extreme situations it might
1477         * keep NTP sane if timeouts are not run for several seconds.
1478         * At boot, the time step can be large when the TOD hardware
1479         * has been read, so on really large steps, we call
1480         * ntp_update_second only twice.  We need to call it twice in
1481         * case we missed a leap second.
1482         */
1483        bt = th->th_offset;
1484        bintime_add(&bt, &boottimebin);
1485        i = bt.sec - tho->th_microtime.tv_sec;
1486        if (i > LARGE_STEP)
1487                i = 2;
1488        for (; i > 0; i--) {
1489                t = bt.sec;
1490                ntp_update_second(&th->th_adjustment, &bt.sec);
1491                if (bt.sec != t)
1492                        boottimebin.sec += bt.sec - t;
1493        }
1494        /* Update the UTC timestamps used by the get*() functions. */
1495        /* XXX shouldn't do this here.  Should force non-`get' versions. */
1496        bintime2timeval(&bt, &th->th_microtime);
1497        bintime2timespec(&bt, &th->th_nanotime);
1498
1499        /* Now is a good time to change timecounters. */
1500        if (th->th_counter != timecounter) {
1501#ifndef __rtems__
1502#ifndef __arm__
1503                if ((timecounter->tc_flags & TC_FLAGS_C2STOP) != 0)
1504                        cpu_disable_c2_sleep++;
1505                if ((th->th_counter->tc_flags & TC_FLAGS_C2STOP) != 0)
1506                        cpu_disable_c2_sleep--;
1507#endif
1508#endif /* __rtems__ */
1509                th->th_counter = timecounter;
1510                th->th_offset_count = ncount;
1511#ifndef __rtems__
1512                tc_min_ticktock_freq = max(1, timecounter->tc_frequency /
1513                    (((uint64_t)timecounter->tc_counter_mask + 1) / 3));
1514#endif /* __rtems__ */
1515#ifdef FFCLOCK
1516                ffclock_change_tc(th);
1517#endif
1518        }
1519
1520        /*-
1521         * Recalculate the scaling factor.  We want the number of 1/2^64
1522         * fractions of a second per period of the hardware counter, taking
1523         * into account the th_adjustment factor which the NTP PLL/adjtime(2)
1524         * processing provides us with.
1525         *
1526         * The th_adjustment is nanoseconds per second with 32 bit binary
1527         * fraction and we want 64 bit binary fraction of second:
1528         *
1529         *       x = a * 2^32 / 10^9 = a * 4.294967296
1530         *
1531         * The range of th_adjustment is +/- 5000PPM so inside a 64bit int
1532         * we can only multiply by about 850 without overflowing, that
1533         * leaves no suitably precise fractions for multiply before divide.
1534         *
1535         * Divide before multiply with a fraction of 2199/512 results in a
1536         * systematic undercompensation of 10PPM of th_adjustment.  On a
1537         * 5000PPM adjustment this is a 0.05PPM error.  This is acceptable.
1538         *
1539         * We happily sacrifice the lowest of the 64 bits of our result
1540         * to the goddess of code clarity.
1541         *
1542         */
1543        scale = (uint64_t)1 << 63;
1544        scale += (th->th_adjustment / 1024) * 2199;
1545        scale /= th->th_counter->tc_frequency;
1546        th->th_scale = scale * 2;
1547
1548        /*
1549         * Now that the struct timehands is again consistent, set the new
1550         * generation number, making sure to not make it zero.
1551         */
1552        if (++ogen == 0)
1553                ogen = 1;
1554        tc_setgen(th, ogen);
1555
1556        /* Go live with the new struct timehands. */
1557#ifdef FFCLOCK
1558        switch (sysclock_active) {
1559        case SYSCLOCK_FBCK:
1560#endif
1561                time_second = th->th_microtime.tv_sec;
1562                time_uptime = th->th_offset.sec;
1563#ifdef FFCLOCK
1564                break;
1565        case SYSCLOCK_FFWD:
1566                time_second = fftimehands->tick_time_lerp.sec;
1567                time_uptime = fftimehands->tick_time_lerp.sec - ffclock_boottime.sec;
1568                break;
1569        }
1570#endif
1571
1572#if defined(RTEMS_SMP)
1573        timehands = th;
1574#endif
1575#ifndef __rtems__
1576        timekeep_push_vdso();
1577#endif /* __rtems__ */
1578#ifdef __rtems__
1579        _Timecounter_Release(lock_context);
1580#endif /* __rtems__ */
1581}
1582
1583#ifndef __rtems__
1584/* Report or change the active timecounter hardware. */
1585static int
1586sysctl_kern_timecounter_hardware(SYSCTL_HANDLER_ARGS)
1587{
1588        char newname[32];
1589        struct timecounter *newtc, *tc;
1590        int error;
1591
1592        tc = timecounter;
1593        strlcpy(newname, tc->tc_name, sizeof(newname));
1594
1595        error = sysctl_handle_string(oidp, &newname[0], sizeof(newname), req);
1596        if (error != 0 || req->newptr == NULL ||
1597            strcmp(newname, tc->tc_name) == 0)
1598                return (error);
1599        for (newtc = timecounters; newtc != NULL; newtc = newtc->tc_next) {
1600                if (strcmp(newname, newtc->tc_name) != 0)
1601                        continue;
1602
1603                /* Warm up new timecounter. */
1604                (void)newtc->tc_get_timecount(newtc);
1605                (void)newtc->tc_get_timecount(newtc);
1606
1607                timecounter = newtc;
1608
1609                /*
1610                 * The vdso timehands update is deferred until the next
1611                 * 'tc_windup()'.
1612                 *
1613                 * This is prudent given that 'timekeep_push_vdso()' does not
1614                 * use any locking and that it can be called in hard interrupt
1615                 * context via 'tc_windup()'.
1616                 */
1617                return (0);
1618        }
1619        return (EINVAL);
1620}
1621
1622SYSCTL_PROC(_kern_timecounter, OID_AUTO, hardware, CTLTYPE_STRING | CTLFLAG_RW,
1623    0, 0, sysctl_kern_timecounter_hardware, "A",
1624    "Timecounter hardware selected");
1625
1626
1627/* Report or change the active timecounter hardware. */
1628static int
1629sysctl_kern_timecounter_choice(SYSCTL_HANDLER_ARGS)
1630{
1631        struct sbuf sb;
1632        struct timecounter *tc;
1633        int error;
1634
1635        sbuf_new_for_sysctl(&sb, NULL, 0, req);
1636        for (tc = timecounters; tc != NULL; tc = tc->tc_next) {
1637                if (tc != timecounters)
1638                        sbuf_putc(&sb, ' ');
1639                sbuf_printf(&sb, "%s(%d)", tc->tc_name, tc->tc_quality);
1640        }
1641        error = sbuf_finish(&sb);
1642        sbuf_delete(&sb);
1643        return (error);
1644}
1645
1646SYSCTL_PROC(_kern_timecounter, OID_AUTO, choice, CTLTYPE_STRING | CTLFLAG_RD,
1647    0, 0, sysctl_kern_timecounter_choice, "A", "Timecounter hardware detected");
1648#endif /* __rtems__ */
1649
1650#ifndef __rtems__
1651/*
1652 * RFC 2783 PPS-API implementation.
1653 */
1654
1655/*
1656 *  Return true if the driver is aware of the abi version extensions in the
1657 *  pps_state structure, and it supports at least the given abi version number.
1658 */
1659static inline int
1660abi_aware(struct pps_state *pps, int vers)
1661{
1662
1663        return ((pps->kcmode & KCMODE_ABIFLAG) && pps->driver_abi >= vers);
1664}
1665
1666static int
1667pps_fetch(struct pps_fetch_args *fapi, struct pps_state *pps)
1668{
1669        int err, timo;
1670        pps_seq_t aseq, cseq;
1671        struct timeval tv;
1672
1673        if (fapi->tsformat && fapi->tsformat != PPS_TSFMT_TSPEC)
1674                return (EINVAL);
1675
1676        /*
1677         * If no timeout is requested, immediately return whatever values were
1678         * most recently captured.  If timeout seconds is -1, that's a request
1679         * to block without a timeout.  WITNESS won't let us sleep forever
1680         * without a lock (we really don't need a lock), so just repeatedly
1681         * sleep a long time.
1682         */
1683        if (fapi->timeout.tv_sec || fapi->timeout.tv_nsec) {
1684                if (fapi->timeout.tv_sec == -1)
1685                        timo = 0x7fffffff;
1686                else {
1687                        tv.tv_sec = fapi->timeout.tv_sec;
1688                        tv.tv_usec = fapi->timeout.tv_nsec / 1000;
1689                        timo = tvtohz(&tv);
1690                }
1691                aseq = pps->ppsinfo.assert_sequence;
1692                cseq = pps->ppsinfo.clear_sequence;
1693                while (aseq == pps->ppsinfo.assert_sequence &&
1694                    cseq == pps->ppsinfo.clear_sequence) {
1695                        if (abi_aware(pps, 1) && pps->driver_mtx != NULL) {
1696                                if (pps->flags & PPSFLAG_MTX_SPIN) {
1697                                        err = msleep_spin(pps, pps->driver_mtx,
1698                                            "ppsfch", timo);
1699                                } else {
1700                                        err = msleep(pps, pps->driver_mtx, PCATCH,
1701                                            "ppsfch", timo);
1702                                }
1703                        } else {
1704                                err = tsleep(pps, PCATCH, "ppsfch", timo);
1705                        }
1706                        if (err == EWOULDBLOCK && fapi->timeout.tv_sec == -1) {
1707                                continue;
1708                        } else if (err != 0) {
1709                                return (err);
1710                        }
1711                }
1712        }
1713
1714        pps->ppsinfo.current_mode = pps->ppsparam.mode;
1715        fapi->pps_info_buf = pps->ppsinfo;
1716
1717        return (0);
1718}
1719
1720int
1721pps_ioctl(u_long cmd, caddr_t data, struct pps_state *pps)
1722{
1723        pps_params_t *app;
1724        struct pps_fetch_args *fapi;
1725#ifdef FFCLOCK
1726        struct pps_fetch_ffc_args *fapi_ffc;
1727#endif
1728#ifdef PPS_SYNC
1729        struct pps_kcbind_args *kapi;
1730#endif
1731
1732        KASSERT(pps != NULL, ("NULL pps pointer in pps_ioctl"));
1733        switch (cmd) {
1734        case PPS_IOC_CREATE:
1735                return (0);
1736        case PPS_IOC_DESTROY:
1737                return (0);
1738        case PPS_IOC_SETPARAMS:
1739                app = (pps_params_t *)data;
1740                if (app->mode & ~pps->ppscap)
1741                        return (EINVAL);
1742#ifdef FFCLOCK
1743                /* Ensure only a single clock is selected for ffc timestamp. */
1744                if ((app->mode & PPS_TSCLK_MASK) == PPS_TSCLK_MASK)
1745                        return (EINVAL);
1746#endif
1747                pps->ppsparam = *app;
1748                return (0);
1749        case PPS_IOC_GETPARAMS:
1750                app = (pps_params_t *)data;
1751                *app = pps->ppsparam;
1752                app->api_version = PPS_API_VERS_1;
1753                return (0);
1754        case PPS_IOC_GETCAP:
1755                *(int*)data = pps->ppscap;
1756                return (0);
1757        case PPS_IOC_FETCH:
1758                fapi = (struct pps_fetch_args *)data;
1759                return (pps_fetch(fapi, pps));
1760#ifdef FFCLOCK
1761        case PPS_IOC_FETCH_FFCOUNTER:
1762                fapi_ffc = (struct pps_fetch_ffc_args *)data;
1763                if (fapi_ffc->tsformat && fapi_ffc->tsformat !=
1764                    PPS_TSFMT_TSPEC)
1765                        return (EINVAL);
1766                if (fapi_ffc->timeout.tv_sec || fapi_ffc->timeout.tv_nsec)
1767                        return (EOPNOTSUPP);
1768                pps->ppsinfo_ffc.current_mode = pps->ppsparam.mode;
1769                fapi_ffc->pps_info_buf_ffc = pps->ppsinfo_ffc;
1770                /* Overwrite timestamps if feedback clock selected. */
1771                switch (pps->ppsparam.mode & PPS_TSCLK_MASK) {
1772                case PPS_TSCLK_FBCK:
1773                        fapi_ffc->pps_info_buf_ffc.assert_timestamp =
1774                            pps->ppsinfo.assert_timestamp;
1775                        fapi_ffc->pps_info_buf_ffc.clear_timestamp =
1776                            pps->ppsinfo.clear_timestamp;
1777                        break;
1778                case PPS_TSCLK_FFWD:
1779                        break;
1780                default:
1781                        break;
1782                }
1783                return (0);
1784#endif /* FFCLOCK */
1785        case PPS_IOC_KCBIND:
1786#ifdef PPS_SYNC
1787                kapi = (struct pps_kcbind_args *)data;
1788                /* XXX Only root should be able to do this */
1789                if (kapi->tsformat && kapi->tsformat != PPS_TSFMT_TSPEC)
1790                        return (EINVAL);
1791                if (kapi->kernel_consumer != PPS_KC_HARDPPS)
1792                        return (EINVAL);
1793                if (kapi->edge & ~pps->ppscap)
1794                        return (EINVAL);
1795                pps->kcmode = (kapi->edge & KCMODE_EDGEMASK) |
1796                    (pps->kcmode & KCMODE_ABIFLAG);
1797                return (0);
1798#else
1799                return (EOPNOTSUPP);
1800#endif
1801        default:
1802                return (ENOIOCTL);
1803        }
1804}
1805
1806void
1807pps_init(struct pps_state *pps)
1808{
1809        pps->ppscap |= PPS_TSFMT_TSPEC | PPS_CANWAIT;
1810        if (pps->ppscap & PPS_CAPTUREASSERT)
1811                pps->ppscap |= PPS_OFFSETASSERT;
1812        if (pps->ppscap & PPS_CAPTURECLEAR)
1813                pps->ppscap |= PPS_OFFSETCLEAR;
1814#ifdef FFCLOCK
1815        pps->ppscap |= PPS_TSCLK_MASK;
1816#endif
1817        pps->kcmode &= ~KCMODE_ABIFLAG;
1818}
1819
1820void
1821pps_init_abi(struct pps_state *pps)
1822{
1823
1824        pps_init(pps);
1825        if (pps->driver_abi > 0) {
1826                pps->kcmode |= KCMODE_ABIFLAG;
1827                pps->kernel_abi = PPS_ABI_VERSION;
1828        }
1829}
1830
1831void
1832pps_capture(struct pps_state *pps)
1833{
1834        struct timehands *th;
1835
1836        KASSERT(pps != NULL, ("NULL pps pointer in pps_capture"));
1837        th = timehands;
1838        pps->capgen = tc_getgen(th);
1839        pps->capth = th;
1840#ifdef FFCLOCK
1841        pps->capffth = fftimehands;
1842#endif
1843        pps->capcount = th->th_counter->tc_get_timecount(th->th_counter);
1844        if (pps->capgen != tc_getgen(th))
1845                pps->capgen = 0;
1846}
1847
1848void
1849pps_event(struct pps_state *pps, int event)
1850{
1851        struct bintime bt;
1852        struct timespec ts, *tsp, *osp;
1853        uint32_t tcount, *pcount;
1854        int foff, fhard;
1855        pps_seq_t *pseq;
1856#ifdef FFCLOCK
1857        struct timespec *tsp_ffc;
1858        pps_seq_t *pseq_ffc;
1859        ffcounter *ffcount;
1860#endif
1861
1862        KASSERT(pps != NULL, ("NULL pps pointer in pps_event"));
1863        /* If the timecounter was wound up underneath us, bail out. */
1864        if (pps->capgen == 0 || pps->capgen != tc_getgen(pps->capth))
1865                return;
1866
1867        /* Things would be easier with arrays. */
1868        if (event == PPS_CAPTUREASSERT) {
1869                tsp = &pps->ppsinfo.assert_timestamp;
1870                osp = &pps->ppsparam.assert_offset;
1871                foff = pps->ppsparam.mode & PPS_OFFSETASSERT;
1872                fhard = pps->kcmode & PPS_CAPTUREASSERT;
1873                pcount = &pps->ppscount[0];
1874                pseq = &pps->ppsinfo.assert_sequence;
1875#ifdef FFCLOCK
1876                ffcount = &pps->ppsinfo_ffc.assert_ffcount;
1877                tsp_ffc = &pps->ppsinfo_ffc.assert_timestamp;
1878                pseq_ffc = &pps->ppsinfo_ffc.assert_sequence;
1879#endif
1880        } else {
1881                tsp = &pps->ppsinfo.clear_timestamp;
1882                osp = &pps->ppsparam.clear_offset;
1883                foff = pps->ppsparam.mode & PPS_OFFSETCLEAR;
1884                fhard = pps->kcmode & PPS_CAPTURECLEAR;
1885                pcount = &pps->ppscount[1];
1886                pseq = &pps->ppsinfo.clear_sequence;
1887#ifdef FFCLOCK
1888                ffcount = &pps->ppsinfo_ffc.clear_ffcount;
1889                tsp_ffc = &pps->ppsinfo_ffc.clear_timestamp;
1890                pseq_ffc = &pps->ppsinfo_ffc.clear_sequence;
1891#endif
1892        }
1893
1894        /*
1895         * If the timecounter changed, we cannot compare the count values, so
1896         * we have to drop the rest of the PPS-stuff until the next event.
1897         */
1898        if (pps->ppstc != pps->capth->th_counter) {
1899                pps->ppstc = pps->capth->th_counter;
1900                *pcount = pps->capcount;
1901                pps->ppscount[2] = pps->capcount;
1902                return;
1903        }
1904
1905        /* Convert the count to a timespec. */
1906        tcount = pps->capcount - pps->capth->th_offset_count;
1907        tcount &= pps->capth->th_counter->tc_counter_mask;
1908        bt = pps->capth->th_offset;
1909        bintime_addx(&bt, pps->capth->th_scale * tcount);
1910        bintime_add(&bt, &boottimebin);
1911        bintime2timespec(&bt, &ts);
1912
1913        /* If the timecounter was wound up underneath us, bail out. */
1914        if (pps->capgen != tc_getgen(pps->capth))
1915                return;
1916
1917        *pcount = pps->capcount;
1918        (*pseq)++;
1919        *tsp = ts;
1920
1921        if (foff) {
1922                timespecadd(tsp, osp);
1923                if (tsp->tv_nsec < 0) {
1924                        tsp->tv_nsec += 1000000000;
1925                        tsp->tv_sec -= 1;
1926                }
1927        }
1928
1929#ifdef FFCLOCK
1930        *ffcount = pps->capffth->tick_ffcount + tcount;
1931        bt = pps->capffth->tick_time;
1932        ffclock_convert_delta(tcount, pps->capffth->cest.period, &bt);
1933        bintime_add(&bt, &pps->capffth->tick_time);
1934        bintime2timespec(&bt, &ts);
1935        (*pseq_ffc)++;
1936        *tsp_ffc = ts;
1937#endif
1938
1939#ifdef PPS_SYNC
1940        if (fhard) {
1941                uint64_t scale;
1942
1943                /*
1944                 * Feed the NTP PLL/FLL.
1945                 * The FLL wants to know how many (hardware) nanoseconds
1946                 * elapsed since the previous event.
1947                 */
1948                tcount = pps->capcount - pps->ppscount[2];
1949                pps->ppscount[2] = pps->capcount;
1950                tcount &= pps->capth->th_counter->tc_counter_mask;
1951                scale = (uint64_t)1 << 63;
1952                scale /= pps->capth->th_counter->tc_frequency;
1953                scale *= 2;
1954                bt.sec = 0;
1955                bt.frac = 0;
1956                bintime_addx(&bt, scale * tcount);
1957                bintime2timespec(&bt, &ts);
1958                hardpps(tsp, ts.tv_nsec + 1000000000 * ts.tv_sec);
1959        }
1960#endif
1961
1962        /* Wakeup anyone sleeping in pps_fetch().  */
1963        wakeup(pps);
1964}
1965#else /* __rtems__ */
1966/* FIXME: https://devel.rtems.org/ticket/2349 */
1967#endif /* __rtems__ */
1968
1969/*
1970 * Timecounters need to be updated every so often to prevent the hardware
1971 * counter from overflowing.  Updating also recalculates the cached values
1972 * used by the get*() family of functions, so their precision depends on
1973 * the update frequency.
1974 */
1975
1976#ifndef __rtems__
1977static int tc_tick;
1978SYSCTL_INT(_kern_timecounter, OID_AUTO, tick, CTLFLAG_RD, &tc_tick, 0,
1979    "Approximate number of hardclock ticks in a millisecond");
1980#endif /* __rtems__ */
1981
1982#ifndef __rtems__
1983void
1984tc_ticktock(int cnt)
1985{
1986        static int count;
1987
1988        count += cnt;
1989        if (count < tc_tick)
1990                return;
1991        count = 0;
1992        tc_windup();
1993}
1994#else /* __rtems__ */
1995void
1996_Timecounter_Tick(void)
1997{
1998        Per_CPU_Control *cpu_self = _Per_CPU_Get();
1999
2000        if (_Per_CPU_Is_boot_processor(cpu_self)) {
2001                tc_windup();
2002        }
2003
2004        _Watchdog_Tick(cpu_self);
2005}
2006
2007void
2008_Timecounter_Tick_simple(uint32_t delta, uint32_t offset,
2009    ISR_lock_Context *lock_context)
2010{
2011        struct bintime bt;
2012        struct timehands *th;
2013        uint32_t ogen;
2014
2015        th = timehands;
2016        ogen = th->th_generation;
2017        th->th_offset_count = offset;
2018        bintime_addx(&th->th_offset, th->th_scale * delta);
2019
2020        bt = th->th_offset;
2021        bintime_add(&bt, &boottimebin);
2022
2023        /* Update the UTC timestamps used by the get*() functions. */
2024        /* XXX shouldn't do this here.  Should force non-`get' versions. */
2025        bintime2timeval(&bt, &th->th_microtime);
2026        bintime2timespec(&bt, &th->th_nanotime);
2027
2028        /*
2029         * Now that the struct timehands is again consistent, set the new
2030         * generation number, making sure to not make it zero.
2031         */
2032        if (++ogen == 0)
2033                ogen = 1;
2034        th->th_generation = ogen;
2035
2036        /* Go live with the new struct timehands. */
2037        time_second = th->th_microtime.tv_sec;
2038        time_uptime = th->th_offset.sec;
2039
2040        _Timecounter_Release(lock_context);
2041
2042        _Watchdog_Tick(_Per_CPU_Get_snapshot());
2043}
2044#endif /* __rtems__ */
2045
2046#ifndef __rtems__
2047static void __inline
2048tc_adjprecision(void)
2049{
2050        int t;
2051
2052        if (tc_timepercentage > 0) {
2053                t = (99 + tc_timepercentage) / tc_timepercentage;
2054                tc_precexp = fls(t + (t >> 1)) - 1;
2055                FREQ2BT(hz / tc_tick, &bt_timethreshold);
2056                FREQ2BT(hz, &bt_tickthreshold);
2057                bintime_shift(&bt_timethreshold, tc_precexp);
2058                bintime_shift(&bt_tickthreshold, tc_precexp);
2059        } else {
2060                tc_precexp = 31;
2061                bt_timethreshold.sec = INT_MAX;
2062                bt_timethreshold.frac = ~(uint64_t)0;
2063                bt_tickthreshold = bt_timethreshold;
2064        }
2065        sbt_timethreshold = bttosbt(bt_timethreshold);
2066        sbt_tickthreshold = bttosbt(bt_tickthreshold);
2067}
2068#endif /* __rtems__ */
2069
2070#ifndef __rtems__
2071static int
2072sysctl_kern_timecounter_adjprecision(SYSCTL_HANDLER_ARGS)
2073{
2074        int error, val;
2075
2076        val = tc_timepercentage;
2077        error = sysctl_handle_int(oidp, &val, 0, req);
2078        if (error != 0 || req->newptr == NULL)
2079                return (error);
2080        tc_timepercentage = val;
2081        if (cold)
2082                goto done;
2083        tc_adjprecision();
2084done:
2085        return (0);
2086}
2087
2088static void
2089inittimecounter(void *dummy)
2090{
2091        u_int p;
2092        int tick_rate;
2093
2094        /*
2095         * Set the initial timeout to
2096         * max(1, <approx. number of hardclock ticks in a millisecond>).
2097         * People should probably not use the sysctl to set the timeout
2098         * to smaller than its inital value, since that value is the
2099         * smallest reasonable one.  If they want better timestamps they
2100         * should use the non-"get"* functions.
2101         */
2102        if (hz > 1000)
2103                tc_tick = (hz + 500) / 1000;
2104        else
2105                tc_tick = 1;
2106        tc_adjprecision();
2107        FREQ2BT(hz, &tick_bt);
2108        tick_sbt = bttosbt(tick_bt);
2109        tick_rate = hz / tc_tick;
2110        FREQ2BT(tick_rate, &tc_tick_bt);
2111        tc_tick_sbt = bttosbt(tc_tick_bt);
2112        p = (tc_tick * 1000000) / hz;
2113        printf("Timecounters tick every %d.%03u msec\n", p / 1000, p % 1000);
2114
2115#ifdef FFCLOCK
2116        ffclock_init();
2117#endif
2118        /* warm up new timecounter (again) and get rolling. */
2119        (void)timecounter->tc_get_timecount(timecounter);
2120        (void)timecounter->tc_get_timecount(timecounter);
2121        tc_windup();
2122}
2123
2124SYSINIT(timecounter, SI_SUB_CLOCKS, SI_ORDER_SECOND, inittimecounter, NULL);
2125
2126/* Cpu tick handling -------------------------------------------------*/
2127
2128static int cpu_tick_variable;
2129static uint64_t cpu_tick_frequency;
2130
2131static uint64_t
2132tc_cpu_ticks(void)
2133{
2134        static uint64_t base;
2135        static unsigned last;
2136        unsigned u;
2137        struct timecounter *tc;
2138
2139        tc = timehands->th_counter;
2140        u = tc->tc_get_timecount(tc) & tc->tc_counter_mask;
2141        if (u < last)
2142                base += (uint64_t)tc->tc_counter_mask + 1;
2143        last = u;
2144        return (u + base);
2145}
2146
2147void
2148cpu_tick_calibration(void)
2149{
2150        static time_t last_calib;
2151
2152        if (time_uptime != last_calib && !(time_uptime & 0xf)) {
2153                cpu_tick_calibrate(0);
2154                last_calib = time_uptime;
2155        }
2156}
2157
2158/*
2159 * This function gets called every 16 seconds on only one designated
2160 * CPU in the system from hardclock() via cpu_tick_calibration()().
2161 *
2162 * Whenever the real time clock is stepped we get called with reset=1
2163 * to make sure we handle suspend/resume and similar events correctly.
2164 */
2165
2166static void
2167cpu_tick_calibrate(int reset)
2168{
2169        static uint64_t c_last;
2170        uint64_t c_this, c_delta;
2171        static struct bintime  t_last;
2172        struct bintime t_this, t_delta;
2173        uint32_t divi;
2174
2175        if (reset) {
2176                /* The clock was stepped, abort & reset */
2177                t_last.sec = 0;
2178                return;
2179        }
2180
2181        /* we don't calibrate fixed rate cputicks */
2182        if (!cpu_tick_variable)
2183                return;
2184
2185        getbinuptime(&t_this);
2186        c_this = cpu_ticks();
2187        if (t_last.sec != 0) {
2188                c_delta = c_this - c_last;
2189                t_delta = t_this;
2190                bintime_sub(&t_delta, &t_last);
2191                /*
2192                 * Headroom:
2193                 *      2^(64-20) / 16[s] =
2194                 *      2^(44) / 16[s] =
2195                 *      17.592.186.044.416 / 16 =
2196                 *      1.099.511.627.776 [Hz]
2197                 */
2198                divi = t_delta.sec << 20;
2199                divi |= t_delta.frac >> (64 - 20);
2200                c_delta <<= 20;
2201                c_delta /= divi;
2202                if (c_delta > cpu_tick_frequency) {
2203                        if (0 && bootverbose)
2204                                printf("cpu_tick increased to %ju Hz\n",
2205                                    c_delta);
2206                        cpu_tick_frequency = c_delta;
2207                }
2208        }
2209        c_last = c_this;
2210        t_last = t_this;
2211}
2212
2213void
2214set_cputicker(cpu_tick_f *func, uint64_t freq, unsigned var)
2215{
2216
2217        if (func == NULL) {
2218                cpu_ticks = tc_cpu_ticks;
2219        } else {
2220                cpu_tick_frequency = freq;
2221                cpu_tick_variable = var;
2222                cpu_ticks = func;
2223        }
2224}
2225
2226uint64_t
2227cpu_tickrate(void)
2228{
2229
2230        if (cpu_ticks == tc_cpu_ticks)
2231                return (tc_getfrequency());
2232        return (cpu_tick_frequency);
2233}
2234
2235/*
2236 * We need to be slightly careful converting cputicks to microseconds.
2237 * There is plenty of margin in 64 bits of microseconds (half a million
2238 * years) and in 64 bits at 4 GHz (146 years), but if we do a multiply
2239 * before divide conversion (to retain precision) we find that the
2240 * margin shrinks to 1.5 hours (one millionth of 146y).
2241 * With a three prong approach we never lose significant bits, no
2242 * matter what the cputick rate and length of timeinterval is.
2243 */
2244
2245uint64_t
2246cputick2usec(uint64_t tick)
2247{
2248
2249        if (tick > 18446744073709551LL)         /* floor(2^64 / 1000) */
2250                return (tick / (cpu_tickrate() / 1000000LL));
2251        else if (tick > 18446744073709LL)       /* floor(2^64 / 1000000) */
2252                return ((tick * 1000LL) / (cpu_tickrate() / 1000LL));
2253        else
2254                return ((tick * 1000000LL) / cpu_tickrate());
2255}
2256
2257cpu_tick_f      *cpu_ticks = tc_cpu_ticks;
2258#endif /* __rtems__ */
2259
2260#ifndef __rtems__
2261static int vdso_th_enable = 1;
2262static int
2263sysctl_fast_gettime(SYSCTL_HANDLER_ARGS)
2264{
2265        int old_vdso_th_enable, error;
2266
2267        old_vdso_th_enable = vdso_th_enable;
2268        error = sysctl_handle_int(oidp, &old_vdso_th_enable, 0, req);
2269        if (error != 0)
2270                return (error);
2271        vdso_th_enable = old_vdso_th_enable;
2272        return (0);
2273}
2274SYSCTL_PROC(_kern_timecounter, OID_AUTO, fast_gettime,
2275    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
2276    NULL, 0, sysctl_fast_gettime, "I", "Enable fast time of day");
2277
2278uint32_t
2279tc_fill_vdso_timehands(struct vdso_timehands *vdso_th)
2280{
2281        struct timehands *th;
2282        uint32_t enabled;
2283
2284        th = timehands;
2285        vdso_th->th_algo = VDSO_TH_ALGO_1;
2286        vdso_th->th_scale = th->th_scale;
2287        vdso_th->th_offset_count = th->th_offset_count;
2288        vdso_th->th_counter_mask = th->th_counter->tc_counter_mask;
2289        vdso_th->th_offset = th->th_offset;
2290        vdso_th->th_boottime = boottimebin;
2291        enabled = cpu_fill_vdso_timehands(vdso_th, th->th_counter);
2292        if (!vdso_th_enable)
2293                enabled = 0;
2294        return (enabled);
2295}
2296#endif /* __rtems__ */
2297
2298#ifdef COMPAT_FREEBSD32
2299uint32_t
2300tc_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32)
2301{
2302        struct timehands *th;
2303        uint32_t enabled;
2304
2305        th = timehands;
2306        vdso_th32->th_algo = VDSO_TH_ALGO_1;
2307        *(uint64_t *)&vdso_th32->th_scale[0] = th->th_scale;
2308        vdso_th32->th_offset_count = th->th_offset_count;
2309        vdso_th32->th_counter_mask = th->th_counter->tc_counter_mask;
2310        vdso_th32->th_offset.sec = th->th_offset.sec;
2311        *(uint64_t *)&vdso_th32->th_offset.frac[0] = th->th_offset.frac;
2312        vdso_th32->th_boottime.sec = boottimebin.sec;
2313        *(uint64_t *)&vdso_th32->th_boottime.frac[0] = boottimebin.frac;
2314        enabled = cpu_fill_vdso_timehands32(vdso_th32, th->th_counter);
2315        if (!vdso_th_enable)
2316                enabled = 0;
2317        return (enabled);
2318}
2319#endif
Note: See TracBrowser for help on using the repository browser.