Changeset 0163063 in rtems


Ignore:
Timestamp:
Jul 8, 2015, 6:42:08 PM (4 years ago)
Author:
Konstantin Belousov <kib@…>
Branches:
master
Children:
ec349b58
Parents:
4d0ade9
git-author:
Konstantin Belousov <kib@…> (07/08/15 18:42:08)
git-committer:
Sebastian Huber <sebastian.huber@…> (10/12/17 05:04:10)
Message:

timecounter: Merge FreeBSD change r285286

Reimplement the ordering requirements for the timehands updates, and for timehands consumers, by using fences.

Ensure that the timehands->th_generation reset to zero is visible
before the data update is visible [*]. tc_setget() allowed data update
writes to become visible before generation (but not on TSO
architectures).

Remove tc_setgen(), tc_getgen() helpers, use atomics inline [].

Noted by: alc [*]
Requested by: bde []
Reviewed by: alc, bde
Sponsored by: The FreeBSD Foundation
MFC after: 3 weeks

Update #3175.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • cpukit/score/src/kern_tc.c

    r4d0ade9 r0163063  
    3535#define boottimebin _Timecounter_Boottimebin
    3636#include <rtems/score/timecounterimpl.h>
     37#include <rtems/score/atomic.h>
    3738#include <rtems/score/smp.h>
    3839#include <rtems/score/todimpl.h>
     
    8384/* FIXME: https://devel.rtems.org/ticket/2348 */
    8485#define ntp_update_second(a, b) do { (void) a; (void) b; } while (0)
     86
     87static inline void
     88atomic_thread_fence_acq(void)
     89{
     90
     91        _Atomic_Fence(ATOMIC_ORDER_ACQUIRE);
     92}
     93
     94static inline void
     95atomic_thread_fence_rel(void)
     96{
     97
     98        _Atomic_Fence(ATOMIC_ORDER_RELEASE);
     99}
     100
     101static inline u_int
     102atomic_load_acq_int(Atomic_Uint *i)
     103{
     104
     105        return (_Atomic_Load_uint(i, ATOMIC_ORDER_ACQUIRE));
     106}
     107
     108static inline void
     109atomic_store_rel_int(Atomic_Uint *i, u_int val)
     110{
     111
     112        _Atomic_Store_uint(i, val, ATOMIC_ORDER_RELEASE);
     113}
    85114#endif /* __rtems__ */
    86115
     
    128157        u_int                   th_generation;
    129158#else /* __rtems__ */
    130         Atomic_Ulong            th_generation;
     159        Atomic_Uint             th_generation;
    131160#endif /* __rtems__ */
    132161        struct timehands        *th_next;
     
    278307}
    279308
    280 static inline u_int
    281 tc_getgen(struct timehands *th)
    282 {
    283 
    284 #ifndef __rtems__
    285 #ifdef SMP
    286         return (atomic_load_acq_int(&th->th_generation));
    287 #else
    288         u_int gen;
    289 
    290         gen = th->th_generation;
    291         __compiler_membar();
    292         return (gen);
    293 #endif
    294 #else /* __rtems__ */
    295         return (_Atomic_Load_ulong(&th->th_generation, ATOMIC_ORDER_ACQUIRE));
    296 #endif /* __rtems__ */
    297 }
    298 
    299 static inline void
    300 tc_setgen(struct timehands *th, u_int newgen)
    301 {
    302 
    303 #ifndef __rtems__
    304 #ifdef SMP
    305         atomic_store_rel_int(&th->th_generation, newgen);
    306 #else
    307         __compiler_membar();
    308         th->th_generation = newgen;
    309 #endif
    310 #else /* __rtems__ */
    311         _Atomic_Store_ulong(&th->th_generation, newgen, ATOMIC_ORDER_RELEASE);
    312 #endif /* __rtems__ */
    313 }
    314 
    315309/*
    316310 * Functions for reading the time.  We have to loop until we are sure that
     
    328322        do {
    329323                th = timehands;
    330                 gen = tc_getgen(th);
     324                gen = atomic_load_acq_int(&th->th_generation);
    331325                *bt = th->th_offset;
    332326                bintime_addx(bt, th->th_scale * tc_delta(th));
    333         } while (gen == 0 || gen != tc_getgen(th));
     327                atomic_thread_fence_acq();
     328        } while (gen == 0 || gen != th->th_generation);
    334329}
    335330
     
    386381        do {
    387382                th = timehands;
    388                 gen = tc_getgen(th);
     383                gen = atomic_load_acq_int(&th->th_generation);
    389384                *bt = th->th_offset;
    390         } while (gen == 0 || gen != tc_getgen(th));
     385                atomic_thread_fence_acq();
     386        } while (gen == 0 || gen != th->th_generation);
    391387}
    392388
     
    399395        do {
    400396                th = timehands;
    401                 gen = tc_getgen(th);
     397                gen = atomic_load_acq_int(&th->th_generation);
    402398                bintime2timespec(&th->th_offset, tsp);
    403         } while (gen == 0 || gen != tc_getgen(th));
     399                atomic_thread_fence_acq();
     400        } while (gen == 0 || gen != th->th_generation);
    404401}
    405402
     
    412409        do {
    413410                th = timehands;
    414                 gen = tc_getgen(th);
     411                gen = atomic_load_acq_int(&th->th_generation);
    415412                bintime2timeval(&th->th_offset, tvp);
    416         } while (gen == 0 || gen != tc_getgen(th));
     413                atomic_thread_fence_acq();
     414        } while (gen == 0 || gen != th->th_generation);
    417415}
    418416
     
    425423        do {
    426424                th = timehands;
    427                 gen = tc_getgen(th);
     425                gen = atomic_load_acq_int(&th->th_generation);
    428426                *bt = th->th_offset;
    429         } while (gen == 0 || gen != tc_getgen(th));
     427                atomic_thread_fence_acq();
     428        } while (gen == 0 || gen != th->th_generation);
    430429        bintime_add(bt, &boottimebin);
    431430}
     
    439438        do {
    440439                th = timehands;
    441                 gen = tc_getgen(th);
     440                gen = atomic_load_acq_int(&th->th_generation);
    442441                *tsp = th->th_nanotime;
    443         } while (gen == 0 || gen != tc_getgen(th));
     442                atomic_thread_fence_acq();
     443        } while (gen == 0 || gen != th->th_generation);
    444444}
    445445
     
    452452        do {
    453453                th = timehands;
    454                 gen = tc_getgen(th);
     454                gen = atomic_load_acq_int(&th->th_generation);
    455455                *tvp = th->th_microtime;
    456         } while (gen == 0 || gen != tc_getgen(th));
     456                atomic_thread_fence_acq();
     457        } while (gen == 0 || gen != th->th_generation);
    457458}
    458459#else /* !FFCLOCK */
     
    465466        do {
    466467                th = timehands;
    467                 gen = tc_getgen(th);
     468                gen = atomic_load_acq_int(&th->th_generation);
    468469                *bt = th->th_offset;
    469470                bintime_addx(bt, th->th_scale * tc_delta(th));
    470         } while (gen == 0 || gen != tc_getgen(th));
     471                atomic_thread_fence_acq();
     472        } while (gen == 0 || gen != th->th_generation);
    471473}
    472474#ifdef __rtems__
     
    480482        do {
    481483                th = timehands;
    482                 gen = tc_getgen(th);
     484                gen = atomic_load_acq_int(&th->th_generation);
    483485                sbt = bttosbt(th->th_offset);
    484486                sbt += (th->th_scale * tc_delta(th)) >> 32;
    485         } while (gen == 0 || gen != tc_getgen(th));
     487                atomic_thread_fence_acq();
     488        } while (gen == 0 || gen != th->th_generation);
    486489
    487490        return (sbt);
     
    541544        do {
    542545                th = timehands;
    543                 gen = tc_getgen(th);
     546                gen = atomic_load_acq_int(&th->th_generation);
    544547                *bt = th->th_offset;
    545         } while (gen == 0 || gen != tc_getgen(th));
     548                atomic_thread_fence_acq();
     549        } while (gen == 0 || gen != th->th_generation);
    546550}
    547551
     
    554558        do {
    555559                th = timehands;
    556                 gen = tc_getgen(th);
     560                gen = atomic_load_acq_int(&th->th_generation);
    557561                bintime2timespec(&th->th_offset, tsp);
    558         } while (gen == 0 || gen != tc_getgen(th));
     562                atomic_thread_fence_acq();
     563        } while (gen == 0 || gen != th->th_generation);
    559564}
    560565
     
    567572        do {
    568573                th = timehands;
    569                 gen = tc_getgen(th);
     574                gen = atomic_load_acq_int(&th->th_generation);
    570575                bintime2timeval(&th->th_offset, tvp);
    571         } while (gen == 0 || gen != tc_getgen(th));
     576                atomic_thread_fence_acq();
     577        } while (gen == 0 || gen != th->th_generation);
    572578}
    573579
     
    580586        do {
    581587                th = timehands;
    582                 gen = tc_getgen(th);
     588                gen = atomic_load_acq_int(&th->th_generation);
    583589                *bt = th->th_offset;
    584         } while (gen == 0 || gen != tc_getgen(th));
     590                atomic_thread_fence_acq();
     591        } while (gen == 0 || gen != th->th_generation);
    585592        bintime_add(bt, &boottimebin);
    586593}
     
    594601        do {
    595602                th = timehands;
    596                 gen = tc_getgen(th);
     603                gen = atomic_load_acq_int(&th->th_generation);
    597604                *tsp = th->th_nanotime;
    598         } while (gen == 0 || gen != tc_getgen(th));
     605                atomic_thread_fence_acq();
     606        } while (gen == 0 || gen != th->th_generation);
    599607}
    600608
     
    607615        do {
    608616                th = timehands;
    609                 gen = tc_getgen(th);
     617                gen = atomic_load_acq_int(&th->th_generation);
    610618                *tvp = th->th_microtime;
    611         } while (gen == 0 || gen != tc_getgen(th));
     619                atomic_thread_fence_acq();
     620        } while (gen == 0 || gen != th->th_generation);
    612621}
    613622#endif /* FFCLOCK */
     
    10221031        do {
    10231032                th = timehands;
    1024                 gen = tc_getgen(th);
     1033                gen = atomic_load_acq_int(&th->th_generation);
    10251034                ffth = fftimehands;
    10261035                delta = tc_delta(th);
    10271036                *ffcount = ffth->tick_ffcount;
    1028         } while (gen == 0 || gen != tc_getgen(th));
     1037                atomic_thread_fence_acq();
     1038        } while (gen == 0 || gen != th->th_generation);
    10291039
    10301040        *ffcount += delta;
     
    11311141        do {
    11321142                th = timehands;
    1133                 gen = tc_getgen(th);
     1143                gen = atomic_load_acq_int(&th->th_generation);
    11341144                *tsp = th->th_nanotime;
    1135         } while (gen == 0 || gen != tc_getgen(th));
     1145                atomic_thread_fence_acq();
     1146        } while (gen == 0 || gen != th->th_generation);
    11361147}
    11371148#endif /* __rtems__ */
     
    11751186        do {
    11761187                th = timehands;
    1177                 gen = tc_getgen(th);
     1188                gen = atomic_load_acq_int(&th->th_generation);
    11781189                fbi->th_scale = th->th_scale;
    11791190                fbi->tick_time = th->th_offset;
     
    11891200                if (!fast)
    11901201                        delta = tc_delta(th);
    1191         } while (gen == 0 || gen != tc_getgen(th));
     1202                atomic_thread_fence_acq();
     1203        } while (gen == 0 || gen != th->th_generation);
    11921204
    11931205        clock_snap->delta = delta;
     
    14381450
    14391451        /*
    1440          * Make the next timehands a copy of the current one, but do not
    1441          * overwrite the generation or next pointer.  While we update
    1442          * the contents, the generation must be zero.
     1452         * Make the next timehands a copy of the current one, but do
     1453         * not overwrite the generation or next pointer.  While we
     1454         * update the contents, the generation must be zero.  We need
     1455         * to ensure that the zero generation is visible before the
     1456         * data updates become visible, which requires release fence.
     1457         * For similar reasons, re-reading of the generation after the
     1458         * data is read should use acquire fence.
    14431459         */
    14441460        tho = timehands;
     
    14491465#endif
    14501466        ogen = th->th_generation;
    1451         tc_setgen(th, 0);
     1467        th->th_generation = 0;
     1468        atomic_thread_fence_rel();
    14521469#if defined(RTEMS_SMP)
    14531470        bcopy(tho, th, offsetof(struct timehands, th_generation));
     
    15721589        if (++ogen == 0)
    15731590                ogen = 1;
    1574         tc_setgen(th, ogen);
     1591        atomic_store_rel_int(&th->th_generation, ogen);
    15751592
    15761593        /* Go live with the new struct timehands. */
     
    18561873        KASSERT(pps != NULL, ("NULL pps pointer in pps_capture"));
    18571874        th = timehands;
    1858         pps->capgen = tc_getgen(th);
     1875        pps->capgen = atomic_load_acq_int(&th->th_generation);
    18591876        pps->capth = th;
    18601877#ifdef FFCLOCK
     
    18621879#endif
    18631880        pps->capcount = th->th_counter->tc_get_timecount(th->th_counter);
    1864         if (pps->capgen != tc_getgen(th))
     1881        atomic_thread_fence_acq();
     1882        if (pps->capgen != th->th_generation)
    18651883                pps->capgen = 0;
    18661884}
     
    18821900        KASSERT(pps != NULL, ("NULL pps pointer in pps_event"));
    18831901        /* If the timecounter was wound up underneath us, bail out. */
    1884         if (pps->capgen == 0 || pps->capgen != tc_getgen(pps->capth))
     1902        if (pps->capgen == 0 || pps->capgen !=
     1903            atomic_load_acq_int(&pps->capth->th_generation))
    18851904                return;
    18861905
     
    19321951
    19331952        /* If the timecounter was wound up underneath us, bail out. */
    1934         if (pps->capgen != tc_getgen(pps->capth))
     1953        atomic_thread_fence_acq();
     1954        if (pps->capgen != pps->capth->th_generation)
    19351955                return;
    19361956
Note: See TracChangeset for help on using the changeset viewer.