Changeset 258ad71 in rtems


Ignore:
Timestamp:
Sep 25, 2015, 12:34:24 PM (4 years ago)
Author:
Sebastian Huber <sebastian.huber@…>
Branches:
master
Children:
f91fbbf
Parents:
aee6a1d0
git-author:
Sebastian Huber <sebastian.huber@…> (09/25/15 12:34:24)
git-committer:
Sebastian Huber <sebastian.huber@…> (09/28/15 11:56:57)
Message:

SMP: Fix and optimize thread dispatching

According to the C11 and C++11 memory models only a read-modify-write
operation guarantees that we read the last value written in modification
order. Avoid the sequential consistent thread fence and instead use the
inter-processor interrupt to set the thread dispatch necessary
indicator.

Files:
11 edited

Legend:

Unmodified
Added
Removed
  • c/src/lib/libbsp/arm/shared/arm-a9mpcore-smp.c

    raee6a1d0 r258ad71  
    11/*
    2  * Copyright (c) 2013 embedded brains GmbH.  All rights reserved.
     2 * Copyright (c) 2013-2015 embedded brains GmbH.  All rights reserved.
    33 *
    44 *  embedded brains GmbH
     
    6363void _CPU_SMP_Send_interrupt( uint32_t target_processor_index )
    6464{
     65  _ARM_Data_memory_barrier();
    6566  arm_gic_irq_generate_software_irq(
    6667    ARM_GIC_IRQ_SGI_0,
  • c/src/lib/libbsp/powerpc/qoriq/startup/bspsmp.c

    raee6a1d0 r258ad71  
    236236void _CPU_SMP_Send_interrupt(uint32_t target_processor_index)
    237237{
     238#ifdef __PPC_CPU_E6500__
     239  ppc_light_weight_synchronize();
     240#else
     241  ppc_synchronize_data();
     242#endif
    238243  qoriq.pic.ipidr [IPI_INDEX].reg = 1U << target_processor_index;
    239244}
  • c/src/lib/libbsp/sparc/shared/irq_asm.S

    raee6a1d0 r258ad71  
    237237         mov    1, %g1
    238238
    239         ! Check if a thread dispatch is necessary
    240         ldub    [%g6 + PER_CPU_DISPATCH_NEEDED], %g1
    241         cmp     %g1, 0
    242         beq     check_is_executing
    243          nop
    244 
    245         ! We have a new heir
    246 
    247         ! Clear the thread dispatch necessary flag
    248         stub    %g0, [%g6 + PER_CPU_DISPATCH_NEEDED]
    249 
    250         ! Here we assume a strong memory order, otherwise a memory barrier must
    251         ! be inserted here
     239        ! We may have a new heir
    252240
    253241        ! Read the executing and heir
    254242        ld      [%g6 + PER_CPU_OFFSET_EXECUTING], %g1
    255243        ld      [%g6 + PER_CPU_OFFSET_HEIR], %g2
     244
     245        ! Update the executing only if necessary to avoid cache line
     246        ! monopolization.
     247        cmp     %g1, %g2
     248        beq     try_update_is_executing
     249         mov    1, %g1
    256250
    257251        ! Calculate the heir context pointer
  • c/src/lib/libcpu/powerpc/new-exceptions/cpu_asm.S

    raee6a1d0 r258ad71  
    414414        lwarx   r7, r0, r6
    415415        cmpwi   r7, 0
    416         bne     check_thread_dispatch_necessary
     416        bne     get_potential_new_heir
    417417
    418418        /* Try to update the is executing indicator of the heir context */
    419419        li      r7, 1
    420420        stwcx.  r7, r0, r6
    421         bne     check_thread_dispatch_necessary
     421        bne     get_potential_new_heir
    422422        isync
    423423#endif
     
    537537
    538538#ifdef RTEMS_SMP
    539 check_thread_dispatch_necessary:
     539get_potential_new_heir:
    540540
    541541        GET_SELF_CPU_CONTROL    r6
    542542
    543         /* Check if a thread dispatch is necessary */
    544         lbz     r7, PER_CPU_DISPATCH_NEEDED(r6)
    545         cmpwi   r7, 0
    546         beq     check_is_executing
    547 
    548         /* We have a new heir */
    549 
    550         /* Clear the thread dispatch necessary flag */
    551         li      r7, 0
    552         stb     r7, PER_CPU_DISPATCH_NEEDED(r6)
    553         msync
     543        /* We may have a new heir */
    554544
    555545        /* Read the executing and heir */
    556546        lwz     r7, PER_CPU_OFFSET_EXECUTING(r6)
    557547        lwz     r8, PER_CPU_OFFSET_HEIR(r6)
     548
     549        /*
     550         * Update the executing only if necessary to avoid cache line
     551         * monopolization.
     552         */
     553        cmpw    r7, r8
     554        beq     check_is_executing
    558555
    559556        /* Calculate the heir context pointer */
  • c/src/lib/libcpu/powerpc/shared/include/powerpc-utility.h

    raee6a1d0 r258ad71  
    99
    1010/*
    11  * Copyright (c) 2008-2014 embedded brains GmbH.
     11 * Copyright (c) 2008-2015 embedded brains GmbH.
    1212 *
    1313 *  embedded brains GmbH
     
    205205
    206206  __asm__ volatile ("sync");
     207}
     208
     209static inline void ppc_light_weight_synchronize(void)
     210{
     211  RTEMS_COMPILER_MEMORY_BARRIER();
     212
     213  __asm__ volatile ("lwsync");
    207214}
    208215
  • cpukit/score/cpu/arm/cpu_asm.S

    raee6a1d0 r258ad71  
    2020 *  Emmanuel Raguet, mailto:raguet@crf.canon.fr
    2121 *
    22  *  Copyright (c) 2013-2014 embedded brains GmbH
     22 *  Copyright (c) 2013-2015 embedded brains GmbH
    2323 *
    2424 *  The license and distribution terms for this file may be
     
    8181        ldrexb  r4, [r3]
    8282        cmp     r4, #0
    83         bne     .L_check_thread_dispatch_necessary
     83        bne     .L_get_potential_new_heir
    8484
    8585        /* Try to update the is executing indicator of the heir context */
     
    8787        strexb  r5, r4, [r3]
    8888        cmp     r5, #0
    89         bne     .L_check_thread_dispatch_necessary
     89        bne     .L_get_potential_new_heir
    9090        dmb
    9191#endif
     
    127127
    128128#ifdef RTEMS_SMP
    129 .L_check_thread_dispatch_necessary:
     129.L_get_potential_new_heir:
    130130
    131131        GET_SELF_CPU_CONTROL    r2, r3
    132132
    133         /* Check if a thread dispatch is necessary */
    134         ldrb    r4, [r2, #PER_CPU_DISPATCH_NEEDED]
    135         cmp     r4, #0
    136         beq     .L_check_is_executing
    137 
    138         /* We have a new heir */
    139 
    140         /* Clear the thread dispatch necessary flag */
    141         mov     r4, #0
    142         strb    r4, [r2, #PER_CPU_DISPATCH_NEEDED]
    143         dmb
     133        /* We may have a new heir */
    144134
    145135        /* Read the executing and heir */
    146136        ldr     r4, [r2, #PER_CPU_OFFSET_EXECUTING]
    147137        ldr     r5, [r2, #PER_CPU_OFFSET_HEIR]
     138
     139        /*
     140         * Update the executing only if necessary to avoid cache line
     141         * monopolization.
     142         */
     143        cmp     r4, r5
     144        beq     .L_check_is_executing
    148145
    149146        /* Calculate the heir context pointer */
  • cpukit/score/cpu/no_cpu/rtems/score/cpu.h

    raee6a1d0 r258ad71  
    560560     * This field must be updated during a context switch.  The context switch
    561561     * to the heir must wait until the heir context indicates that it is no
    562      * longer executing on a processor.  The context switch must also check if
    563      * a thread dispatch is necessary to honor updates of the heir thread for
    564      * this processor.  This indicator must be updated using an atomic test and
    565      * set operation to ensure that at most one processor uses the heir
    566      * context at the same time.
     562     * longer executing on a processor.  This indicator must be updated using
     563     * an atomic test and set operation to ensure that at most one processor
     564     * uses the heir context at the same time.  The context switch must also
     565     * check for a potential new heir thread for this processor in case the
     566     * heir context is not immediately available.  Update the executing thread
     567     * for this processor only if necessary to avoid a cache line
     568     * monopolization.
    567569     *
    568570     * @code
    569571     * void _CPU_Context_switch(
    570      *   Context_Control *executing,
    571      *   Context_Control *heir
     572     *   Context_Control *executing_context,
     573     *   Context_Control *heir_context
    572574     * )
    573575     * {
    574      *   save( executing );
     576     *   save( executing_context );
    575577     *
    576      *   executing->is_executing = false;
     578     *   executing_context->is_executing = false;
    577579     *   memory_barrier();
    578580     *
    579      *   if ( test_and_set( &heir->is_executing ) ) {
     581     *   if ( test_and_set( &heir_context->is_executing ) ) {
    580582     *     do {
    581583     *       Per_CPU_Control *cpu_self = _Per_CPU_Get_snapshot();
     584     *       Thread_Control *executing = cpu_self->executing;
     585     *       Thread_Control *heir = cpu_self->heir;
    582586     *
    583      *       if ( cpu_self->dispatch_necessary ) {
    584      *         heir = _Thread_Get_heir_and_make_it_executing( cpu_self );
     587     *       if ( heir != executing ) {
     588     *         cpu_self->executing = heir;
     589     *         heir_context = (Context_Control *)
     590     *           ((uintptr_t) heir + (uintptr_t) executing_context
     591     *             - (uintptr_t) executing)
    585592     *       }
    586      *     } while ( test_and_set( &heir->is_executing ) );
     593     *     } while ( test_and_set( &heir_context->is_executing ) );
    587594     *   }
    588595     *
    589      *   restore( heir );
     596     *   restore( heir_context );
    590597     * }
    591598     * @endcode
     
    15791586   * processor.
    15801587   *
     1588   * This interrupt send and the corresponding inter-processor interrupt must
     1589   * act as an release/acquire barrier so that all values written by the
     1590   * sending processor are visible to the target processor.
     1591   *
    15811592   * This operation is undefined for target processor indices out of range.
    15821593   *
  • cpukit/score/include/rtems/score/percpu.h

    raee6a1d0 r258ad71  
    280280   *
    281281   * This field is not protected by a lock.  The only writer after multitasking
    282    * start is the scheduler owning this processor.  This processor will set the
    283    * dispatch necessary indicator to false, before it reads the heir.  This
    284    * field is used in combination with the dispatch necessary indicator.
     282   * start is the scheduler owning this processor.  It is assumed that stores
     283   * to pointers are atomic on all supported SMP architectures.  The CPU port
     284   * specific code (inter-processor interrupt handling and
     285   * _CPU_SMP_Send_interrupt()) must guarantee that this processor observes the
     286   * last value written.
    285287   *
    286288   * A thread can be a heir on at most one processor in the system.
     
    291293
    292294  /**
    293    * @brief This is set to true when this processor needs to run the
     295   * @brief This is set to true when this processor needs to run the thread
    294296   * dispatcher.
    295297   *
    296298   * It is volatile since interrupts may alter this flag.
    297299   *
    298    * This field is not protected by a lock.  There are two writers after
    299    * multitasking start.  The scheduler owning this processor sets this
    300    * indicator to true, after it updated the heir field.  This processor sets
    301    * this indicator to false, before it reads the heir.  This field is used in
    302    * combination with the heir field.
     300   * This field is not protected by a lock and must be accessed only by this
     301   * processor.  Code (e.g. scheduler and post-switch action requests) running
     302   * on another processors must use an inter-processor interrupt to set the
     303   * thread dispatch necessary indicator to true.
    303304   *
    304305   * @see _Thread_Get_heir_and_make_it_executing().
  • cpukit/score/include/rtems/score/smpimpl.h

    raee6a1d0 r258ad71  
    147147  Per_CPU_Control *cpu_self = _Per_CPU_Get();
    148148
     149  /*
     150   * In the common case the inter-processor interrupt is issued to carry out a
     151   * thread dispatch.
     152   */
     153  cpu_self->dispatch_necessary = true;
     154
    149155  if ( _Atomic_Load_ulong( &cpu_self->message, ATOMIC_ORDER_RELAXED ) != 0 ) {
    150156    unsigned long message = _Atomic_Exchange_ulong(
  • cpukit/score/include/rtems/score/threadimpl.h

    raee6a1d0 r258ad71  
    1212 *  On-Line Applications Research Corporation (OAR).
    1313 *
    14  *  Copyright (c) 2014 embedded brains GmbH.
     14 *  Copyright (c) 2014-2015 embedded brains GmbH.
    1515 *
    1616 *  The license and distribution terms for this file may be
     
    794794 * @brief Gets the heir of the processor and makes it executing.
    795795 *
    796  * The thread dispatch necessary indicator is cleared as a side-effect.
     796 * Must be called with interrupts disabled.  The thread dispatch necessary
     797 * indicator is cleared as a side-effect.
    797798 *
    798799 * @return The heir thread.
     
    807808  Thread_Control *heir;
    808809
     810  heir = cpu_self->heir;
    809811  cpu_self->dispatch_necessary = false;
    810 
    811 #if defined( RTEMS_SMP )
    812   /*
    813    * It is critical that we first update the dispatch necessary and then the
    814    * read the heir so that we don't miss an update by
    815    * _Thread_Dispatch_update_heir().
    816    */
    817   _Atomic_Fence( ATOMIC_ORDER_SEQ_CST );
    818 #endif
    819 
    820   heir = cpu_self->heir;
    821812  cpu_self->executing = heir;
    822813
     
    833824  cpu_for_heir->heir = heir;
    834825
    835   /*
    836    * It is critical that we first update the heir and then the dispatch
    837    * necessary so that _Thread_Get_heir_and_make_it_executing() cannot miss an
    838    * update.
    839    */
    840   _Atomic_Fence( ATOMIC_ORDER_SEQ_CST );
    841 
    842   /*
    843    * Only update the dispatch necessary indicator if not already set to
    844    * avoid superfluous inter-processor interrupts.
    845    */
    846   if ( !cpu_for_heir->dispatch_necessary ) {
    847     cpu_for_heir->dispatch_necessary = true;
    848 
    849     if ( cpu_for_heir != cpu_self ) {
    850       _Per_CPU_Send_interrupt( cpu_for_heir );
    851     }
     826  if ( cpu_for_heir == cpu_self ) {
     827    cpu_self->dispatch_necessary = true;
     828  } else {
     829    _Per_CPU_Send_interrupt( cpu_for_heir );
    852830  }
    853831}
     
    931909
    932910  cpu_of_thread = _Thread_Action_ISR_disable_and_acquire( thread, &level );
    933   cpu_of_thread->dispatch_necessary = true;
    934 
    935 #if defined(RTEMS_SMP)
    936   if ( _Per_CPU_Get() != cpu_of_thread ) {
     911
     912#if defined(RTEMS_SMP)
     913  if ( _Per_CPU_Get() == cpu_of_thread ) {
     914    cpu_of_thread->dispatch_necessary = true;
     915  } else {
    937916    _Per_CPU_Send_interrupt( cpu_of_thread );
    938917  }
     918#else
     919  cpu_of_thread->dispatch_necessary = true;
    939920#endif
    940921
  • testsuites/smptests/smpthreadlife01/init.c

    raee6a1d0 r258ad71  
    202202
    203203  _ISR_Disable_without_giant(level);
    204   (void) level;
    205204
    206205  /* (C) */
     
    216215    _Thread_Disable_dispatch();
    217216  }
     217
     218  _ISR_Enable_without_giant(level);
    218219
    219220  /*
Note: See TracChangeset for help on using the changeset viewer.