Changeset e90486a in rtems


Ignore:
Timestamp:
Apr 11, 2019, 1:16:40 PM (2 months ago)
Author:
Sebastian Huber <sebastian.huber@…>
Branches:
master
Children:
f410b31b
Parents:
df8d7bd7
git-author:
Sebastian Huber <sebastian.huber@…> (04/11/19 13:16:40)
git-committer:
Sebastian Huber <sebastian.huber@…> (04/12/19 07:44:48)
Message:

score: Rework SMP multicast action

Use a FIFO list of jobs per processor to carry out the SMP multicast
action. Use a done indicator per job to reduce the bus traffic a bit.

Files:
6 edited

Legend:

Unmodified
Added
Removed
  • cpukit/include/rtems/score/percpu.h

    rdf8d7bd7 re90486a  
    7474
    7575struct Scheduler_Context;
     76
     77struct Per_CPU_Job;
    7678
    7779/**
     
    496498
    497499    /**
     500     * @brief FIFO list of jobs to be performed by this processor.
     501     *
     502     * @see _SMP_Multicast_action().
     503     */
     504    struct {
     505      /**
     506       * @brief Lock to protect the FIFO list of jobs to be performed by this
     507       * processor.
     508       */
     509      ISR_lock_Control Lock;
     510
     511      /**
     512       * @brief Head of the FIFO list of jobs to be performed by this
     513       * processor.
     514       *
     515       * This member is protected by the Per_CPU_Control::Jobs::Lock lock.
     516       */
     517      struct Per_CPU_Job *head;
     518
     519      /**
     520       * @brief Tail of the FIFO list of jobs to be performed by this
     521       * processor.
     522       *
     523       * This member is only valid if the head is not @c NULL.
     524       *
     525       * This member is protected by the Per_CPU_Control::Jobs::Lock lock.
     526       */
     527      struct Per_CPU_Job **tail;
     528    } Jobs;
     529
     530    /**
    498531     * @brief Indicates if the processor has been successfully started via
    499532     * _CPU_SMP_Start_processor().
     
    710743  uint32_t timeout_in_ns
    711744);
     745
     746/**
     747 * @brief Performs the jobs of the specified processor.
     748 *
     749 * @param[in, out] cpu The jobs of this processor will be performed.
     750 */
     751void _Per_CPU_Perform_jobs( Per_CPU_Control *cpu );
    712752
    713753#endif /* defined( RTEMS_SMP ) */
  • cpukit/include/rtems/score/smpimpl.h

    rdf8d7bd7 re90486a  
    5252
    5353/**
    54  * @brief SMP message to request a multicast action.
     54 * @brief SMP message to perform per-processor jobs.
    5555 *
    5656 * @see _SMP_Send_message().
    5757 */
    58 #define SMP_MESSAGE_MULTICAST_ACTION 0x4UL
     58#define SMP_MESSAGE_PERFORM_JOBS 0x4UL
    5959
    6060/**
     
    159159
    160160/**
    161  * @brief Processes all pending multicast actions.
    162  */
    163 void _SMP_Multicast_actions_process( void );
    164 
    165 /**
    166161 * @brief Interrupt handler for inter-processor interrupts.
    167162 *
     
    196191    }
    197192
    198     if ( ( message & SMP_MESSAGE_MULTICAST_ACTION ) != 0 ) {
    199       _SMP_Multicast_actions_process();
     193    if ( ( message & SMP_MESSAGE_PERFORM_JOBS ) != 0 ) {
     194      _Per_CPU_Perform_jobs( cpu_self );
    200195    }
    201196  }
  • cpukit/score/src/smp.c

    rdf8d7bd7 re90486a  
    119119    cpu = _Per_CPU_Get_by_index( cpu_index );
    120120    _ISR_lock_Set_name( &cpu->Lock, "Per-CPU" );
     121    _ISR_lock_Set_name( &cpu->Jobs.Lock, "Per-CPU Jobs" );
    121122    _ISR_lock_Set_name( &cpu->Watchdog.Lock, "Per-CPU Watchdog" );
    122123    _Chain_Initialize_empty( &cpu->Threads_in_need_for_help );
  • cpukit/score/src/smpmulticastaction.c

    rdf8d7bd7 re90486a  
    11/*
    2  * Copyright (c) 2014 Aeroflex Gaisler AB.  All rights reserved.
     2 * SPDX-License-Identifier: BSD-2-Clause
    33 *
    4  * The license and distribution terms for this file may be
    5  * found in the file LICENSE in this distribution or at
    6  * http://www.rtems.org/license/LICENSE.
     4 * Copyright (C) 2019 embedded brains GmbH
     5 *
     6 * Redistribution and use in source and binary forms, with or without
     7 * modification, are permitted provided that the following conditions
     8 * are met:
     9 * 1. Redistributions of source code must retain the above copyright
     10 *    notice, this list of conditions and the following disclaimer.
     11 * 2. Redistributions in binary form must reproduce the above copyright
     12 *    notice, this list of conditions and the following disclaimer in the
     13 *    documentation and/or other materials provided with the distribution.
     14 *
     15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
     16 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     18 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
     19 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     20 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     21 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     22 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     24 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     25 * POSSIBILITY OF SUCH DAMAGE.
    726 */
    827
    928#ifdef HAVE_CONFIG_H
    10   #include "config.h"
     29#include "config.h"
    1130#endif
    1231
    1332#include <rtems/score/smpimpl.h>
    14 #include <rtems/score/isrlock.h>
    15 #include <rtems/score/chainimpl.h>
     33#include <rtems/score/assert.h>
     34#include <rtems/score/threaddispatch.h>
    1635#include <rtems/score/sysstate.h>
    1736
    18 typedef struct {
    19   Chain_Node          Node;
    20   SMP_Action_handler  handler;
    21   void               *arg;
    22   Processor_mask      targets;
    23   Atomic_Ulong        done;
    24 } SMP_Multicast_action;
    25 
    26 typedef struct {
    27   ISR_lock_Control Lock;
    28   Chain_Control    Actions;
    29 } SMP_Multicast_context;
    30 
    31 static SMP_Multicast_context _SMP_Multicast = {
    32   .Lock = ISR_LOCK_INITIALIZER( "SMP Multicast Action" ),
    33   .Actions = CHAIN_INITIALIZER_EMPTY( _SMP_Multicast.Actions )
     37typedef struct Per_CPU_Job Per_CPU_Job;
     38
     39typedef struct Per_CPU_Jobs Per_CPU_Jobs;
     40
     41/**
     42 * @brief A per-processor job.
     43 */
     44struct Per_CPU_Job {
     45  union {
     46    /**
     47     * @brief The next job in the corresponding per-processor job list.
     48     */
     49    Per_CPU_Job *next;
     50
     51    /**
     52     * @brief Indication if the job is done.
     53     *
     54     * A job is done if this member has the value one.  This assumes that one
     55     * is not a valid pointer value.
     56     */
     57    Atomic_Ulong done;
     58  };
     59
     60  /**
     61   * @brief Back pointer to the jobs to get the handler and argument.
     62   */
     63  Per_CPU_Jobs *jobs;
    3464};
    3565
    36 void _SMP_Multicast_actions_process( void )
    37 {
    38   ISR_lock_Context      lock_context;
    39   uint32_t              cpu_self_index;
    40   SMP_Multicast_action *node;
    41   SMP_Multicast_action *next;
    42 
    43   _ISR_lock_ISR_disable_and_acquire( &_SMP_Multicast.Lock, &lock_context );
    44   cpu_self_index = _SMP_Get_current_processor();
    45   node = (SMP_Multicast_action *) _Chain_First( &_SMP_Multicast.Actions );
    46 
    47   while ( !_Chain_Is_tail( &_SMP_Multicast.Actions, &node->Node ) ) {
    48     next = (SMP_Multicast_action *) _Chain_Next( &node->Node );
    49 
    50     if ( _Processor_mask_Is_set( &node->targets, cpu_self_index ) ) {
    51       _Processor_mask_Clear( &node->targets, cpu_self_index );
    52 
    53       ( *node->handler )( node->arg );
    54 
    55       if ( _Processor_mask_Is_zero( &node->targets ) ) {
    56         _Chain_Extract_unprotected( &node->Node );
    57         _Atomic_Store_ulong( &node->done, 1, ATOMIC_ORDER_RELEASE );
     66/**
     67 * @brief A collection of jobs, one for each processor.
     68 */
     69struct Per_CPU_Jobs {
     70  /**
     71   * @brief The job handler.
     72   */
     73  SMP_Action_handler handler;
     74
     75  /**
     76   * @brief The job handler argument.
     77   */
     78  void *arg;
     79
     80  /**
     81   * @brief One job for each potential processor.
     82   */
     83  Per_CPU_Job Jobs[ CPU_MAXIMUM_PROCESSORS ];
     84};
     85
     86void _Per_CPU_Perform_jobs( Per_CPU_Control *cpu )
     87{
     88  ISR_lock_Context  lock_context;
     89  Per_CPU_Job      *job;
     90
     91  _ISR_lock_ISR_disable( &lock_context );
     92  _Per_CPU_Acquire( cpu, &lock_context );
     93
     94  while ( ( job = cpu->Jobs.head ) != NULL ) {
     95    Per_CPU_Jobs *jobs;
     96
     97    cpu->Jobs.head = job->next;
     98    _Per_CPU_Release( cpu, &lock_context );
     99    _ISR_lock_ISR_enable( &lock_context );
     100
     101    jobs = job->jobs;
     102    ( *jobs->handler )( jobs->arg );
     103    _Atomic_Store_ulong( &job->done, 1, ATOMIC_ORDER_RELEASE );
     104
     105     _ISR_lock_ISR_disable( &lock_context );
     106    _Per_CPU_Acquire( cpu, &lock_context );
     107  }
     108
     109  _Per_CPU_Release( cpu, &lock_context );
     110  _ISR_lock_ISR_enable( &lock_context );
     111}
     112
     113static void _Per_CPU_Try_perform_jobs( Per_CPU_Control *cpu_self )
     114{
     115  unsigned long message;
     116
     117  message = _Atomic_Load_ulong( &cpu_self->message, ATOMIC_ORDER_RELAXED );
     118
     119  if ( ( message & SMP_MESSAGE_PERFORM_JOBS ) != 0 ) {
     120    bool success;
     121
     122    success = _Atomic_Compare_exchange_ulong(
     123      &cpu_self->message, &message,
     124      message & ~SMP_MESSAGE_PERFORM_JOBS, ATOMIC_ORDER_RELAXED,
     125      ATOMIC_ORDER_RELAXED
     126    );
     127
     128    if ( success ) {
     129      _Per_CPU_Perform_jobs( cpu_self );
     130    }
     131  }
     132}
     133
     134static void _SMP_Issue_action_jobs(
     135  const Processor_mask *targets,
     136  Per_CPU_Jobs         *jobs,
     137  uint32_t              cpu_max
     138)
     139{
     140  uint32_t cpu_index;
     141
     142  for ( cpu_index = 0; cpu_index < cpu_max; ++cpu_index ) {
     143    if ( _Processor_mask_Is_set( targets, cpu_index ) ) {
     144      ISR_lock_Context  lock_context;
     145      Per_CPU_Job      *job;
     146      Per_CPU_Control  *cpu;
     147
     148      job = &jobs->Jobs[ cpu_index ];
     149      _Atomic_Store_ulong( &job->done, 0, ATOMIC_ORDER_RELAXED );
     150      _Assert( job->next == NULL );
     151      job->jobs = jobs;
     152
     153      cpu = _Per_CPU_Get_by_index( cpu_index );
     154      _ISR_lock_ISR_disable( &lock_context );
     155      _Per_CPU_Acquire( cpu, &lock_context );
     156
     157      if ( cpu->Jobs.head == NULL ) {
     158        cpu->Jobs.head = job;
     159      } else {
     160        *cpu->Jobs.tail = job;
     161      }
     162
     163      cpu->Jobs.tail = &job->next;
     164
     165      _Per_CPU_Release( cpu, &lock_context );
     166      _ISR_lock_ISR_enable( &lock_context );
     167      _SMP_Send_message( cpu_index, SMP_MESSAGE_PERFORM_JOBS );
     168    }
     169  }
     170}
     171
     172static void _SMP_Wait_for_action_jobs(
     173  const Processor_mask *targets,
     174  const Per_CPU_Jobs   *jobs,
     175  uint32_t              cpu_max,
     176  Per_CPU_Control      *cpu_self
     177)
     178{
     179  uint32_t cpu_index;
     180
     181  for ( cpu_index = 0; cpu_index < cpu_max; ++cpu_index ) {
     182    if ( _Processor_mask_Is_set( targets, cpu_index ) ) {
     183      const Per_CPU_Job *job;
     184
     185      job = &jobs->Jobs[ cpu_index ];
     186
     187      while ( _Atomic_Load_ulong( &job->done, ATOMIC_ORDER_ACQUIRE ) == 0 ) {
     188        _Per_CPU_Try_perform_jobs( cpu_self );
    58189      }
    59190    }
    60 
    61     node = next;
    62   }
    63 
    64   _ISR_lock_Release_and_ISR_enable( &_SMP_Multicast.Lock, &lock_context );
    65 }
    66 
    67 static void
    68 _SMP_Multicasts_try_process( void )
    69 {
    70   unsigned long message;
    71   Per_CPU_Control *cpu_self;
    72   ISR_Level isr_level;
    73 
    74   _ISR_Local_disable( isr_level );
    75 
    76   cpu_self = _Per_CPU_Get();
    77 
    78   message = _Atomic_Load_ulong( &cpu_self->message, ATOMIC_ORDER_RELAXED );
    79 
    80   if ( message & SMP_MESSAGE_MULTICAST_ACTION ) {
    81     if ( _Atomic_Compare_exchange_ulong( &cpu_self->message, &message,
    82         message & ~SMP_MESSAGE_MULTICAST_ACTION, ATOMIC_ORDER_RELAXED,
    83         ATOMIC_ORDER_RELAXED ) ) {
    84       _SMP_Multicast_actions_process();
    85     }
    86   }
    87 
    88   _ISR_Local_enable( isr_level );
     191  }
    89192}
    90193
     
    95198)
    96199{
    97   SMP_Multicast_action node;
    98   ISR_lock_Context     lock_context;
    99   uint32_t             i;
     200  Per_CPU_Jobs     jobs;
     201  uint32_t         cpu_max;
     202  Per_CPU_Control *cpu_self;
     203
     204  cpu_max = _SMP_Get_processor_maximum();
     205  _Assert( cpu_max <= CPU_MAXIMUM_PROCESSORS );
    100206
    101207  if ( ! _System_state_Is_up( _System_state_Get() ) ) {
     
    104210  }
    105211
    106   if( targets == NULL ) {
     212  if ( targets == NULL ) {
    107213    targets = _SMP_Get_online_processors();
    108214  }
    109215
    110   _Chain_Initialize_node( &node.Node );
    111   node.handler = handler;
    112   node.arg = arg;
    113   _Processor_mask_Assign( &node.targets, targets );
    114   _Atomic_Store_ulong( &node.done, 0, ATOMIC_ORDER_RELAXED );
    115 
    116   _ISR_lock_ISR_disable_and_acquire( &_SMP_Multicast.Lock, &lock_context );
    117   _Chain_Prepend_unprotected( &_SMP_Multicast.Actions, &node.Node );
    118   _ISR_lock_Release_and_ISR_enable( &_SMP_Multicast.Lock, &lock_context );
    119 
    120   _SMP_Send_message_multicast( targets, SMP_MESSAGE_MULTICAST_ACTION );
    121   _SMP_Multicasts_try_process();
    122 
    123   while ( _Atomic_Load_ulong( &node.done, ATOMIC_ORDER_ACQUIRE ) == 0 ) {
    124     /* Wait */
    125   };
    126 }
     216  jobs.handler = handler;
     217  jobs.arg = arg;
     218
     219  cpu_self = _Thread_Dispatch_disable();
     220  _SMP_Issue_action_jobs( targets, &jobs, cpu_max );
     221  _SMP_Wait_for_action_jobs( targets, &jobs, cpu_max, cpu_self );
     222  _Thread_Dispatch_enable( cpu_self );
     223}
  • testsuites/smptests/smpcache01/init.c

    rdf8d7bd7 re90486a  
    127127
    128128  for (i = 0; i < RTEMS_ARRAY_SIZE( test_cases ); ++i) {
    129     ISR_Level isr_level;
    130 
     129    Per_CPU_Control *cpu_self;
     130    ISR_Level        isr_level;
     131
     132    cpu_self = _Thread_Dispatch_disable();
    131133    _ISR_Local_disable( isr_level );
    132134    barrier( bs );
    133135    ( *test_cases[ i ] )();
    134136    _ISR_Local_enable( isr_level );
     137    _Thread_Dispatch_enable( cpu_self );
    135138    barrier( bs );
    136139  }
  • testsuites/smptests/smpmulticast01/init.c

    rdf8d7bd7 re90486a  
    6060)
    6161{
     62  Per_CPU_Control *cpu_self;
    6263  rtems_interrupt_level level;
    6364
     65  cpu_self = _Thread_Dispatch_disable();
    6466  rtems_interrupt_local_disable(level);
    6567  _SMP_Multicast_action(targets, handler, arg);
    6668  rtems_interrupt_local_enable(level);
     69  _Thread_Dispatch_enable(cpu_self);
    6770}
    6871
Note: See TracChangeset for help on using the changeset viewer.