source: rtems/c/src/lib/libcpu/shared/src/cache_manager.c @ ddbc3f8d

4.115
Last change on this file since ddbc3f8d was ddbc3f8d, checked in by Daniel Cederman <cederman@…>, on Jul 11, 2014 at 2:37:56 PM

score: Add SMP support to the cache manager

Adds functions that allows the user to specify which cores that should
perform the cache operation. SMP messages are sent to all the specified
cores and the caller waits until all cores have acknowledged that they
have flushed their cache. If CPU_CACHE_NO_INSTRUCTION_CACHE_SNOOPING is
defined the instruction cache invalidation function will perform the
operation on all cores using the previous method.

  • Property mode set to 100644
File size: 14.2 KB
Line 
1/*
2 *  Cache Manager
3 *
4 *  COPYRIGHT (c) 1989-1999.
5 *  On-Line Applications Research Corporation (OAR).
6 *
7 *  The license and distribution terms for this file may be
8 *  found in the file LICENSE in this distribution or at
9 *  http://www.rtems.org/license/LICENSE.
10 *
11 *
12 *  The functions in this file implement the API to the RTEMS Cache Manager and
13 *  are divided into data cache and instruction cache functions. Data cache
14 *  functions only have bodies if a data cache is supported. Instruction
15 *  cache functions only have bodies if an instruction cache is supported.
16 *  Support for a particular cache exists only if CPU_x_CACHE_ALIGNMENT is
17 *  defined, where x E {DATA, INSTRUCTION}. These definitions are found in
18 *  the Cache Manager Wrapper header files, often
19 *
20 *  rtems/c/src/lib/libcpu/CPU/cache_.h
21 *
22 *  The cache implementation header file can define
23 *  CPU_CACHE_SUPPORT_PROVIDES_RANGE_FUNCTIONS
24 *  if it provides cache maintenance functions which operate on multiple lines.
25 *  Otherwise a generic loop with single line operations will be used.
26 *
27 *  The functions below are implemented with CPU dependent inline routines
28 *  found in the cache.c files for each CPU. In the event that a CPU does
29 *  not support a specific function for a cache it has, the CPU dependent
30 *  routine does nothing (but does exist).
31 *
32 *  At this point, the Cache Manager makes no considerations, and provides no
33 *  support for BSP specific issues such as a secondary cache. In such a system,
34 *  the CPU dependent routines would have to be modified, or a BSP layer added
35 *  to this Manager.
36 */
37
38#include <rtems.h>
39#include "cache_.h"
40#include <rtems/score/smpimpl.h>
41#include <rtems/score/smplock.h>
42#include <rtems/score/chainimpl.h>
43#include <rtems/score/sysstate.h>
44
45#if defined( RTEMS_SMP )
46
47typedef void (*Cache_manager_Function_ptr)(const void *d_addr, size_t n_bytes);
48
49typedef struct {
50  Chain_Node Node;
51  Cache_manager_Function_ptr func;
52  const void *addr;
53  size_t size;
54  cpu_set_t *recipients;
55  size_t setsize;
56  Atomic_Ulong done;
57} Cache_manager_SMP_node;
58
59typedef struct {
60  SMP_lock_Control Lock;
61  Chain_Control List;
62} Cache_manager_SMP_control;
63
64static Cache_manager_SMP_control _Cache_manager_SMP_control = {
65  .Lock = SMP_LOCK_INITIALIZER("cachemgr"),
66  .List = CHAIN_INITIALIZER_EMPTY(_Cache_manager_SMP_control.List)
67};
68
69void
70_SMP_Cache_manager_message_handler(void)
71{
72  SMP_lock_Context lock_context;
73  Cache_manager_SMP_node *node;
74  Cache_manager_SMP_node *next;
75  uint32_t cpu_self_idx;
76
77  _SMP_lock_ISR_disable_and_acquire( &_Cache_manager_SMP_control.Lock,
78      &lock_context );
79  cpu_self_idx = _SMP_Get_current_processor();
80
81  node = (Cache_manager_SMP_node*)_Chain_First(
82      &_Cache_manager_SMP_control.List );
83  while ( !_Chain_Is_tail( &_Cache_manager_SMP_control.List, &node->Node ) ) {
84    next = (Cache_manager_SMP_node*)_Chain_Next( &node->Node );
85    if ( CPU_ISSET_S ( cpu_self_idx, node->setsize, node->recipients ) ) {
86      CPU_CLR_S ( cpu_self_idx, node->setsize, node->recipients );
87
88      node->func( node->addr, node->size );
89
90      if ( CPU_COUNT_S( node->setsize, node->recipients ) == 0 ) {
91        _Chain_Extract_unprotected( &node->Node );
92        _Atomic_Store_ulong( &node->done, 1, ATOMIC_ORDER_RELEASE );
93      }
94    }
95    node = next;
96  }
97
98  _SMP_lock_Release_and_ISR_enable( &_Cache_manager_SMP_control.Lock,
99      &lock_context );
100}
101
102#if defined(CPU_DATA_CACHE_ALIGNMENT) || \
103    (defined(CPU_INSTRUCTION_CACHE_ALIGNMENT) && \
104    defined(CPU_CACHE_NO_INSTRUCTION_CACHE_SNOOPING))
105
106static void
107_Cache_manager_Process_cache_messages( void )
108{
109  unsigned long message;
110  Per_CPU_Control *cpu_self;
111  ISR_Level isr_level;
112
113  _ISR_Disable_without_giant( isr_level );
114
115  cpu_self = _Per_CPU_Get();
116
117  message = _Atomic_Load_ulong( &cpu_self->message, ATOMIC_ORDER_RELAXED );
118
119  if ( message & SMP_MESSAGE_CACHE_MANAGER ) {
120    if ( _Atomic_Compare_exchange_ulong( &cpu_self->message, &message,
121        message & ~SMP_MESSAGE_CACHE_MANAGER, ATOMIC_ORDER_RELAXED,
122        ATOMIC_ORDER_RELAXED ) ) {
123      _SMP_Cache_manager_message_handler();
124    }
125  }
126
127  _ISR_Enable_without_giant( isr_level );
128}
129
130/*
131 * We can not make this function static as we need to access it
132 * from the test program.
133 */
134void
135_Cache_manager_Send_smp_msg(
136    const size_t setsize,
137    const cpu_set_t *set,
138    Cache_manager_Function_ptr func,
139    const void * addr,
140    size_t size
141  );
142
143void
144_Cache_manager_Send_smp_msg(
145    const size_t setsize,
146    const cpu_set_t *set,
147    Cache_manager_Function_ptr func,
148    const void * addr,
149    size_t size
150  )
151{
152  uint32_t i;
153  Cache_manager_SMP_node node;
154  size_t set_size = CPU_ALLOC_SIZE( _SMP_Get_processor_count() );
155  char cpu_set_copy[set_size];
156  SMP_lock_Context lock_context;
157
158  if ( ! _System_state_Is_up( _System_state_Get() ) ) {
159    func( addr, size );
160    return;
161  }
162
163  memset( cpu_set_copy, 0, set_size );
164  if( set == NULL ) {
165    for( i=0; i<_SMP_Get_processor_count(); ++i )
166      CPU_SET_S( i, set_size, (cpu_set_t *)cpu_set_copy );
167  } else {
168    for( i=0; i<_SMP_Get_processor_count(); ++i )
169      if( CPU_ISSET_S( i, set_size, set ) )
170        CPU_SET_S( i, set_size, (cpu_set_t *)cpu_set_copy );
171  }
172
173  node.func = func;
174  node.addr = addr;
175  node.size = size;
176  node.setsize = set_size;
177  node.recipients = (cpu_set_t *)cpu_set_copy;
178  _Atomic_Store_ulong( &node.done, 0, ATOMIC_ORDER_RELAXED );
179
180
181  _SMP_lock_ISR_disable_and_acquire( &_Cache_manager_SMP_control.Lock,
182      &lock_context );
183  _Chain_Prepend_unprotected( &_Cache_manager_SMP_control.List, &node.Node );
184  _SMP_lock_Release_and_ISR_enable( &_Cache_manager_SMP_control.Lock,
185      &lock_context );
186
187  _SMP_Send_message_multicast( set_size, node.recipients,
188      SMP_MESSAGE_CACHE_MANAGER );
189
190  _Cache_manager_Process_cache_messages();
191
192  while ( !_Atomic_Load_uint( &node.done, ATOMIC_ORDER_ACQUIRE ) );
193}
194#endif
195
196void
197rtems_cache_flush_multiple_data_lines_processor_set(
198  const void *addr,
199  size_t size,
200  const size_t setsize,
201  const cpu_set_t *set
202)
203{
204#if defined(CPU_DATA_CACHE_ALIGNMENT)
205  _Cache_manager_Send_smp_msg( setsize, set,
206      rtems_cache_flush_multiple_data_lines, addr, size );
207#endif
208}
209
210void
211rtems_cache_invalidate_multiple_data_lines_processor_set(
212  const void *addr,
213  size_t size,
214  const size_t setsize,
215  const cpu_set_t *set
216)
217{
218#if defined(CPU_DATA_CACHE_ALIGNMENT)
219  _Cache_manager_Send_smp_msg( setsize, set,
220      rtems_cache_invalidate_multiple_data_lines, addr, size );
221#endif
222}
223
224void
225rtems_cache_flush_entire_data_processor_set(
226  const size_t setsize,
227  const cpu_set_t *set
228)
229{
230#if defined(CPU_DATA_CACHE_ALIGNMENT)
231  _Cache_manager_Send_smp_msg( setsize, set,
232      (Cache_manager_Function_ptr)rtems_cache_flush_entire_data, 0, 0 );
233#endif
234}
235
236void
237rtems_cache_invalidate_entire_data_processor_set(
238  const size_t setsize,
239  const cpu_set_t *set
240)
241{
242#if defined(CPU_DATA_CACHE_ALIGNMENT)
243  _Cache_manager_Send_smp_msg( setsize, set,
244      (Cache_manager_Function_ptr)rtems_cache_invalidate_entire_data, 0, 0 );
245#endif
246}
247#endif
248
249/*
250 * THESE FUNCTIONS ONLY HAVE BODIES IF WE HAVE A DATA CACHE
251 */
252
253/*
254 * This function is called to flush the data cache by performing cache
255 * copybacks. It must determine how many cache lines need to be copied
256 * back and then perform the copybacks.
257 */
258void
259rtems_cache_flush_multiple_data_lines( const void * d_addr, size_t n_bytes )
260{
261#if defined(CPU_DATA_CACHE_ALIGNMENT)
262#if defined(CPU_CACHE_SUPPORT_PROVIDES_RANGE_FUNCTIONS)
263  _CPU_cache_flush_data_range( d_addr, n_bytes );
264#else
265  const void * final_address;
266
267 /*
268  * Set d_addr to the beginning of the cache line; final_address indicates
269  * the last address_t which needs to be pushed. Increment d_addr and push
270  * the resulting line until final_address is passed.
271  */
272
273  if( n_bytes == 0 )
274    /* Do nothing if number of bytes to flush is zero */
275    return;
276
277  final_address = (void *)((size_t)d_addr + n_bytes - 1);
278  d_addr = (void *)((size_t)d_addr & ~(CPU_DATA_CACHE_ALIGNMENT - 1));
279  while( d_addr <= final_address )  {
280    _CPU_cache_flush_1_data_line( d_addr );
281    d_addr = (void *)((size_t)d_addr + CPU_DATA_CACHE_ALIGNMENT);
282  }
283#endif
284#endif
285}
286
287
288/*
289 * This function is responsible for performing a data cache invalidate.
290 * It must determine how many cache lines need to be invalidated and then
291 * perform the invalidations.
292 */
293
294void
295rtems_cache_invalidate_multiple_data_lines( const void * d_addr, size_t n_bytes )
296{
297#if defined(CPU_DATA_CACHE_ALIGNMENT)
298#if defined(CPU_CACHE_SUPPORT_PROVIDES_RANGE_FUNCTIONS)
299  _CPU_cache_invalidate_data_range( d_addr, n_bytes );
300#else
301  const void * final_address;
302
303 /*
304  * Set d_addr to the beginning of the cache line; final_address indicates
305  * the last address_t which needs to be invalidated. Increment d_addr and
306  * invalidate the resulting line until final_address is passed.
307  */
308
309  if( n_bytes == 0 )
310    /* Do nothing if number of bytes to invalidate is zero */
311    return;
312
313  final_address = (void *)((size_t)d_addr + n_bytes - 1);
314  d_addr = (void *)((size_t)d_addr & ~(CPU_DATA_CACHE_ALIGNMENT - 1));
315  while( final_address >= d_addr ) {
316    _CPU_cache_invalidate_1_data_line( d_addr );
317    d_addr = (void *)((size_t)d_addr + CPU_DATA_CACHE_ALIGNMENT);
318  }
319#endif
320#endif
321}
322
323
324/*
325 * This function is responsible for performing a data cache flush.
326 * It flushes the entire cache.
327 */
328void
329rtems_cache_flush_entire_data( void )
330{
331#if defined(CPU_DATA_CACHE_ALIGNMENT)
332   /*
333    * Call the CPU-specific routine
334    */
335   _CPU_cache_flush_entire_data();
336#endif
337}
338
339
340/*
341 * This function is responsible for performing a data cache
342 * invalidate. It invalidates the entire cache.
343 */
344void
345rtems_cache_invalidate_entire_data( void )
346{
347#if defined(CPU_DATA_CACHE_ALIGNMENT)
348 /*
349  * Call the CPU-specific routine
350  */
351
352 _CPU_cache_invalidate_entire_data();
353#endif
354}
355
356
357/*
358 * This function returns the data cache granularity.
359 */
360size_t
361rtems_cache_get_data_line_size( void )
362{
363#if defined(CPU_DATA_CACHE_ALIGNMENT)
364  return CPU_DATA_CACHE_ALIGNMENT;
365#else
366  return 0;
367#endif
368}
369
370
371size_t
372rtems_cache_get_data_cache_size( uint32_t level )
373{
374#if defined(CPU_CACHE_SUPPORT_PROVIDES_CACHE_SIZE_FUNCTIONS)
375  return _CPU_cache_get_data_cache_size( level );
376#else
377  return 0;
378#endif
379}
380
381/*
382 * This function freezes the data cache; cache lines
383 * are not replaced.
384 */
385void
386rtems_cache_freeze_data( void )
387{
388#if defined(CPU_DATA_CACHE_ALIGNMENT)
389  _CPU_cache_freeze_data();
390#endif
391}
392
393
394/*
395 * This function unfreezes the instruction cache.
396 */
397void rtems_cache_unfreeze_data( void )
398{
399#if defined(CPU_DATA_CACHE_ALIGNMENT)
400  _CPU_cache_unfreeze_data();
401#endif
402}
403
404
405/* Turn on the data cache. */
406void
407rtems_cache_enable_data( void )
408{
409#if defined(CPU_DATA_CACHE_ALIGNMENT)
410  _CPU_cache_enable_data();
411#endif
412}
413
414
415/* Turn off the data cache. */
416void
417rtems_cache_disable_data( void )
418{
419#if defined(CPU_DATA_CACHE_ALIGNMENT)
420  _CPU_cache_disable_data();
421#endif
422}
423
424
425
426/*
427 * THESE FUNCTIONS ONLY HAVE BODIES IF WE HAVE AN INSTRUCTION CACHE
428 */
429
430
431
432/*
433 * This function is responsible for performing an instruction cache
434 * invalidate. It must determine how many cache lines need to be invalidated
435 * and then perform the invalidations.
436 */
437
438#if !defined(CPU_CACHE_SUPPORT_PROVIDES_RANGE_FUNCTIONS)
439static void
440_invalidate_multiple_instruction_lines_no_range_functions(
441  const void * i_addr,
442  size_t n_bytes
443)
444{
445  const void * final_address;
446
447 /*
448  * Set i_addr to the beginning of the cache line; final_address indicates
449  * the last address_t which needs to be invalidated. Increment i_addr and
450  * invalidate the resulting line until final_address is passed.
451  */
452
453  if( n_bytes == 0 )
454    /* Do nothing if number of bytes to invalidate is zero */
455    return;
456
457  final_address = (void *)((size_t)i_addr + n_bytes - 1);
458  i_addr = (void *)((size_t)i_addr & ~(CPU_INSTRUCTION_CACHE_ALIGNMENT - 1));
459  while( final_address >= i_addr ) {
460    _CPU_cache_invalidate_1_instruction_line( i_addr );
461    i_addr = (void *)((size_t)i_addr + CPU_INSTRUCTION_CACHE_ALIGNMENT);
462  }
463}
464#endif
465
466void
467rtems_cache_invalidate_multiple_instruction_lines(
468  const void * i_addr,
469  size_t n_bytes
470)
471{
472#if defined(CPU_INSTRUCTION_CACHE_ALIGNMENT)
473#if defined(CPU_CACHE_SUPPORT_PROVIDES_RANGE_FUNCTIONS)
474
475#if defined(RTEMS_SMP) && defined(CPU_CACHE_NO_INSTRUCTION_CACHE_SNOOPING)
476  _Cache_manager_Send_smp_msg( 0, 0, _CPU_cache_invalidate_instruction_range,
477      i_addr, n_bytes );
478#else
479  _CPU_cache_invalidate_instruction_range( i_addr, n_bytes );
480#endif
481
482#else
483
484#if defined(RTEMS_SMP) && defined(CPU_CACHE_NO_INSTRUCTION_CACHE_SNOOPING)
485  _Cache_manager_Send_smp_msg( 0, 0,
486      _invalidate_multiple_instruction_lines_no_range_functions, i_addr,
487      n_bytes );
488#else
489  _invalidate_multiple_instruction_lines_no_range_functions( i_addr, n_bytes );
490#endif
491
492#endif
493#endif
494}
495
496
497/*
498 * This function is responsible for performing an instruction cache
499 * invalidate. It invalidates the entire cache.
500 */
501void
502rtems_cache_invalidate_entire_instruction( void )
503{
504#if defined(CPU_INSTRUCTION_CACHE_ALIGNMENT)
505 /*
506  * Call the CPU-specific routine
507  */
508
509#if defined(RTEMS_SMP) && defined(CPU_CACHE_NO_INSTRUCTION_CACHE_SNOOPING)
510  _Cache_manager_Send_smp_msg( 0, 0,
511      (Cache_manager_Function_ptr)_CPU_cache_invalidate_entire_instruction,
512      0, 0 );
513#else
514 _CPU_cache_invalidate_entire_instruction();
515#endif
516#endif
517}
518
519
520/*
521 * This function returns the instruction cache granularity.
522 */
523size_t
524rtems_cache_get_instruction_line_size( void )
525{
526#if defined(CPU_INSTRUCTION_CACHE_ALIGNMENT)
527  return CPU_INSTRUCTION_CACHE_ALIGNMENT;
528#else
529  return 0;
530#endif
531}
532
533
534size_t
535rtems_cache_get_instruction_cache_size( uint32_t level )
536{
537#if defined(CPU_CACHE_SUPPORT_PROVIDES_CACHE_SIZE_FUNCTIONS)
538  return _CPU_cache_get_instruction_cache_size( level );
539#else
540  return 0;
541#endif
542}
543
544
545/*
546 * This function freezes the instruction cache; cache lines
547 * are not replaced.
548 */
549void
550rtems_cache_freeze_instruction( void )
551{
552#if defined(CPU_INSTRUCTION_CACHE_ALIGNMENT)
553  _CPU_cache_freeze_instruction();
554#endif
555}
556
557
558/*
559 * This function unfreezes the instruction cache.
560 */
561void rtems_cache_unfreeze_instruction( void )
562{
563#if defined(CPU_INSTRUCTION_CACHE_ALIGNMENT)
564  _CPU_cache_unfreeze_instruction();
565#endif
566}
567
568
569/* Turn on the instruction cache. */
570void
571rtems_cache_enable_instruction( void )
572{
573#if defined(CPU_INSTRUCTION_CACHE_ALIGNMENT)
574  _CPU_cache_enable_instruction();
575#endif
576}
577
578
579/* Turn off the instruction cache. */
580void
581rtems_cache_disable_instruction( void )
582{
583#if defined(CPU_INSTRUCTION_CACHE_ALIGNMENT)
584  _CPU_cache_disable_instruction();
585#endif
586}
Note: See TracBrowser for help on using the repository browser.