source: rtems/cpukit/libblock/src/bdbuf.c @ 2cc4f3c

4.11
Last change on this file since 2cc4f3c was 2cc4f3c, checked in by Sebastian Huber <sebastian.huber@…>, on Jul 3, 2012 at 8:20:58 AM

libblock: Move read-ahead trigger check

In case the read-ahead task has a higher priority than the reading task,
this ensures that the read-ahead request is issued after the read miss
is serviced.

  • Property mode set to 100644
File size: 83.5 KB
Line 
1/**
2 * @file
3 *
4 * @ingroup rtems_bdbuf
5 *
6 * Block device buffer management.
7 */
8
9/*
10 * Disk I/O buffering
11 * Buffer managment
12 *
13 * Copyright (C) 2001 OKTET Ltd., St.-Peterburg, Russia
14 * Author: Andrey G. Ivanov <Andrey.Ivanov@oktet.ru>
15 *         Victor V. Vengerov <vvv@oktet.ru>
16 *         Alexander Kukuta <kam@oktet.ru>
17 *
18 * Copyright (C) 2008,2009 Chris Johns <chrisj@rtems.org>
19 *    Rewritten to remove score mutex access. Fixes many performance
20 *    issues.
21 *
22 * Copyright (c) 2009-2012 embedded brains GmbH.
23 *
24 * @(#) bdbuf.c,v 1.14 2004/04/17 08:15:17 ralf Exp
25 */
26
27/**
28 * Set to 1 to enable debug tracing.
29 */
30#define RTEMS_BDBUF_TRACE 0
31
32#if HAVE_CONFIG_H
33#include "config.h"
34#endif
35#include <limits.h>
36#include <errno.h>
37#include <stdio.h>
38#include <string.h>
39#include <inttypes.h>
40
41#include <rtems.h>
42#include <rtems/error.h>
43#include <rtems/malloc.h>
44
45#include "rtems/bdbuf.h"
46
47#define BDBUF_INVALID_DEV NULL
48
49/*
50 * Simpler label for this file.
51 */
52#define bdbuf_config rtems_bdbuf_configuration
53
54/**
55 * A swapout transfer transaction data. This data is passed to a worked thread
56 * to handle the write phase of the transfer.
57 */
58typedef struct rtems_bdbuf_swapout_transfer
59{
60  rtems_chain_control   bds;         /**< The transfer list of BDs. */
61  rtems_disk_device    *dd;          /**< The device the transfer is for. */
62  bool                  syncing;     /**< The data is a sync'ing. */
63  rtems_blkdev_request* write_req;   /**< The write request array. */
64} rtems_bdbuf_swapout_transfer;
65
66/**
67 * Swapout worker thread. These are available to take processing from the
68 * main swapout thread and handle the I/O operation.
69 */
70typedef struct rtems_bdbuf_swapout_worker
71{
72  rtems_chain_node             link;     /**< The threads sit on a chain when
73                                          * idle. */
74  rtems_id                     id;       /**< The id of the task so we can wake
75                                          * it. */
76  bool                         enabled;  /**< The worker is enabled. */
77  rtems_bdbuf_swapout_transfer transfer; /**< The transfer data for this
78                                          * thread. */
79} rtems_bdbuf_swapout_worker;
80
81/**
82 * Buffer waiters synchronization.
83 */
84typedef struct rtems_bdbuf_waiters {
85  unsigned count;
86  rtems_id sema;
87} rtems_bdbuf_waiters;
88
89/**
90 * The BD buffer cache.
91 */
92typedef struct rtems_bdbuf_cache
93{
94  rtems_id            swapout;           /**< Swapout task ID */
95  bool                swapout_enabled;   /**< Swapout is only running if
96                                          * enabled. Set to false to kill the
97                                          * swap out task. It deletes itself. */
98  rtems_chain_control swapout_workers;   /**< The work threads for the swapout
99                                          * task. */
100
101  rtems_bdbuf_buffer* bds;               /**< Pointer to table of buffer
102                                          * descriptors. */
103  void*               buffers;           /**< The buffer's memory. */
104  size_t              buffer_min_count;  /**< Number of minimum size buffers
105                                          * that fit the buffer memory. */
106  size_t              max_bds_per_group; /**< The number of BDs of minimum
107                                          * buffer size that fit in a group. */
108  uint32_t            flags;             /**< Configuration flags. */
109
110  rtems_id            lock;              /**< The cache lock. It locks all
111                                          * cache data, BD and lists. */
112  rtems_id            sync_lock;         /**< Sync calls block writes. */
113  bool                sync_active;       /**< True if a sync is active. */
114  rtems_id            sync_requester;    /**< The sync requester. */
115  rtems_disk_device  *sync_device;       /**< The device to sync and
116                                          * BDBUF_INVALID_DEV not a device
117                                          * sync. */
118
119  rtems_bdbuf_buffer* tree;              /**< Buffer descriptor lookup AVL tree
120                                          * root. There is only one. */
121  rtems_chain_control lru;               /**< Least recently used list */
122  rtems_chain_control modified;          /**< Modified buffers list */
123  rtems_chain_control sync;              /**< Buffers to sync list */
124
125  rtems_bdbuf_waiters access_waiters;    /**< Wait for a buffer in
126                                          * ACCESS_CACHED, ACCESS_MODIFIED or
127                                          * ACCESS_EMPTY
128                                          * state. */
129  rtems_bdbuf_waiters transfer_waiters;  /**< Wait for a buffer in TRANSFER
130                                          * state. */
131  rtems_bdbuf_waiters buffer_waiters;    /**< Wait for a buffer and no one is
132                                          * available. */
133
134  size_t              group_count;       /**< The number of groups. */
135  rtems_bdbuf_group*  groups;            /**< The groups. */
136  rtems_id            read_ahead_task;   /**< Read-ahead task */
137  rtems_chain_control read_ahead_chain;  /**< Read-ahead request chain */
138  bool                read_ahead_enabled; /**< Read-ahead enabled */
139
140  bool                initialised;       /**< Initialised state. */
141} rtems_bdbuf_cache;
142
143/**
144 * Fatal errors
145 */
146#define RTEMS_BLKDEV_FATAL_ERROR(n) \
147  (((uint32_t)'B' << 24) | ((uint32_t)(n) & (uint32_t)0x00FFFFFF))
148
149#define RTEMS_BLKDEV_FATAL_BDBUF_STATE_11      RTEMS_BLKDEV_FATAL_ERROR(1)
150#define RTEMS_BLKDEV_FATAL_BDBUF_STATE_4       RTEMS_BLKDEV_FATAL_ERROR(2)
151#define RTEMS_BLKDEV_FATAL_BDBUF_STATE_5       RTEMS_BLKDEV_FATAL_ERROR(3)
152#define RTEMS_BLKDEV_FATAL_BDBUF_STATE_6       RTEMS_BLKDEV_FATAL_ERROR(4)
153#define RTEMS_BLKDEV_FATAL_BDBUF_STATE_7       RTEMS_BLKDEV_FATAL_ERROR(5)
154#define RTEMS_BLKDEV_FATAL_BDBUF_STATE_8       RTEMS_BLKDEV_FATAL_ERROR(6)
155#define RTEMS_BLKDEV_FATAL_BDBUF_STATE_9       RTEMS_BLKDEV_FATAL_ERROR(7)
156#define RTEMS_BLKDEV_FATAL_BDBUF_STATE_10      RTEMS_BLKDEV_FATAL_ERROR(8)
157#define RTEMS_BLKDEV_FATAL_BDBUF_TREE_RM       RTEMS_BLKDEV_FATAL_ERROR(9)
158#define RTEMS_BLKDEV_FATAL_BDBUF_SWAPOUT       RTEMS_BLKDEV_FATAL_ERROR(10)
159
160/*
161 * The lock/unlock fatal errors occur in case the bdbuf is not initialized with
162 * rtems_bdbuf_init().  General system corruption like stack overflow etc. may
163 * also trigger these fatal errors.
164 */
165#define RTEMS_BLKDEV_FATAL_BDBUF_SYNC_LOCK     RTEMS_BLKDEV_FATAL_ERROR(11)
166#define RTEMS_BLKDEV_FATAL_BDBUF_SYNC_UNLOCK   RTEMS_BLKDEV_FATAL_ERROR(12)
167#define RTEMS_BLKDEV_FATAL_BDBUF_CACHE_LOCK    RTEMS_BLKDEV_FATAL_ERROR(13)
168#define RTEMS_BLKDEV_FATAL_BDBUF_CACHE_UNLOCK  RTEMS_BLKDEV_FATAL_ERROR(14)
169
170#define RTEMS_BLKDEV_FATAL_BDBUF_PREEMPT_DIS   RTEMS_BLKDEV_FATAL_ERROR(15)
171#define RTEMS_BLKDEV_FATAL_BDBUF_CACHE_WAIT_2  RTEMS_BLKDEV_FATAL_ERROR(16)
172#define RTEMS_BLKDEV_FATAL_BDBUF_PREEMPT_RST   RTEMS_BLKDEV_FATAL_ERROR(17)
173#define RTEMS_BLKDEV_FATAL_BDBUF_CACHE_WAIT_TO RTEMS_BLKDEV_FATAL_ERROR(18)
174#define RTEMS_BLKDEV_FATAL_BDBUF_CACHE_WAKE    RTEMS_BLKDEV_FATAL_ERROR(19)
175#define RTEMS_BLKDEV_FATAL_BDBUF_SO_WAKE       RTEMS_BLKDEV_FATAL_ERROR(20)
176#define RTEMS_BLKDEV_FATAL_BDBUF_SO_NOMEM      RTEMS_BLKDEV_FATAL_ERROR(21)
177#define RTEMS_BLKDEV_FATAL_BDBUF_SO_WK_CREATE  RTEMS_BLKDEV_FATAL_ERROR(22)
178#define BLKDEV_FATAL_BDBUF_SWAPOUT_RE          RTEMS_BLKDEV_FATAL_ERROR(24)
179#define BLKDEV_FATAL_BDBUF_SWAPOUT_TS          RTEMS_BLKDEV_FATAL_ERROR(25)
180#define RTEMS_BLKDEV_FATAL_BDBUF_WAIT_EVNT     RTEMS_BLKDEV_FATAL_ERROR(26)
181#define RTEMS_BLKDEV_FATAL_BDBUF_RECYCLE       RTEMS_BLKDEV_FATAL_ERROR(27)
182#define RTEMS_BLKDEV_FATAL_BDBUF_STATE_0       RTEMS_BLKDEV_FATAL_ERROR(28)
183#define RTEMS_BLKDEV_FATAL_BDBUF_STATE_1       RTEMS_BLKDEV_FATAL_ERROR(29)
184#define RTEMS_BLKDEV_FATAL_BDBUF_STATE_2       RTEMS_BLKDEV_FATAL_ERROR(30)
185#define RTEMS_BLKDEV_FATAL_BDBUF_RA_WAKE_UP    RTEMS_BLKDEV_FATAL_ERROR(31)
186
187/**
188 * The events used in this code. These should be system events rather than
189 * application events.
190 */
191#define RTEMS_BDBUF_TRANSFER_SYNC  RTEMS_EVENT_1
192#define RTEMS_BDBUF_SWAPOUT_SYNC   RTEMS_EVENT_2
193#define RTEMS_BDBUF_READ_AHEAD_WAKE_UP RTEMS_EVENT_1
194
195/**
196 * Lock semaphore attributes. This is used for locking type mutexes.
197 *
198 * @warning Priority inheritance is on.
199 */
200#define RTEMS_BDBUF_CACHE_LOCK_ATTRIBS \
201  (RTEMS_PRIORITY | RTEMS_BINARY_SEMAPHORE | \
202   RTEMS_INHERIT_PRIORITY | RTEMS_NO_PRIORITY_CEILING | RTEMS_LOCAL)
203
204/**
205 * Waiter semaphore attributes.
206 *
207 * @warning Do not configure as inherit priority. If a driver is in the driver
208 *          initialisation table this locked semaphore will have the IDLE task
209 *          as the holder and a blocking task will raise the priority of the
210 *          IDLE task which can cause unsual side effects.
211 */
212#define RTEMS_BDBUF_CACHE_WAITER_ATTRIBS \
213  (RTEMS_PRIORITY | RTEMS_SIMPLE_BINARY_SEMAPHORE | \
214   RTEMS_NO_INHERIT_PRIORITY | RTEMS_NO_PRIORITY_CEILING | RTEMS_LOCAL)
215
216/**
217 * Waiter timeout. Set to non-zero to find some info on a waiter that is
218 * waiting too long.
219 */
220#define RTEMS_BDBUF_WAIT_TIMEOUT RTEMS_NO_TIMEOUT
221#if !defined (RTEMS_BDBUF_WAIT_TIMEOUT)
222#define RTEMS_BDBUF_WAIT_TIMEOUT \
223  (TOD_MICROSECONDS_TO_TICKS (20000000))
224#endif
225
226static rtems_task rtems_bdbuf_swapout_task(rtems_task_argument arg);
227
228static rtems_task rtems_bdbuf_read_ahead_task(rtems_task_argument arg);
229
230/**
231 * The Buffer Descriptor cache.
232 */
233static rtems_bdbuf_cache bdbuf_cache;
234
235#if RTEMS_BDBUF_TRACE
236/**
237 * If true output the trace message.
238 */
239bool rtems_bdbuf_tracer;
240
241/**
242 * Return the number of items on the list.
243 *
244 * @param list The chain control.
245 * @return uint32_t The number of items on the list.
246 */
247uint32_t
248rtems_bdbuf_list_count (rtems_chain_control* list)
249{
250  rtems_chain_node* node = rtems_chain_first (list);
251  uint32_t          count = 0;
252  while (!rtems_chain_is_tail (list, node))
253  {
254    count++;
255    node = rtems_chain_next (node);
256  }
257  return count;
258}
259
260/**
261 * Show the usage for the bdbuf cache.
262 */
263void
264rtems_bdbuf_show_usage (void)
265{
266  uint32_t group;
267  uint32_t total = 0;
268  uint32_t val;
269
270  for (group = 0; group < bdbuf_cache.group_count; group++)
271    total += bdbuf_cache.groups[group].users;
272  printf ("bdbuf:group users=%lu", total);
273  val = rtems_bdbuf_list_count (&bdbuf_cache.lru);
274  printf (", lru=%lu", val);
275  total = val;
276  val = rtems_bdbuf_list_count (&bdbuf_cache.modified);
277  printf (", mod=%lu", val);
278  total += val;
279  val = rtems_bdbuf_list_count (&bdbuf_cache.sync);
280  printf (", sync=%lu", val);
281  total += val;
282  printf (", total=%lu\n", total);
283}
284
285/**
286 * Show the users for a group of a bd.
287 *
288 * @param where A label to show the context of output.
289 * @param bd The bd to show the users of.
290 */
291void
292rtems_bdbuf_show_users (const char* where, rtems_bdbuf_buffer* bd)
293{
294  const char* states[] =
295    { "FR", "EM", "CH", "AC", "AM", "AE", "AP", "MD", "SY", "TR", "TP" };
296
297  printf ("bdbuf:users: %15s: [%" PRIu32 " (%s)] %td:%td = %" PRIu32 " %s\n",
298          where,
299          bd->block, states[bd->state],
300          bd->group - bdbuf_cache.groups,
301          bd - bdbuf_cache.bds,
302          bd->group->users,
303          bd->group->users > 8 ? "<<<<<<<" : "");
304}
305#else
306#define rtems_bdbuf_tracer (0)
307#define rtems_bdbuf_show_usage() ((void) 0)
308#define rtems_bdbuf_show_users(_w, _b) ((void) 0)
309#endif
310
311/**
312 * The default maximum height of 32 allows for AVL trees having between
313 * 5,704,880 and 4,294,967,295 nodes, depending on order of insertion.  You may
314 * change this compile-time constant as you wish.
315 */
316#ifndef RTEMS_BDBUF_AVL_MAX_HEIGHT
317#define RTEMS_BDBUF_AVL_MAX_HEIGHT (32)
318#endif
319
320static void
321rtems_bdbuf_fatal (rtems_bdbuf_buf_state state, uint32_t error)
322{
323  rtems_fatal_error_occurred ((((uint32_t) state) << 16) | error);
324}
325
326/**
327 * Searches for the node with specified dd/block.
328 *
329 * @param root pointer to the root node of the AVL-Tree
330 * @param dd disk device search key
331 * @param block block search key
332 * @retval NULL node with the specified dd/block is not found
333 * @return pointer to the node with specified dd/block
334 */
335static rtems_bdbuf_buffer *
336rtems_bdbuf_avl_search (rtems_bdbuf_buffer** root,
337                        const rtems_disk_device *dd,
338                        rtems_blkdev_bnum    block)
339{
340  rtems_bdbuf_buffer* p = *root;
341
342  while ((p != NULL) && ((p->dd != dd) || (p->block != block)))
343  {
344    if (((uintptr_t) p->dd < (uintptr_t) dd)
345        || ((p->dd == dd) && (p->block < block)))
346    {
347      p = p->avl.right;
348    }
349    else
350    {
351      p = p->avl.left;
352    }
353  }
354
355  return p;
356}
357
358/**
359 * Inserts the specified node to the AVl-Tree.
360 *
361 * @param root pointer to the root node of the AVL-Tree
362 * @param node Pointer to the node to add.
363 * @retval 0 The node added successfully
364 * @retval -1 An error occured
365 */
366static int
367rtems_bdbuf_avl_insert(rtems_bdbuf_buffer** root,
368                       rtems_bdbuf_buffer*  node)
369{
370  const rtems_disk_device *dd = node->dd;
371  rtems_blkdev_bnum block = node->block;
372
373  rtems_bdbuf_buffer*  p = *root;
374  rtems_bdbuf_buffer*  q;
375  rtems_bdbuf_buffer*  p1;
376  rtems_bdbuf_buffer*  p2;
377  rtems_bdbuf_buffer*  buf_stack[RTEMS_BDBUF_AVL_MAX_HEIGHT];
378  rtems_bdbuf_buffer** buf_prev = buf_stack;
379
380  bool modified = false;
381
382  if (p == NULL)
383  {
384    *root = node;
385    node->avl.left = NULL;
386    node->avl.right = NULL;
387    node->avl.bal = 0;
388    return 0;
389  }
390
391  while (p != NULL)
392  {
393    *buf_prev++ = p;
394
395    if (((uintptr_t) p->dd < (uintptr_t) dd)
396        || ((p->dd == dd) && (p->block < block)))
397    {
398      p->avl.cache = 1;
399      q = p->avl.right;
400      if (q == NULL)
401      {
402        q = node;
403        p->avl.right = q = node;
404        break;
405      }
406    }
407    else if ((p->dd != dd) || (p->block != block))
408    {
409      p->avl.cache = -1;
410      q = p->avl.left;
411      if (q == NULL)
412      {
413        q = node;
414        p->avl.left = q;
415        break;
416      }
417    }
418    else
419    {
420      return -1;
421    }
422
423    p = q;
424  }
425
426  q->avl.left = q->avl.right = NULL;
427  q->avl.bal = 0;
428  modified = true;
429  buf_prev--;
430
431  while (modified)
432  {
433    if (p->avl.cache == -1)
434    {
435      switch (p->avl.bal)
436      {
437        case 1:
438          p->avl.bal = 0;
439          modified = false;
440          break;
441
442        case 0:
443          p->avl.bal = -1;
444          break;
445
446        case -1:
447          p1 = p->avl.left;
448          if (p1->avl.bal == -1) /* simple LL-turn */
449          {
450            p->avl.left = p1->avl.right;
451            p1->avl.right = p;
452            p->avl.bal = 0;
453            p = p1;
454          }
455          else /* double LR-turn */
456          {
457            p2 = p1->avl.right;
458            p1->avl.right = p2->avl.left;
459            p2->avl.left = p1;
460            p->avl.left = p2->avl.right;
461            p2->avl.right = p;
462            if (p2->avl.bal == -1) p->avl.bal = +1; else p->avl.bal = 0;
463            if (p2->avl.bal == +1) p1->avl.bal = -1; else p1->avl.bal = 0;
464            p = p2;
465          }
466          p->avl.bal = 0;
467          modified = false;
468          break;
469
470        default:
471          break;
472      }
473    }
474    else
475    {
476      switch (p->avl.bal)
477      {
478        case -1:
479          p->avl.bal = 0;
480          modified = false;
481          break;
482
483        case 0:
484          p->avl.bal = 1;
485          break;
486
487        case 1:
488          p1 = p->avl.right;
489          if (p1->avl.bal == 1) /* simple RR-turn */
490          {
491            p->avl.right = p1->avl.left;
492            p1->avl.left = p;
493            p->avl.bal = 0;
494            p = p1;
495          }
496          else /* double RL-turn */
497          {
498            p2 = p1->avl.left;
499            p1->avl.left = p2->avl.right;
500            p2->avl.right = p1;
501            p->avl.right = p2->avl.left;
502            p2->avl.left = p;
503            if (p2->avl.bal == +1) p->avl.bal = -1; else p->avl.bal = 0;
504            if (p2->avl.bal == -1) p1->avl.bal = +1; else p1->avl.bal = 0;
505            p = p2;
506          }
507          p->avl.bal = 0;
508          modified = false;
509          break;
510
511        default:
512          break;
513      }
514    }
515    q = p;
516    if (buf_prev > buf_stack)
517    {
518      p = *--buf_prev;
519
520      if (p->avl.cache == -1)
521      {
522        p->avl.left = q;
523      }
524      else
525      {
526        p->avl.right = q;
527      }
528    }
529    else
530    {
531      *root = p;
532      break;
533    }
534  };
535
536  return 0;
537}
538
539
540/**
541 * Removes the node from the tree.
542 *
543 * @param root Pointer to pointer to the root node
544 * @param node Pointer to the node to remove
545 * @retval 0 Item removed
546 * @retval -1 No such item found
547 */
548static int
549rtems_bdbuf_avl_remove(rtems_bdbuf_buffer**      root,
550                       const rtems_bdbuf_buffer* node)
551{
552  const rtems_disk_device *dd = node->dd;
553  rtems_blkdev_bnum block = node->block;
554
555  rtems_bdbuf_buffer*  p = *root;
556  rtems_bdbuf_buffer*  q;
557  rtems_bdbuf_buffer*  r;
558  rtems_bdbuf_buffer*  s;
559  rtems_bdbuf_buffer*  p1;
560  rtems_bdbuf_buffer*  p2;
561  rtems_bdbuf_buffer*  buf_stack[RTEMS_BDBUF_AVL_MAX_HEIGHT];
562  rtems_bdbuf_buffer** buf_prev = buf_stack;
563
564  bool modified = false;
565
566  memset (buf_stack, 0, sizeof(buf_stack));
567
568  while (p != NULL)
569  {
570    *buf_prev++ = p;
571
572    if (((uintptr_t) p->dd < (uintptr_t) dd)
573        || ((p->dd == dd) && (p->block < block)))
574    {
575      p->avl.cache = 1;
576      p = p->avl.right;
577    }
578    else if ((p->dd != dd) || (p->block != block))
579    {
580      p->avl.cache = -1;
581      p = p->avl.left;
582    }
583    else
584    {
585      /* node found */
586      break;
587    }
588  }
589
590  if (p == NULL)
591  {
592    /* there is no such node */
593    return -1;
594  }
595
596  q = p;
597
598  buf_prev--;
599  if (buf_prev > buf_stack)
600  {
601    p = *(buf_prev - 1);
602  }
603  else
604  {
605    p = NULL;
606  }
607
608  /* at this moment q - is a node to delete, p is q's parent */
609  if (q->avl.right == NULL)
610  {
611    r = q->avl.left;
612    if (r != NULL)
613    {
614      r->avl.bal = 0;
615    }
616    q = r;
617  }
618  else
619  {
620    rtems_bdbuf_buffer **t;
621
622    r = q->avl.right;
623
624    if (r->avl.left == NULL)
625    {
626      r->avl.left = q->avl.left;
627      r->avl.bal = q->avl.bal;
628      r->avl.cache = 1;
629      *buf_prev++ = q = r;
630    }
631    else
632    {
633      t = buf_prev++;
634      s = r;
635
636      while (s->avl.left != NULL)
637      {
638        *buf_prev++ = r = s;
639        s = r->avl.left;
640        r->avl.cache = -1;
641      }
642
643      s->avl.left = q->avl.left;
644      r->avl.left = s->avl.right;
645      s->avl.right = q->avl.right;
646      s->avl.bal = q->avl.bal;
647      s->avl.cache = 1;
648
649      *t = q = s;
650    }
651  }
652
653  if (p != NULL)
654  {
655    if (p->avl.cache == -1)
656    {
657      p->avl.left = q;
658    }
659    else
660    {
661      p->avl.right = q;
662    }
663  }
664  else
665  {
666    *root = q;
667  }
668
669  modified = true;
670
671  while (modified)
672  {
673    if (buf_prev > buf_stack)
674    {
675      p = *--buf_prev;
676    }
677    else
678    {
679      break;
680    }
681
682    if (p->avl.cache == -1)
683    {
684      /* rebalance left branch */
685      switch (p->avl.bal)
686      {
687        case -1:
688          p->avl.bal = 0;
689          break;
690        case  0:
691          p->avl.bal = 1;
692          modified = false;
693          break;
694
695        case +1:
696          p1 = p->avl.right;
697
698          if (p1->avl.bal >= 0) /* simple RR-turn */
699          {
700            p->avl.right = p1->avl.left;
701            p1->avl.left = p;
702
703            if (p1->avl.bal == 0)
704            {
705              p1->avl.bal = -1;
706              modified = false;
707            }
708            else
709            {
710              p->avl.bal = 0;
711              p1->avl.bal = 0;
712            }
713            p = p1;
714          }
715          else /* double RL-turn */
716          {
717            p2 = p1->avl.left;
718
719            p1->avl.left = p2->avl.right;
720            p2->avl.right = p1;
721            p->avl.right = p2->avl.left;
722            p2->avl.left = p;
723
724            if (p2->avl.bal == +1) p->avl.bal = -1; else p->avl.bal = 0;
725            if (p2->avl.bal == -1) p1->avl.bal = 1; else p1->avl.bal = 0;
726
727            p = p2;
728            p2->avl.bal = 0;
729          }
730          break;
731
732        default:
733          break;
734      }
735    }
736    else
737    {
738      /* rebalance right branch */
739      switch (p->avl.bal)
740      {
741        case +1:
742          p->avl.bal = 0;
743          break;
744
745        case  0:
746          p->avl.bal = -1;
747          modified = false;
748          break;
749
750        case -1:
751          p1 = p->avl.left;
752
753          if (p1->avl.bal <= 0) /* simple LL-turn */
754          {
755            p->avl.left = p1->avl.right;
756            p1->avl.right = p;
757            if (p1->avl.bal == 0)
758            {
759              p1->avl.bal = 1;
760              modified = false;
761            }
762            else
763            {
764              p->avl.bal = 0;
765              p1->avl.bal = 0;
766            }
767            p = p1;
768          }
769          else /* double LR-turn */
770          {
771            p2 = p1->avl.right;
772
773            p1->avl.right = p2->avl.left;
774            p2->avl.left = p1;
775            p->avl.left = p2->avl.right;
776            p2->avl.right = p;
777
778            if (p2->avl.bal == -1) p->avl.bal = 1; else p->avl.bal = 0;
779            if (p2->avl.bal == +1) p1->avl.bal = -1; else p1->avl.bal = 0;
780
781            p = p2;
782            p2->avl.bal = 0;
783          }
784          break;
785
786        default:
787          break;
788      }
789    }
790
791    if (buf_prev > buf_stack)
792    {
793      q = *(buf_prev - 1);
794
795      if (q->avl.cache == -1)
796      {
797        q->avl.left = p;
798      }
799      else
800      {
801        q->avl.right = p;
802      }
803    }
804    else
805    {
806      *root = p;
807      break;
808    }
809
810  }
811
812  return 0;
813}
814
815static void
816rtems_bdbuf_set_state (rtems_bdbuf_buffer *bd, rtems_bdbuf_buf_state state)
817{
818  bd->state = state;
819}
820
821static rtems_blkdev_bnum
822rtems_bdbuf_media_block (const rtems_disk_device *dd, rtems_blkdev_bnum block)
823{
824  if (dd->block_to_media_block_shift >= 0)
825    return block << dd->block_to_media_block_shift;
826  else
827    /*
828     * Change the block number for the block size to the block number for the media
829     * block size. We have to use 64bit maths. There is no short cut here.
830     */
831    return (rtems_blkdev_bnum)
832      ((((uint64_t) block) * dd->block_size) / dd->media_block_size);
833}
834
835/**
836 * Lock the mutex. A single task can nest calls.
837 *
838 * @param lock The mutex to lock.
839 * @param fatal_error_code The error code if the call fails.
840 */
841static void
842rtems_bdbuf_lock (rtems_id lock, uint32_t fatal_error_code)
843{
844  rtems_status_code sc = rtems_semaphore_obtain (lock,
845                                                 RTEMS_WAIT,
846                                                 RTEMS_NO_TIMEOUT);
847  if (sc != RTEMS_SUCCESSFUL)
848    rtems_fatal_error_occurred (fatal_error_code);
849}
850
851/**
852 * Unlock the mutex.
853 *
854 * @param lock The mutex to unlock.
855 * @param fatal_error_code The error code if the call fails.
856 */
857static void
858rtems_bdbuf_unlock (rtems_id lock, uint32_t fatal_error_code)
859{
860  rtems_status_code sc = rtems_semaphore_release (lock);
861  if (sc != RTEMS_SUCCESSFUL)
862    rtems_fatal_error_occurred (fatal_error_code);
863}
864
865/**
866 * Lock the cache. A single task can nest calls.
867 */
868static void
869rtems_bdbuf_lock_cache (void)
870{
871  rtems_bdbuf_lock (bdbuf_cache.lock, RTEMS_BLKDEV_FATAL_BDBUF_CACHE_LOCK);
872}
873
874/**
875 * Unlock the cache.
876 */
877static void
878rtems_bdbuf_unlock_cache (void)
879{
880  rtems_bdbuf_unlock (bdbuf_cache.lock, RTEMS_BLKDEV_FATAL_BDBUF_CACHE_UNLOCK);
881}
882
883/**
884 * Lock the cache's sync. A single task can nest calls.
885 */
886static void
887rtems_bdbuf_lock_sync (void)
888{
889  rtems_bdbuf_lock (bdbuf_cache.sync_lock, RTEMS_BLKDEV_FATAL_BDBUF_SYNC_LOCK);
890}
891
892/**
893 * Unlock the cache's sync lock. Any blocked writers are woken.
894 */
895static void
896rtems_bdbuf_unlock_sync (void)
897{
898  rtems_bdbuf_unlock (bdbuf_cache.sync_lock,
899                      RTEMS_BLKDEV_FATAL_BDBUF_SYNC_UNLOCK);
900}
901
902static void
903rtems_bdbuf_group_obtain (rtems_bdbuf_buffer *bd)
904{
905  ++bd->group->users;
906}
907
908static void
909rtems_bdbuf_group_release (rtems_bdbuf_buffer *bd)
910{
911  --bd->group->users;
912}
913
914static rtems_mode
915rtems_bdbuf_disable_preemption (void)
916{
917  rtems_status_code sc = RTEMS_SUCCESSFUL;
918  rtems_mode prev_mode = 0;
919
920  sc = rtems_task_mode (RTEMS_NO_PREEMPT, RTEMS_PREEMPT_MASK, &prev_mode);
921  if (sc != RTEMS_SUCCESSFUL)
922    rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_PREEMPT_DIS);
923
924  return prev_mode;
925}
926
927static void
928rtems_bdbuf_restore_preemption (rtems_mode prev_mode)
929{
930  rtems_status_code sc = RTEMS_SUCCESSFUL;
931
932  sc = rtems_task_mode (prev_mode, RTEMS_ALL_MODE_MASKS, &prev_mode);
933  if (sc != RTEMS_SUCCESSFUL)
934    rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_PREEMPT_RST);
935}
936
937/**
938 * Wait until woken. Semaphores are used so a number of tasks can wait and can
939 * be woken at once. Task events would require we maintain a list of tasks to
940 * be woken and this would require storage and we do not know the number of
941 * tasks that could be waiting.
942 *
943 * While we have the cache locked we can try and claim the semaphore and
944 * therefore know when we release the lock to the cache we will block until the
945 * semaphore is released. This may even happen before we get to block.
946 *
947 * A counter is used to save the release call when no one is waiting.
948 *
949 * The function assumes the cache is locked on entry and it will be locked on
950 * exit.
951 */
952static void
953rtems_bdbuf_anonymous_wait (rtems_bdbuf_waiters *waiters)
954{
955  rtems_status_code sc;
956  rtems_mode        prev_mode;
957
958  /*
959   * Indicate we are waiting.
960   */
961  ++waiters->count;
962
963  /*
964   * Disable preemption then unlock the cache and block.  There is no POSIX
965   * condition variable in the core API so this is a work around.
966   *
967   * The issue is a task could preempt after the cache is unlocked because it is
968   * blocking or just hits that window, and before this task has blocked on the
969   * semaphore. If the preempting task flushes the queue this task will not see
970   * the flush and may block for ever or until another transaction flushes this
971   * semaphore.
972   */
973  prev_mode = rtems_bdbuf_disable_preemption ();
974
975  /*
976   * Unlock the cache, wait, and lock the cache when we return.
977   */
978  rtems_bdbuf_unlock_cache ();
979
980  sc = rtems_semaphore_obtain (waiters->sema, RTEMS_WAIT, RTEMS_BDBUF_WAIT_TIMEOUT);
981
982  if (sc == RTEMS_TIMEOUT)
983    rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_CACHE_WAIT_TO);
984
985  if (sc != RTEMS_UNSATISFIED)
986    rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_CACHE_WAIT_2);
987
988  rtems_bdbuf_lock_cache ();
989
990  rtems_bdbuf_restore_preemption (prev_mode);
991
992  --waiters->count;
993}
994
995static void
996rtems_bdbuf_wait (rtems_bdbuf_buffer *bd, rtems_bdbuf_waiters *waiters)
997{
998  rtems_bdbuf_group_obtain (bd);
999  ++bd->waiters;
1000  rtems_bdbuf_anonymous_wait (waiters);
1001  --bd->waiters;
1002  rtems_bdbuf_group_release (bd);
1003}
1004
1005/**
1006 * Wake a blocked resource. The resource has a counter that lets us know if
1007 * there are any waiters.
1008 */
1009static void
1010rtems_bdbuf_wake (const rtems_bdbuf_waiters *waiters)
1011{
1012  rtems_status_code sc = RTEMS_SUCCESSFUL;
1013
1014  if (waiters->count > 0)
1015  {
1016    sc = rtems_semaphore_flush (waiters->sema);
1017    if (sc != RTEMS_SUCCESSFUL)
1018      rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_CACHE_WAKE);
1019  }
1020}
1021
1022static void
1023rtems_bdbuf_wake_swapper (void)
1024{
1025  rtems_status_code sc = rtems_event_send (bdbuf_cache.swapout,
1026                                           RTEMS_BDBUF_SWAPOUT_SYNC);
1027  if (sc != RTEMS_SUCCESSFUL)
1028    rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_SO_WAKE);
1029}
1030
1031static bool
1032rtems_bdbuf_has_buffer_waiters (void)
1033{
1034  return bdbuf_cache.buffer_waiters.count;
1035}
1036
1037static void
1038rtems_bdbuf_remove_from_tree (rtems_bdbuf_buffer *bd)
1039{
1040  if (rtems_bdbuf_avl_remove (&bdbuf_cache.tree, bd) != 0)
1041    rtems_bdbuf_fatal (bd->state, RTEMS_BLKDEV_FATAL_BDBUF_TREE_RM);
1042}
1043
1044static void
1045rtems_bdbuf_remove_from_tree_and_lru_list (rtems_bdbuf_buffer *bd)
1046{
1047  switch (bd->state)
1048  {
1049    case RTEMS_BDBUF_STATE_FREE:
1050      break;
1051    case RTEMS_BDBUF_STATE_CACHED:
1052      rtems_bdbuf_remove_from_tree (bd);
1053      break;
1054    default:
1055      rtems_bdbuf_fatal (bd->state, RTEMS_BLKDEV_FATAL_BDBUF_STATE_10);
1056  }
1057
1058  rtems_chain_extract_unprotected (&bd->link);
1059}
1060
1061static void
1062rtems_bdbuf_make_free_and_add_to_lru_list (rtems_bdbuf_buffer *bd)
1063{
1064  rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_FREE);
1065  rtems_chain_prepend_unprotected (&bdbuf_cache.lru, &bd->link);
1066}
1067
1068static void
1069rtems_bdbuf_make_empty (rtems_bdbuf_buffer *bd)
1070{
1071  rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_EMPTY);
1072}
1073
1074static void
1075rtems_bdbuf_make_cached_and_add_to_lru_list (rtems_bdbuf_buffer *bd)
1076{
1077  rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_CACHED);
1078  rtems_chain_append_unprotected (&bdbuf_cache.lru, &bd->link);
1079}
1080
1081static void
1082rtems_bdbuf_discard_buffer (rtems_bdbuf_buffer *bd)
1083{
1084  rtems_bdbuf_make_empty (bd);
1085
1086  if (bd->waiters == 0)
1087  {
1088    rtems_bdbuf_remove_from_tree (bd);
1089    rtems_bdbuf_make_free_and_add_to_lru_list (bd);
1090  }
1091}
1092
1093static void
1094rtems_bdbuf_add_to_modified_list_after_access (rtems_bdbuf_buffer *bd)
1095{
1096  if (bdbuf_cache.sync_active && bdbuf_cache.sync_device == bd->dd)
1097  {
1098    rtems_bdbuf_unlock_cache ();
1099
1100    /*
1101     * Wait for the sync lock.
1102     */
1103    rtems_bdbuf_lock_sync ();
1104
1105    rtems_bdbuf_unlock_sync ();
1106    rtems_bdbuf_lock_cache ();
1107  }
1108
1109  /*
1110   * Only the first modified release sets the timer and any further user
1111   * accesses do not change the timer value which should move down. This
1112   * assumes the user's hold of the buffer is much less than the time on the
1113   * modified list. Resetting the timer on each access which could result in a
1114   * buffer never getting to 0 and never being forced onto disk. This raises a
1115   * difficult question. Is a snapshot of a block that is changing better than
1116   * nothing being written? We have tended to think we should hold changes for
1117   * only a specific period of time even if still changing and get onto disk
1118   * and letting the file system try and recover this position if it can.
1119   */
1120  if (bd->state == RTEMS_BDBUF_STATE_ACCESS_CACHED
1121        || bd->state == RTEMS_BDBUF_STATE_ACCESS_EMPTY)
1122    bd->hold_timer = bdbuf_config.swap_block_hold;
1123
1124  rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_MODIFIED);
1125  rtems_chain_append_unprotected (&bdbuf_cache.modified, &bd->link);
1126
1127  if (bd->waiters)
1128    rtems_bdbuf_wake (&bdbuf_cache.access_waiters);
1129  else if (rtems_bdbuf_has_buffer_waiters ())
1130    rtems_bdbuf_wake_swapper ();
1131}
1132
1133static void
1134rtems_bdbuf_add_to_lru_list_after_access (rtems_bdbuf_buffer *bd)
1135{
1136  rtems_bdbuf_group_release (bd);
1137  rtems_bdbuf_make_cached_and_add_to_lru_list (bd);
1138
1139  if (bd->waiters)
1140    rtems_bdbuf_wake (&bdbuf_cache.access_waiters);
1141  else
1142    rtems_bdbuf_wake (&bdbuf_cache.buffer_waiters);
1143}
1144
1145/**
1146 * Compute the number of BDs per group for a given buffer size.
1147 *
1148 * @param size The buffer size. It can be any size and we scale up.
1149 */
1150static size_t
1151rtems_bdbuf_bds_per_group (size_t size)
1152{
1153  size_t bufs_per_size;
1154  size_t bds_per_size;
1155
1156  if (size > bdbuf_config.buffer_max)
1157    return 0;
1158
1159  bufs_per_size = ((size - 1) / bdbuf_config.buffer_min) + 1;
1160
1161  for (bds_per_size = 1;
1162       bds_per_size < bufs_per_size;
1163       bds_per_size <<= 1)
1164    ;
1165
1166  return bdbuf_cache.max_bds_per_group / bds_per_size;
1167}
1168
1169static void
1170rtems_bdbuf_discard_buffer_after_access (rtems_bdbuf_buffer *bd)
1171{
1172  rtems_bdbuf_group_release (bd);
1173  rtems_bdbuf_discard_buffer (bd);
1174
1175  if (bd->waiters)
1176    rtems_bdbuf_wake (&bdbuf_cache.access_waiters);
1177  else
1178    rtems_bdbuf_wake (&bdbuf_cache.buffer_waiters);
1179}
1180
1181/**
1182 * Reallocate a group. The BDs currently allocated in the group are removed
1183 * from the ALV tree and any lists then the new BD's are prepended to the ready
1184 * list of the cache.
1185 *
1186 * @param group The group to reallocate.
1187 * @param new_bds_per_group The new count of BDs per group.
1188 * @return A buffer of this group.
1189 */
1190static rtems_bdbuf_buffer *
1191rtems_bdbuf_group_realloc (rtems_bdbuf_group* group, size_t new_bds_per_group)
1192{
1193  rtems_bdbuf_buffer* bd;
1194  size_t              b;
1195  size_t              bufs_per_bd;
1196
1197  if (rtems_bdbuf_tracer)
1198    printf ("bdbuf:realloc: %tu: %zd -> %zd\n",
1199            group - bdbuf_cache.groups, group->bds_per_group,
1200            new_bds_per_group);
1201
1202  bufs_per_bd = bdbuf_cache.max_bds_per_group / group->bds_per_group;
1203
1204  for (b = 0, bd = group->bdbuf;
1205       b < group->bds_per_group;
1206       b++, bd += bufs_per_bd)
1207    rtems_bdbuf_remove_from_tree_and_lru_list (bd);
1208
1209  group->bds_per_group = new_bds_per_group;
1210  bufs_per_bd = bdbuf_cache.max_bds_per_group / new_bds_per_group;
1211
1212  for (b = 1, bd = group->bdbuf + bufs_per_bd;
1213       b < group->bds_per_group;
1214       b++, bd += bufs_per_bd)
1215    rtems_bdbuf_make_free_and_add_to_lru_list (bd);
1216
1217  if (b > 1)
1218    rtems_bdbuf_wake (&bdbuf_cache.buffer_waiters);
1219
1220  return group->bdbuf;
1221}
1222
1223static void
1224rtems_bdbuf_setup_empty_buffer (rtems_bdbuf_buffer *bd,
1225                                rtems_disk_device  *dd,
1226                                rtems_blkdev_bnum   block)
1227{
1228  bd->dd        = dd ;
1229  bd->block     = block;
1230  bd->avl.left  = NULL;
1231  bd->avl.right = NULL;
1232  bd->waiters   = 0;
1233
1234  if (rtems_bdbuf_avl_insert (&bdbuf_cache.tree, bd) != 0)
1235    rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_RECYCLE);
1236
1237  rtems_bdbuf_make_empty (bd);
1238}
1239
1240static rtems_bdbuf_buffer *
1241rtems_bdbuf_get_buffer_from_lru_list (rtems_disk_device *dd,
1242                                      rtems_blkdev_bnum  block)
1243{
1244  rtems_chain_node *node = rtems_chain_first (&bdbuf_cache.lru);
1245
1246  while (!rtems_chain_is_tail (&bdbuf_cache.lru, node))
1247  {
1248    rtems_bdbuf_buffer *bd = (rtems_bdbuf_buffer *) node;
1249    rtems_bdbuf_buffer *empty_bd = NULL;
1250
1251    if (rtems_bdbuf_tracer)
1252      printf ("bdbuf:next-bd: %tu (%td:%" PRId32 ") %zd -> %zd\n",
1253              bd - bdbuf_cache.bds,
1254              bd->group - bdbuf_cache.groups, bd->group->users,
1255              bd->group->bds_per_group, dd->bds_per_group);
1256
1257    /*
1258     * If nobody waits for this BD, we may recycle it.
1259     */
1260    if (bd->waiters == 0)
1261    {
1262      if (bd->group->bds_per_group == dd->bds_per_group)
1263      {
1264        rtems_bdbuf_remove_from_tree_and_lru_list (bd);
1265
1266        empty_bd = bd;
1267      }
1268      else if (bd->group->users == 0)
1269        empty_bd = rtems_bdbuf_group_realloc (bd->group, dd->bds_per_group);
1270    }
1271
1272    if (empty_bd != NULL)
1273    {
1274      rtems_bdbuf_setup_empty_buffer (empty_bd, dd, block);
1275
1276      return empty_bd;
1277    }
1278
1279    node = rtems_chain_next (node);
1280  }
1281
1282  return NULL;
1283}
1284
1285static rtems_status_code
1286rtems_bdbuf_create_task(
1287  rtems_name name,
1288  rtems_task_priority priority,
1289  rtems_task_priority default_priority,
1290  rtems_task_entry entry,
1291  rtems_task_argument arg,
1292  rtems_id *id
1293)
1294{
1295  rtems_status_code sc;
1296  size_t stack_size = bdbuf_config.task_stack_size ?
1297    bdbuf_config.task_stack_size : RTEMS_BDBUF_TASK_STACK_SIZE_DEFAULT;
1298
1299  priority = priority != 0 ? priority : default_priority;
1300
1301  sc = rtems_task_create (name,
1302                          priority,
1303                          stack_size,
1304                          RTEMS_PREEMPT | RTEMS_NO_TIMESLICE | RTEMS_NO_ASR,
1305                          RTEMS_LOCAL | RTEMS_NO_FLOATING_POINT,
1306                          id);
1307
1308  if (sc == RTEMS_SUCCESSFUL)
1309    sc = rtems_task_start (*id, entry, arg);
1310
1311  return sc;
1312}
1313
1314/**
1315 * Initialise the cache.
1316 *
1317 * @return rtems_status_code The initialisation status.
1318 */
1319rtems_status_code
1320rtems_bdbuf_init (void)
1321{
1322  rtems_bdbuf_group*  group;
1323  rtems_bdbuf_buffer* bd;
1324  uint8_t*            buffer;
1325  size_t              b;
1326  size_t              cache_aligment;
1327  rtems_status_code   sc;
1328  rtems_mode          prev_mode;
1329
1330  if (rtems_bdbuf_tracer)
1331    printf ("bdbuf:init\n");
1332
1333  if (rtems_interrupt_is_in_progress())
1334    return RTEMS_CALLED_FROM_ISR;
1335
1336  /*
1337   * Check the configuration table values.
1338   */
1339  if ((bdbuf_config.buffer_max % bdbuf_config.buffer_min) != 0)
1340    return RTEMS_INVALID_NUMBER;
1341
1342  /*
1343   * We use a special variable to manage the initialisation incase we have
1344   * completing threads doing this. You may get errors if the another thread
1345   * makes a call and we have not finished initialisation.
1346   */
1347  prev_mode = rtems_bdbuf_disable_preemption ();
1348  if (bdbuf_cache.initialised)
1349  {
1350    rtems_bdbuf_restore_preemption (prev_mode);
1351    return RTEMS_RESOURCE_IN_USE;
1352  }
1353
1354  memset(&bdbuf_cache, 0, sizeof(bdbuf_cache));
1355  bdbuf_cache.initialised = true;
1356  rtems_bdbuf_restore_preemption (prev_mode);
1357
1358  /*
1359   * For unspecified cache alignments we use the CPU alignment.
1360   */
1361  cache_aligment = 32; /* FIXME rtems_cache_get_data_line_size() */
1362  if (cache_aligment <= 0)
1363    cache_aligment = CPU_ALIGNMENT;
1364
1365  bdbuf_cache.sync_device = BDBUF_INVALID_DEV;
1366
1367  rtems_chain_initialize_empty (&bdbuf_cache.swapout_workers);
1368  rtems_chain_initialize_empty (&bdbuf_cache.lru);
1369  rtems_chain_initialize_empty (&bdbuf_cache.modified);
1370  rtems_chain_initialize_empty (&bdbuf_cache.sync);
1371  rtems_chain_initialize_empty (&bdbuf_cache.read_ahead_chain);
1372
1373  /*
1374   * Create the locks for the cache.
1375   */
1376  sc = rtems_semaphore_create (rtems_build_name ('B', 'D', 'C', 'l'),
1377                               1, RTEMS_BDBUF_CACHE_LOCK_ATTRIBS, 0,
1378                               &bdbuf_cache.lock);
1379  if (sc != RTEMS_SUCCESSFUL)
1380    goto error;
1381
1382  rtems_bdbuf_lock_cache ();
1383
1384  sc = rtems_semaphore_create (rtems_build_name ('B', 'D', 'C', 's'),
1385                               1, RTEMS_BDBUF_CACHE_LOCK_ATTRIBS, 0,
1386                               &bdbuf_cache.sync_lock);
1387  if (sc != RTEMS_SUCCESSFUL)
1388    goto error;
1389
1390  sc = rtems_semaphore_create (rtems_build_name ('B', 'D', 'C', 'a'),
1391                               0, RTEMS_BDBUF_CACHE_WAITER_ATTRIBS, 0,
1392                               &bdbuf_cache.access_waiters.sema);
1393  if (sc != RTEMS_SUCCESSFUL)
1394    goto error;
1395
1396  sc = rtems_semaphore_create (rtems_build_name ('B', 'D', 'C', 't'),
1397                               0, RTEMS_BDBUF_CACHE_WAITER_ATTRIBS, 0,
1398                               &bdbuf_cache.transfer_waiters.sema);
1399  if (sc != RTEMS_SUCCESSFUL)
1400    goto error;
1401
1402  sc = rtems_semaphore_create (rtems_build_name ('B', 'D', 'C', 'b'),
1403                               0, RTEMS_BDBUF_CACHE_WAITER_ATTRIBS, 0,
1404                               &bdbuf_cache.buffer_waiters.sema);
1405  if (sc != RTEMS_SUCCESSFUL)
1406    goto error;
1407
1408  /*
1409   * Compute the various number of elements in the cache.
1410   */
1411  bdbuf_cache.buffer_min_count =
1412    bdbuf_config.size / bdbuf_config.buffer_min;
1413  bdbuf_cache.max_bds_per_group =
1414    bdbuf_config.buffer_max / bdbuf_config.buffer_min;
1415  bdbuf_cache.group_count =
1416    bdbuf_cache.buffer_min_count / bdbuf_cache.max_bds_per_group;
1417
1418  /*
1419   * Allocate the memory for the buffer descriptors.
1420   */
1421  bdbuf_cache.bds = calloc (sizeof (rtems_bdbuf_buffer),
1422                            bdbuf_cache.buffer_min_count);
1423  if (!bdbuf_cache.bds)
1424    goto error;
1425
1426  /*
1427   * Allocate the memory for the buffer descriptors.
1428   */
1429  bdbuf_cache.groups = calloc (sizeof (rtems_bdbuf_group),
1430                               bdbuf_cache.group_count);
1431  if (!bdbuf_cache.groups)
1432    goto error;
1433
1434  /*
1435   * Allocate memory for buffer memory. The buffer memory will be cache
1436   * aligned. It is possible to free the memory allocated by rtems_memalign()
1437   * with free(). Return 0 if allocated.
1438   *
1439   * The memory allocate allows a
1440   */
1441  if (rtems_memalign ((void **) &bdbuf_cache.buffers,
1442                      cache_aligment,
1443                      bdbuf_cache.buffer_min_count * bdbuf_config.buffer_min) != 0)
1444    goto error;
1445
1446  /*
1447   * The cache is empty after opening so we need to add all the buffers to it
1448   * and initialise the groups.
1449   */
1450  for (b = 0, group = bdbuf_cache.groups,
1451         bd = bdbuf_cache.bds, buffer = bdbuf_cache.buffers;
1452       b < bdbuf_cache.buffer_min_count;
1453       b++, bd++, buffer += bdbuf_config.buffer_min)
1454  {
1455    bd->dd    = BDBUF_INVALID_DEV;
1456    bd->group  = group;
1457    bd->buffer = buffer;
1458
1459    rtems_chain_append_unprotected (&bdbuf_cache.lru, &bd->link);
1460
1461    if ((b % bdbuf_cache.max_bds_per_group) ==
1462        (bdbuf_cache.max_bds_per_group - 1))
1463      group++;
1464  }
1465
1466  for (b = 0,
1467         group = bdbuf_cache.groups,
1468         bd = bdbuf_cache.bds;
1469       b < bdbuf_cache.group_count;
1470       b++,
1471         group++,
1472         bd += bdbuf_cache.max_bds_per_group)
1473  {
1474    group->bds_per_group = bdbuf_cache.max_bds_per_group;
1475    group->bdbuf = bd;
1476  }
1477
1478  /*
1479   * Create and start swapout task. This task will create and manage the worker
1480   * threads.
1481   */
1482  bdbuf_cache.swapout_enabled = true;
1483
1484  sc = rtems_bdbuf_create_task (rtems_build_name('B', 'S', 'W', 'P'),
1485                                bdbuf_config.swapout_priority,
1486                                RTEMS_BDBUF_SWAPOUT_TASK_PRIORITY_DEFAULT,
1487                                rtems_bdbuf_swapout_task,
1488                                0,
1489                                &bdbuf_cache.swapout);
1490  if (sc != RTEMS_SUCCESSFUL)
1491    goto error;
1492
1493  if (bdbuf_config.max_read_ahead_blocks > 0)
1494  {
1495    bdbuf_cache.read_ahead_enabled = true;
1496    sc = rtems_bdbuf_create_task (rtems_build_name('B', 'R', 'D', 'A'),
1497                                  bdbuf_config.read_ahead_priority,
1498                                  RTEMS_BDBUF_READ_AHEAD_TASK_PRIORITY_DEFAULT,
1499                                  rtems_bdbuf_read_ahead_task,
1500                                  0,
1501                                  &bdbuf_cache.read_ahead_task);
1502    if (sc != RTEMS_SUCCESSFUL)
1503      goto error;
1504  }
1505
1506  rtems_bdbuf_unlock_cache ();
1507
1508  return RTEMS_SUCCESSFUL;
1509
1510error:
1511
1512  if (bdbuf_cache.read_ahead_task != 0)
1513    rtems_task_delete (bdbuf_cache.read_ahead_task);
1514
1515  if (bdbuf_cache.swapout != 0)
1516    rtems_task_delete (bdbuf_cache.swapout);
1517
1518  free (bdbuf_cache.buffers);
1519  free (bdbuf_cache.groups);
1520  free (bdbuf_cache.bds);
1521
1522  rtems_semaphore_delete (bdbuf_cache.buffer_waiters.sema);
1523  rtems_semaphore_delete (bdbuf_cache.access_waiters.sema);
1524  rtems_semaphore_delete (bdbuf_cache.transfer_waiters.sema);
1525  rtems_semaphore_delete (bdbuf_cache.sync_lock);
1526
1527  if (bdbuf_cache.lock != 0)
1528  {
1529    rtems_bdbuf_unlock_cache ();
1530    rtems_semaphore_delete (bdbuf_cache.lock);
1531  }
1532
1533  bdbuf_cache.initialised = false;
1534
1535  return RTEMS_UNSATISFIED;
1536}
1537
1538static void
1539rtems_bdbuf_wait_for_event (rtems_event_set event)
1540{
1541  rtems_status_code sc = RTEMS_SUCCESSFUL;
1542  rtems_event_set   out = 0;
1543
1544  sc = rtems_event_receive (event,
1545                            RTEMS_EVENT_ALL | RTEMS_WAIT,
1546                            RTEMS_NO_TIMEOUT,
1547                            &out);
1548
1549  if (sc != RTEMS_SUCCESSFUL || out != event)
1550    rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_WAIT_EVNT);
1551}
1552
1553static void
1554rtems_bdbuf_wait_for_access (rtems_bdbuf_buffer *bd)
1555{
1556  while (true)
1557  {
1558    switch (bd->state)
1559    {
1560      case RTEMS_BDBUF_STATE_MODIFIED:
1561        rtems_bdbuf_group_release (bd);
1562        /* Fall through */
1563      case RTEMS_BDBUF_STATE_CACHED:
1564        rtems_chain_extract_unprotected (&bd->link);
1565        /* Fall through */
1566      case RTEMS_BDBUF_STATE_EMPTY:
1567        return;
1568      case RTEMS_BDBUF_STATE_ACCESS_CACHED:
1569      case RTEMS_BDBUF_STATE_ACCESS_EMPTY:
1570      case RTEMS_BDBUF_STATE_ACCESS_MODIFIED:
1571      case RTEMS_BDBUF_STATE_ACCESS_PURGED:
1572        rtems_bdbuf_wait (bd, &bdbuf_cache.access_waiters);
1573        break;
1574      case RTEMS_BDBUF_STATE_SYNC:
1575      case RTEMS_BDBUF_STATE_TRANSFER:
1576      case RTEMS_BDBUF_STATE_TRANSFER_PURGED:
1577        rtems_bdbuf_wait (bd, &bdbuf_cache.transfer_waiters);
1578        break;
1579      default:
1580        rtems_bdbuf_fatal (bd->state, RTEMS_BLKDEV_FATAL_BDBUF_STATE_7);
1581    }
1582  }
1583}
1584
1585static void
1586rtems_bdbuf_request_sync_for_modified_buffer (rtems_bdbuf_buffer *bd)
1587{
1588  rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_SYNC);
1589  rtems_chain_extract_unprotected (&bd->link);
1590  rtems_chain_append_unprotected (&bdbuf_cache.sync, &bd->link);
1591  rtems_bdbuf_wake_swapper ();
1592}
1593
1594/**
1595 * @brief Waits until the buffer is ready for recycling.
1596 *
1597 * @retval @c true Buffer is valid and may be recycled.
1598 * @retval @c false Buffer is invalid and has to searched again.
1599 */
1600static bool
1601rtems_bdbuf_wait_for_recycle (rtems_bdbuf_buffer *bd)
1602{
1603  while (true)
1604  {
1605    switch (bd->state)
1606    {
1607      case RTEMS_BDBUF_STATE_FREE:
1608        return true;
1609      case RTEMS_BDBUF_STATE_MODIFIED:
1610        rtems_bdbuf_request_sync_for_modified_buffer (bd);
1611        break;
1612      case RTEMS_BDBUF_STATE_CACHED:
1613      case RTEMS_BDBUF_STATE_EMPTY:
1614        if (bd->waiters == 0)
1615          return true;
1616        else
1617        {
1618          /*
1619           * It is essential that we wait here without a special wait count and
1620           * without the group in use.  Otherwise we could trigger a wait ping
1621           * pong with another recycle waiter.  The state of the buffer is
1622           * arbitrary afterwards.
1623           */
1624          rtems_bdbuf_anonymous_wait (&bdbuf_cache.buffer_waiters);
1625          return false;
1626        }
1627      case RTEMS_BDBUF_STATE_ACCESS_CACHED:
1628      case RTEMS_BDBUF_STATE_ACCESS_EMPTY:
1629      case RTEMS_BDBUF_STATE_ACCESS_MODIFIED:
1630      case RTEMS_BDBUF_STATE_ACCESS_PURGED:
1631        rtems_bdbuf_wait (bd, &bdbuf_cache.access_waiters);
1632        break;
1633      case RTEMS_BDBUF_STATE_SYNC:
1634      case RTEMS_BDBUF_STATE_TRANSFER:
1635      case RTEMS_BDBUF_STATE_TRANSFER_PURGED:
1636        rtems_bdbuf_wait (bd, &bdbuf_cache.transfer_waiters);
1637        break;
1638      default:
1639        rtems_bdbuf_fatal (bd->state, RTEMS_BLKDEV_FATAL_BDBUF_STATE_8);
1640    }
1641  }
1642}
1643
1644static void
1645rtems_bdbuf_wait_for_sync_done (rtems_bdbuf_buffer *bd)
1646{
1647  while (true)
1648  {
1649    switch (bd->state)
1650    {
1651      case RTEMS_BDBUF_STATE_CACHED:
1652      case RTEMS_BDBUF_STATE_EMPTY:
1653      case RTEMS_BDBUF_STATE_MODIFIED:
1654      case RTEMS_BDBUF_STATE_ACCESS_CACHED:
1655      case RTEMS_BDBUF_STATE_ACCESS_EMPTY:
1656      case RTEMS_BDBUF_STATE_ACCESS_MODIFIED:
1657      case RTEMS_BDBUF_STATE_ACCESS_PURGED:
1658        return;
1659      case RTEMS_BDBUF_STATE_SYNC:
1660      case RTEMS_BDBUF_STATE_TRANSFER:
1661      case RTEMS_BDBUF_STATE_TRANSFER_PURGED:
1662        rtems_bdbuf_wait (bd, &bdbuf_cache.transfer_waiters);
1663        break;
1664      default:
1665        rtems_bdbuf_fatal (bd->state, RTEMS_BLKDEV_FATAL_BDBUF_STATE_9);
1666    }
1667  }
1668}
1669
1670static void
1671rtems_bdbuf_wait_for_buffer (void)
1672{
1673  if (!rtems_chain_is_empty (&bdbuf_cache.modified))
1674    rtems_bdbuf_wake_swapper ();
1675
1676  rtems_bdbuf_anonymous_wait (&bdbuf_cache.buffer_waiters);
1677}
1678
1679static void
1680rtems_bdbuf_sync_after_access (rtems_bdbuf_buffer *bd)
1681{
1682  rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_SYNC);
1683
1684  rtems_chain_append_unprotected (&bdbuf_cache.sync, &bd->link);
1685
1686  if (bd->waiters)
1687    rtems_bdbuf_wake (&bdbuf_cache.access_waiters);
1688
1689  rtems_bdbuf_wake_swapper ();
1690  rtems_bdbuf_wait_for_sync_done (bd);
1691
1692  /*
1693   * We may have created a cached or empty buffer which may be recycled.
1694   */
1695  if (bd->waiters == 0
1696        && (bd->state == RTEMS_BDBUF_STATE_CACHED
1697          || bd->state == RTEMS_BDBUF_STATE_EMPTY))
1698  {
1699    if (bd->state == RTEMS_BDBUF_STATE_EMPTY)
1700    {
1701      rtems_bdbuf_remove_from_tree (bd);
1702      rtems_bdbuf_make_free_and_add_to_lru_list (bd);
1703    }
1704    rtems_bdbuf_wake (&bdbuf_cache.buffer_waiters);
1705  }
1706}
1707
1708static rtems_bdbuf_buffer *
1709rtems_bdbuf_get_buffer_for_read_ahead (rtems_disk_device *dd,
1710                                       rtems_blkdev_bnum  block)
1711{
1712  rtems_bdbuf_buffer *bd = NULL;
1713
1714  bd = rtems_bdbuf_avl_search (&bdbuf_cache.tree, dd, block);
1715
1716  if (bd == NULL)
1717  {
1718    bd = rtems_bdbuf_get_buffer_from_lru_list (dd, block);
1719
1720    if (bd != NULL)
1721      rtems_bdbuf_group_obtain (bd);
1722  }
1723  else
1724    /*
1725     * The buffer is in the cache.  So it is already available or in use, and
1726     * thus no need for a read ahead.
1727     */
1728    bd = NULL;
1729
1730  return bd;
1731}
1732
1733static rtems_bdbuf_buffer *
1734rtems_bdbuf_get_buffer_for_access (rtems_disk_device *dd,
1735                                   rtems_blkdev_bnum  block)
1736{
1737  rtems_bdbuf_buffer *bd = NULL;
1738
1739  do
1740  {
1741    bd = rtems_bdbuf_avl_search (&bdbuf_cache.tree, dd, block);
1742
1743    if (bd != NULL)
1744    {
1745      if (bd->group->bds_per_group != dd->bds_per_group)
1746      {
1747        if (rtems_bdbuf_wait_for_recycle (bd))
1748        {
1749          rtems_bdbuf_remove_from_tree_and_lru_list (bd);
1750          rtems_bdbuf_make_free_and_add_to_lru_list (bd);
1751          rtems_bdbuf_wake (&bdbuf_cache.buffer_waiters);
1752        }
1753        bd = NULL;
1754      }
1755    }
1756    else
1757    {
1758      bd = rtems_bdbuf_get_buffer_from_lru_list (dd, block);
1759
1760      if (bd == NULL)
1761        rtems_bdbuf_wait_for_buffer ();
1762    }
1763  }
1764  while (bd == NULL);
1765
1766  rtems_bdbuf_wait_for_access (bd);
1767  rtems_bdbuf_group_obtain (bd);
1768
1769  return bd;
1770}
1771
1772static rtems_status_code
1773rtems_bdbuf_get_media_block (const rtems_disk_device *dd,
1774                             rtems_blkdev_bnum        block,
1775                             rtems_blkdev_bnum       *media_block_ptr)
1776{
1777  rtems_status_code sc = RTEMS_SUCCESSFUL;
1778
1779  if (block < dd->block_count)
1780  {
1781    /*
1782     * Compute the media block number. Drivers work with media block number not
1783     * the block number a BD may have as this depends on the block size set by
1784     * the user.
1785     */
1786    *media_block_ptr = rtems_bdbuf_media_block (dd, block) + dd->start;
1787  }
1788  else
1789  {
1790    sc = RTEMS_INVALID_ID;
1791  }
1792
1793  return sc;
1794}
1795
1796rtems_status_code
1797rtems_bdbuf_get (rtems_disk_device   *dd,
1798                 rtems_blkdev_bnum    block,
1799                 rtems_bdbuf_buffer **bd_ptr)
1800{
1801  rtems_status_code   sc = RTEMS_SUCCESSFUL;
1802  rtems_bdbuf_buffer *bd = NULL;
1803  rtems_blkdev_bnum   media_block;
1804
1805  rtems_bdbuf_lock_cache ();
1806
1807  sc = rtems_bdbuf_get_media_block (dd, block, &media_block);
1808  if (sc == RTEMS_SUCCESSFUL)
1809  {
1810    /*
1811     * Print the block index relative to the physical disk.
1812     */
1813    if (rtems_bdbuf_tracer)
1814      printf ("bdbuf:get: %" PRIu32 " (%" PRIu32 ") (dev = %08x)\n",
1815              media_block, block, (unsigned) dd->dev);
1816
1817    bd = rtems_bdbuf_get_buffer_for_access (dd, media_block);
1818
1819    switch (bd->state)
1820    {
1821      case RTEMS_BDBUF_STATE_CACHED:
1822        rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_ACCESS_CACHED);
1823        break;
1824      case RTEMS_BDBUF_STATE_EMPTY:
1825        rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_ACCESS_EMPTY);
1826        break;
1827      case RTEMS_BDBUF_STATE_MODIFIED:
1828        /*
1829         * To get a modified buffer could be considered a bug in the caller
1830         * because you should not be getting an already modified buffer but
1831         * user may have modified a byte in a block then decided to seek the
1832         * start and write the whole block and the file system will have no
1833         * record of this so just gets the block to fill.
1834         */
1835        rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_ACCESS_MODIFIED);
1836        break;
1837      default:
1838        rtems_bdbuf_fatal (bd->state, RTEMS_BLKDEV_FATAL_BDBUF_STATE_2);
1839        break;
1840    }
1841
1842    if (rtems_bdbuf_tracer)
1843    {
1844      rtems_bdbuf_show_users ("get", bd);
1845      rtems_bdbuf_show_usage ();
1846    }
1847  }
1848
1849  rtems_bdbuf_unlock_cache ();
1850
1851  *bd_ptr = bd;
1852
1853  return sc;
1854}
1855
1856/**
1857 * Call back handler called by the low level driver when the transfer has
1858 * completed. This function may be invoked from interrupt handler.
1859 *
1860 * @param arg Arbitrary argument specified in block device request
1861 *            structure (in this case - pointer to the appropriate
1862 *            block device request structure).
1863 * @param status I/O completion status
1864 */
1865static void
1866rtems_bdbuf_transfer_done (void* arg, rtems_status_code status)
1867{
1868  rtems_blkdev_request* req = (rtems_blkdev_request*) arg;
1869
1870  req->status = status;
1871
1872  rtems_event_send (req->io_task, RTEMS_BDBUF_TRANSFER_SYNC);
1873}
1874
1875static rtems_status_code
1876rtems_bdbuf_execute_transfer_request (rtems_disk_device    *dd,
1877                                      rtems_blkdev_request *req,
1878                                      bool                  cache_locked)
1879{
1880  rtems_status_code sc = RTEMS_SUCCESSFUL;
1881  int result = 0;
1882  uint32_t transfer_index = 0;
1883  bool wake_transfer_waiters = false;
1884  bool wake_buffer_waiters = false;
1885
1886  if (cache_locked)
1887    rtems_bdbuf_unlock_cache ();
1888
1889  result = dd->ioctl (dd->phys_dev, RTEMS_BLKIO_REQUEST, req);
1890
1891  if (result == 0)
1892  {
1893    rtems_bdbuf_wait_for_event (RTEMS_BDBUF_TRANSFER_SYNC);
1894    sc = req->status;
1895  }
1896  else
1897    sc = RTEMS_IO_ERROR;
1898
1899  rtems_bdbuf_lock_cache ();
1900
1901  /* Statistics */
1902  if (req->req == RTEMS_BLKDEV_REQ_READ)
1903  {
1904    dd->stats.read_blocks += req->bufnum;
1905    if (sc != RTEMS_SUCCESSFUL)
1906      ++dd->stats.read_errors;
1907  }
1908  else
1909  {
1910    dd->stats.write_blocks += req->bufnum;
1911    ++dd->stats.write_transfers;
1912    if (sc != RTEMS_SUCCESSFUL)
1913      ++dd->stats.write_errors;
1914  }
1915
1916  for (transfer_index = 0; transfer_index < req->bufnum; ++transfer_index)
1917  {
1918    rtems_bdbuf_buffer *bd = req->bufs [transfer_index].user;
1919    bool waiters = bd->waiters;
1920
1921    if (waiters)
1922      wake_transfer_waiters = true;
1923    else
1924      wake_buffer_waiters = true;
1925
1926    rtems_bdbuf_group_release (bd);
1927
1928    if (sc == RTEMS_SUCCESSFUL && bd->state == RTEMS_BDBUF_STATE_TRANSFER)
1929      rtems_bdbuf_make_cached_and_add_to_lru_list (bd);
1930    else
1931      rtems_bdbuf_discard_buffer (bd);
1932
1933    if (rtems_bdbuf_tracer)
1934      rtems_bdbuf_show_users ("transfer", bd);
1935  }
1936
1937  if (wake_transfer_waiters)
1938    rtems_bdbuf_wake (&bdbuf_cache.transfer_waiters);
1939
1940  if (wake_buffer_waiters)
1941    rtems_bdbuf_wake (&bdbuf_cache.buffer_waiters);
1942
1943  if (!cache_locked)
1944    rtems_bdbuf_unlock_cache ();
1945
1946  if (sc == RTEMS_SUCCESSFUL || sc == RTEMS_UNSATISFIED)
1947    return sc;
1948  else
1949    return RTEMS_IO_ERROR;
1950}
1951
1952static rtems_status_code
1953rtems_bdbuf_execute_read_request (rtems_disk_device  *dd,
1954                                  rtems_bdbuf_buffer *bd,
1955                                  uint32_t            transfer_count)
1956{
1957  rtems_blkdev_request *req = NULL;
1958  rtems_blkdev_bnum media_block = bd->block;
1959  uint32_t media_blocks_per_block = dd->media_blocks_per_block;
1960  uint32_t block_size = dd->block_size;
1961  uint32_t transfer_index = 1;
1962
1963  /*
1964   * TODO: This type of request structure is wrong and should be removed.
1965   */
1966#define bdbuf_alloc(size) __builtin_alloca (size)
1967
1968  req = bdbuf_alloc (sizeof (rtems_blkdev_request) +
1969                     sizeof (rtems_blkdev_sg_buffer) * transfer_count);
1970
1971  req->req = RTEMS_BLKDEV_REQ_READ;
1972  req->req_done = rtems_bdbuf_transfer_done;
1973  req->done_arg = req;
1974  req->io_task = rtems_task_self ();
1975  req->status = RTEMS_RESOURCE_IN_USE;
1976  req->bufnum = 0;
1977
1978  rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_TRANSFER);
1979
1980  req->bufs [0].user   = bd;
1981  req->bufs [0].block  = media_block;
1982  req->bufs [0].length = block_size;
1983  req->bufs [0].buffer = bd->buffer;
1984
1985  if (rtems_bdbuf_tracer)
1986    rtems_bdbuf_show_users ("read", bd);
1987
1988  while (transfer_index < transfer_count)
1989  {
1990    media_block += media_blocks_per_block;
1991
1992    bd = rtems_bdbuf_get_buffer_for_read_ahead (dd, media_block);
1993
1994    if (bd == NULL)
1995      break;
1996
1997    rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_TRANSFER);
1998
1999    req->bufs [transfer_index].user   = bd;
2000    req->bufs [transfer_index].block  = media_block;
2001    req->bufs [transfer_index].length = block_size;
2002    req->bufs [transfer_index].buffer = bd->buffer;
2003
2004    if (rtems_bdbuf_tracer)
2005      rtems_bdbuf_show_users ("read", bd);
2006
2007    ++transfer_index;
2008  }
2009
2010  req->bufnum = transfer_index;
2011
2012  return rtems_bdbuf_execute_transfer_request (dd, req, true);
2013}
2014
2015static bool
2016rtems_bdbuf_is_read_ahead_active (const rtems_disk_device *dd)
2017{
2018  return !rtems_chain_is_node_off_chain (&dd->read_ahead.node);
2019}
2020
2021static void
2022rtems_bdbuf_read_ahead_cancel (rtems_disk_device *dd)
2023{
2024  if (rtems_bdbuf_is_read_ahead_active (dd))
2025  {
2026    rtems_chain_extract_unprotected (&dd->read_ahead.node);
2027    rtems_chain_set_off_chain (&dd->read_ahead.node);
2028  }
2029}
2030
2031static void
2032rtems_bdbuf_read_ahead_reset (rtems_disk_device *dd)
2033{
2034  rtems_bdbuf_read_ahead_cancel (dd);
2035  dd->read_ahead.trigger = RTEMS_DISK_READ_AHEAD_NO_TRIGGER;
2036}
2037
2038static void
2039rtems_bdbuf_check_read_ahead_trigger (rtems_disk_device *dd,
2040                                      rtems_blkdev_bnum  block)
2041{
2042  if (dd->read_ahead.trigger == block
2043      && !rtems_bdbuf_is_read_ahead_active (dd))
2044  {
2045    rtems_status_code sc;
2046    rtems_chain_control *chain = &bdbuf_cache.read_ahead_chain;
2047
2048    rtems_chain_append_unprotected (chain, &dd->read_ahead.node);
2049    sc = rtems_event_send (bdbuf_cache.read_ahead_task,
2050                           RTEMS_BDBUF_READ_AHEAD_WAKE_UP);
2051    if (sc != RTEMS_SUCCESSFUL)
2052      rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_RA_WAKE_UP);
2053  }
2054}
2055
2056static void
2057rtems_bdbuf_set_read_ahead_trigger (rtems_disk_device *dd,
2058                                    rtems_blkdev_bnum  block)
2059{
2060  if (dd->read_ahead.trigger != block)
2061  {
2062    rtems_bdbuf_read_ahead_cancel (dd);
2063    dd->read_ahead.trigger = block + 1;
2064    dd->read_ahead.next = block + 2;
2065  }
2066}
2067
2068rtems_status_code
2069rtems_bdbuf_read (rtems_disk_device   *dd,
2070                  rtems_blkdev_bnum    block,
2071                  rtems_bdbuf_buffer **bd_ptr)
2072{
2073  rtems_status_code     sc = RTEMS_SUCCESSFUL;
2074  rtems_bdbuf_buffer   *bd = NULL;
2075  rtems_blkdev_bnum     media_block;
2076
2077  rtems_bdbuf_lock_cache ();
2078
2079  sc = rtems_bdbuf_get_media_block (dd, block, &media_block);
2080  if (sc == RTEMS_SUCCESSFUL)
2081  {
2082    if (rtems_bdbuf_tracer)
2083      printf ("bdbuf:read: %" PRIu32 " (%" PRIu32 ") (dev = %08x)\n",
2084              media_block + dd->start, block, (unsigned) dd->dev);
2085
2086    bd = rtems_bdbuf_get_buffer_for_access (dd, media_block);
2087    switch (bd->state)
2088    {
2089      case RTEMS_BDBUF_STATE_CACHED:
2090        ++dd->stats.read_hits;
2091        rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_ACCESS_CACHED);
2092        break;
2093      case RTEMS_BDBUF_STATE_MODIFIED:
2094        ++dd->stats.read_hits;
2095        rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_ACCESS_MODIFIED);
2096        break;
2097      case RTEMS_BDBUF_STATE_EMPTY:
2098        ++dd->stats.read_misses;
2099        rtems_bdbuf_set_read_ahead_trigger (dd, block);
2100        sc = rtems_bdbuf_execute_read_request (dd, bd, 1);
2101        if (sc == RTEMS_SUCCESSFUL)
2102        {
2103          rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_ACCESS_CACHED);
2104          rtems_chain_extract_unprotected (&bd->link);
2105          rtems_bdbuf_group_obtain (bd);
2106        }
2107        else
2108        {
2109          bd = NULL;
2110        }
2111        break;
2112      default:
2113        rtems_bdbuf_fatal (bd->state, RTEMS_BLKDEV_FATAL_BDBUF_STATE_4);
2114        break;
2115    }
2116
2117    rtems_bdbuf_check_read_ahead_trigger (dd, block);
2118  }
2119
2120  rtems_bdbuf_unlock_cache ();
2121
2122  *bd_ptr = bd;
2123
2124  return sc;
2125}
2126
2127static rtems_status_code
2128rtems_bdbuf_check_bd_and_lock_cache (rtems_bdbuf_buffer *bd, const char *kind)
2129{
2130  if (bd == NULL)
2131    return RTEMS_INVALID_ADDRESS;
2132  if (rtems_bdbuf_tracer)
2133  {
2134    printf ("bdbuf:%s: %" PRIu32 "\n", kind, bd->block);
2135    rtems_bdbuf_show_users (kind, bd);
2136  }
2137  rtems_bdbuf_lock_cache();
2138
2139  return RTEMS_SUCCESSFUL;
2140}
2141
2142rtems_status_code
2143rtems_bdbuf_release (rtems_bdbuf_buffer *bd)
2144{
2145  rtems_status_code sc = RTEMS_SUCCESSFUL;
2146
2147  sc = rtems_bdbuf_check_bd_and_lock_cache (bd, "release");
2148  if (sc != RTEMS_SUCCESSFUL)
2149    return sc;
2150
2151  switch (bd->state)
2152  {
2153    case RTEMS_BDBUF_STATE_ACCESS_CACHED:
2154      rtems_bdbuf_add_to_lru_list_after_access (bd);
2155      break;
2156    case RTEMS_BDBUF_STATE_ACCESS_EMPTY:
2157    case RTEMS_BDBUF_STATE_ACCESS_PURGED:
2158      rtems_bdbuf_discard_buffer_after_access (bd);
2159      break;
2160    case RTEMS_BDBUF_STATE_ACCESS_MODIFIED:
2161      rtems_bdbuf_add_to_modified_list_after_access (bd);
2162      break;
2163    default:
2164      rtems_bdbuf_fatal (bd->state, RTEMS_BLKDEV_FATAL_BDBUF_STATE_0);
2165      break;
2166  }
2167
2168  if (rtems_bdbuf_tracer)
2169    rtems_bdbuf_show_usage ();
2170
2171  rtems_bdbuf_unlock_cache ();
2172
2173  return RTEMS_SUCCESSFUL;
2174}
2175
2176rtems_status_code
2177rtems_bdbuf_release_modified (rtems_bdbuf_buffer *bd)
2178{
2179  rtems_status_code sc = RTEMS_SUCCESSFUL;
2180
2181  sc = rtems_bdbuf_check_bd_and_lock_cache (bd, "release modified");
2182  if (sc != RTEMS_SUCCESSFUL)
2183    return sc;
2184
2185  switch (bd->state)
2186  {
2187    case RTEMS_BDBUF_STATE_ACCESS_CACHED:
2188    case RTEMS_BDBUF_STATE_ACCESS_EMPTY:
2189    case RTEMS_BDBUF_STATE_ACCESS_MODIFIED:
2190      rtems_bdbuf_add_to_modified_list_after_access (bd);
2191      break;
2192    case RTEMS_BDBUF_STATE_ACCESS_PURGED:
2193      rtems_bdbuf_discard_buffer_after_access (bd);
2194      break;
2195    default:
2196      rtems_bdbuf_fatal (bd->state, RTEMS_BLKDEV_FATAL_BDBUF_STATE_6);
2197      break;
2198  }
2199
2200  if (rtems_bdbuf_tracer)
2201    rtems_bdbuf_show_usage ();
2202
2203  rtems_bdbuf_unlock_cache ();
2204
2205  return RTEMS_SUCCESSFUL;
2206}
2207
2208rtems_status_code
2209rtems_bdbuf_sync (rtems_bdbuf_buffer *bd)
2210{
2211  rtems_status_code sc = RTEMS_SUCCESSFUL;
2212
2213  sc = rtems_bdbuf_check_bd_and_lock_cache (bd, "sync");
2214  if (sc != RTEMS_SUCCESSFUL)
2215    return sc;
2216
2217  switch (bd->state)
2218  {
2219    case RTEMS_BDBUF_STATE_ACCESS_CACHED:
2220    case RTEMS_BDBUF_STATE_ACCESS_EMPTY:
2221    case RTEMS_BDBUF_STATE_ACCESS_MODIFIED:
2222      rtems_bdbuf_sync_after_access (bd);
2223      break;
2224    case RTEMS_BDBUF_STATE_ACCESS_PURGED:
2225      rtems_bdbuf_discard_buffer_after_access (bd);
2226      break;
2227    default:
2228      rtems_bdbuf_fatal (bd->state, RTEMS_BLKDEV_FATAL_BDBUF_STATE_5);
2229      break;
2230  }
2231
2232  if (rtems_bdbuf_tracer)
2233    rtems_bdbuf_show_usage ();
2234
2235  rtems_bdbuf_unlock_cache ();
2236
2237  return RTEMS_SUCCESSFUL;
2238}
2239
2240rtems_status_code
2241rtems_bdbuf_syncdev (rtems_disk_device *dd)
2242{
2243  if (rtems_bdbuf_tracer)
2244    printf ("bdbuf:syncdev: %08x\n", (unsigned) dd->dev);
2245
2246  /*
2247   * Take the sync lock before locking the cache. Once we have the sync lock we
2248   * can lock the cache. If another thread has the sync lock it will cause this
2249   * thread to block until it owns the sync lock then it can own the cache. The
2250   * sync lock can only be obtained with the cache unlocked.
2251   */
2252  rtems_bdbuf_lock_sync ();
2253  rtems_bdbuf_lock_cache ();
2254
2255  /*
2256   * Set the cache to have a sync active for a specific device and let the swap
2257   * out task know the id of the requester to wake when done.
2258   *
2259   * The swap out task will negate the sync active flag when no more buffers
2260   * for the device are held on the "modified for sync" queues.
2261   */
2262  bdbuf_cache.sync_active    = true;
2263  bdbuf_cache.sync_requester = rtems_task_self ();
2264  bdbuf_cache.sync_device    = dd;
2265
2266  rtems_bdbuf_wake_swapper ();
2267  rtems_bdbuf_unlock_cache ();
2268  rtems_bdbuf_wait_for_event (RTEMS_BDBUF_TRANSFER_SYNC);
2269  rtems_bdbuf_unlock_sync ();
2270
2271  return RTEMS_SUCCESSFUL;
2272}
2273
2274/**
2275 * Swapout transfer to the driver. The driver will break this I/O into groups
2276 * of consecutive write requests is multiple consecutive buffers are required
2277 * by the driver. The cache is not locked.
2278 *
2279 * @param transfer The transfer transaction.
2280 */
2281static void
2282rtems_bdbuf_swapout_write (rtems_bdbuf_swapout_transfer* transfer)
2283{
2284  rtems_chain_node *node;
2285
2286  if (rtems_bdbuf_tracer)
2287    printf ("bdbuf:swapout transfer: %08x\n", (unsigned) transfer->dd->dev);
2288
2289  /*
2290   * If there are buffers to transfer to the media transfer them.
2291   */
2292  if (!rtems_chain_is_empty (&transfer->bds))
2293  {
2294    /*
2295     * The last block number used when the driver only supports
2296     * continuous blocks in a single request.
2297     */
2298    uint32_t last_block = 0;
2299
2300    rtems_disk_device *dd = transfer->dd;
2301    uint32_t media_blocks_per_block = dd->media_blocks_per_block;
2302    bool need_continuous_blocks =
2303      (dd->phys_dev->capabilities & RTEMS_BLKDEV_CAP_MULTISECTOR_CONT) != 0;
2304
2305    /*
2306     * Take as many buffers as configured and pass to the driver. Note, the
2307     * API to the drivers has an array of buffers and if a chain was passed
2308     * we could have just passed the list. If the driver API is updated it
2309     * should be possible to make this change with little effect in this
2310     * code. The array that is passed is broken in design and should be
2311     * removed. Merging members of a struct into the first member is
2312     * trouble waiting to happen.
2313     */
2314    transfer->write_req->status = RTEMS_RESOURCE_IN_USE;
2315    transfer->write_req->bufnum = 0;
2316
2317    while ((node = rtems_chain_get_unprotected(&transfer->bds)) != NULL)
2318    {
2319      rtems_bdbuf_buffer* bd = (rtems_bdbuf_buffer*) node;
2320      bool                write = false;
2321
2322      /*
2323       * If the device only accepts sequential buffers and this is not the
2324       * first buffer (the first is always sequential, and the buffer is not
2325       * sequential then put the buffer back on the transfer chain and write
2326       * the committed buffers.
2327       */
2328
2329      if (rtems_bdbuf_tracer)
2330        printf ("bdbuf:swapout write: bd:%" PRIu32 ", bufnum:%" PRIu32 " mode:%s\n",
2331                bd->block, transfer->write_req->bufnum,
2332                need_continuous_blocks ? "MULTI" : "SCAT");
2333
2334      if (need_continuous_blocks && transfer->write_req->bufnum &&
2335          bd->block != last_block + media_blocks_per_block)
2336      {
2337        rtems_chain_prepend_unprotected (&transfer->bds, &bd->link);
2338        write = true;
2339      }
2340      else
2341      {
2342        rtems_blkdev_sg_buffer* buf;
2343        buf = &transfer->write_req->bufs[transfer->write_req->bufnum];
2344        transfer->write_req->bufnum++;
2345        buf->user   = bd;
2346        buf->block  = bd->block;
2347        buf->length = dd->block_size;
2348        buf->buffer = bd->buffer;
2349        last_block  = bd->block;
2350      }
2351
2352      /*
2353       * Perform the transfer if there are no more buffers, or the transfer
2354       * size has reached the configured max. value.
2355       */
2356
2357      if (rtems_chain_is_empty (&transfer->bds) ||
2358          (transfer->write_req->bufnum >= bdbuf_config.max_write_blocks))
2359        write = true;
2360
2361      if (write)
2362      {
2363        rtems_bdbuf_execute_transfer_request (dd, transfer->write_req, false);
2364
2365        transfer->write_req->status = RTEMS_RESOURCE_IN_USE;
2366        transfer->write_req->bufnum = 0;
2367      }
2368    }
2369
2370    /*
2371     * If sync'ing and the deivce is capability of handling a sync IO control
2372     * call perform the call.
2373     */
2374    if (transfer->syncing &&
2375        (dd->phys_dev->capabilities & RTEMS_BLKDEV_CAP_SYNC))
2376    {
2377      /* int result = */ dd->ioctl (dd->phys_dev, RTEMS_BLKDEV_REQ_SYNC, NULL);
2378      /* How should the error be handled ? */
2379    }
2380  }
2381}
2382
2383/**
2384 * Process the modified list of buffers. There is a sync or modified list that
2385 * needs to be handled so we have a common function to do the work.
2386 *
2387 * @param dd_ptr Pointer to the device to handle. If BDBUF_INVALID_DEV no
2388 * device is selected so select the device of the first buffer to be written to
2389 * disk.
2390 * @param chain The modified chain to process.
2391 * @param transfer The chain to append buffers to be written too.
2392 * @param sync_active If true this is a sync operation so expire all timers.
2393 * @param update_timers If true update the timers.
2394 * @param timer_delta It update_timers is true update the timers by this
2395 *                    amount.
2396 */
2397static void
2398rtems_bdbuf_swapout_modified_processing (rtems_disk_device  **dd_ptr,
2399                                         rtems_chain_control* chain,
2400                                         rtems_chain_control* transfer,
2401                                         bool                 sync_active,
2402                                         bool                 update_timers,
2403                                         uint32_t             timer_delta)
2404{
2405  if (!rtems_chain_is_empty (chain))
2406  {
2407    rtems_chain_node* node = rtems_chain_head (chain);
2408    bool              sync_all;
2409   
2410    node = node->next;
2411
2412    /*
2413     * A sync active with no valid dev means sync all.
2414     */
2415    if (sync_active && (*dd_ptr == BDBUF_INVALID_DEV))
2416      sync_all = true;
2417    else
2418      sync_all = false;
2419   
2420    while (!rtems_chain_is_tail (chain, node))
2421    {
2422      rtems_bdbuf_buffer* bd = (rtems_bdbuf_buffer*) node;
2423
2424      /*
2425       * Check if the buffer's hold timer has reached 0. If a sync is active
2426       * or someone waits for a buffer written force all the timers to 0.
2427       *
2428       * @note Lots of sync requests will skew this timer. It should be based
2429       *       on TOD to be accurate. Does it matter ?
2430       */
2431      if (sync_all || (sync_active && (*dd_ptr == bd->dd))
2432          || rtems_bdbuf_has_buffer_waiters ())
2433        bd->hold_timer = 0;
2434
2435      if (bd->hold_timer)
2436      {
2437        if (update_timers)
2438        {
2439          if (bd->hold_timer > timer_delta)
2440            bd->hold_timer -= timer_delta;
2441          else
2442            bd->hold_timer = 0;
2443        }
2444
2445        if (bd->hold_timer)
2446        {
2447          node = node->next;
2448          continue;
2449        }
2450      }
2451
2452      /*
2453       * This assumes we can set it to BDBUF_INVALID_DEV which is just an
2454       * assumption. Cannot use the transfer list being empty the sync dev
2455       * calls sets the dev to use.
2456       */
2457      if (*dd_ptr == BDBUF_INVALID_DEV)
2458        *dd_ptr = bd->dd;
2459
2460      if (bd->dd == *dd_ptr)
2461      {
2462        rtems_chain_node* next_node = node->next;
2463        rtems_chain_node* tnode = rtems_chain_tail (transfer);
2464
2465        /*
2466         * The blocks on the transfer list are sorted in block order. This
2467         * means multi-block transfers for drivers that require consecutive
2468         * blocks perform better with sorted blocks and for real disks it may
2469         * help lower head movement.
2470         */
2471
2472        rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_TRANSFER);
2473
2474        rtems_chain_extract_unprotected (node);
2475
2476        tnode = tnode->previous;
2477
2478        while (node && !rtems_chain_is_head (transfer, tnode))
2479        {
2480          rtems_bdbuf_buffer* tbd = (rtems_bdbuf_buffer*) tnode;
2481
2482          if (bd->block > tbd->block)
2483          {
2484            rtems_chain_insert_unprotected (tnode, node);
2485            node = NULL;
2486          }
2487          else
2488            tnode = tnode->previous;
2489        }
2490
2491        if (node)
2492          rtems_chain_prepend_unprotected (transfer, node);
2493
2494        node = next_node;
2495      }
2496      else
2497      {
2498        node = node->next;
2499      }
2500    }
2501  }
2502}
2503
2504/**
2505 * Process the cache's modified buffers. Check the sync list first then the
2506 * modified list extracting the buffers suitable to be written to disk. We have
2507 * a device at a time. The task level loop will repeat this operation while
2508 * there are buffers to be written. If the transfer fails place the buffers
2509 * back on the modified list and try again later. The cache is unlocked while
2510 * the buffers are being written to disk.
2511 *
2512 * @param timer_delta It update_timers is true update the timers by this
2513 *                    amount.
2514 * @param update_timers If true update the timers.
2515 * @param transfer The transfer transaction data.
2516 *
2517 * @retval true Buffers where written to disk so scan again.
2518 * @retval false No buffers where written to disk.
2519 */
2520static bool
2521rtems_bdbuf_swapout_processing (unsigned long                 timer_delta,
2522                                bool                          update_timers,
2523                                rtems_bdbuf_swapout_transfer* transfer)
2524{
2525  rtems_bdbuf_swapout_worker* worker;
2526  bool                        transfered_buffers = false;
2527
2528  rtems_bdbuf_lock_cache ();
2529
2530  /*
2531   * If a sync is active do not use a worker because the current code does not
2532   * cleaning up after. We need to know the buffers have been written when
2533   * syncing to release sync lock and currently worker threads do not return to
2534   * here. We do not know the worker is the last in a sequence of sync writes
2535   * until after we have it running so we do not know to tell it to release the
2536   * lock. The simplest solution is to get the main swap out task perform all
2537   * sync operations.
2538   */
2539  if (bdbuf_cache.sync_active)
2540    worker = NULL;
2541  else
2542  {
2543    worker = (rtems_bdbuf_swapout_worker*)
2544      rtems_chain_get_unprotected (&bdbuf_cache.swapout_workers);
2545    if (worker)
2546      transfer = &worker->transfer;
2547  }
2548
2549  rtems_chain_initialize_empty (&transfer->bds);
2550  transfer->dd = BDBUF_INVALID_DEV;
2551  transfer->syncing = bdbuf_cache.sync_active;
2552 
2553  /*
2554   * When the sync is for a device limit the sync to that device. If the sync
2555   * is for a buffer handle process the devices in the order on the sync
2556   * list. This means the dev is BDBUF_INVALID_DEV.
2557   */
2558  if (bdbuf_cache.sync_active)
2559    transfer->dd = bdbuf_cache.sync_device;
2560   
2561  /*
2562   * If we have any buffers in the sync queue move them to the modified
2563   * list. The first sync buffer will select the device we use.
2564   */
2565  rtems_bdbuf_swapout_modified_processing (&transfer->dd,
2566                                           &bdbuf_cache.sync,
2567                                           &transfer->bds,
2568                                           true, false,
2569                                           timer_delta);
2570
2571  /*
2572   * Process the cache's modified list.
2573   */
2574  rtems_bdbuf_swapout_modified_processing (&transfer->dd,
2575                                           &bdbuf_cache.modified,
2576                                           &transfer->bds,
2577                                           bdbuf_cache.sync_active,
2578                                           update_timers,
2579                                           timer_delta);
2580
2581  /*
2582   * We have all the buffers that have been modified for this device so the
2583   * cache can be unlocked because the state of each buffer has been set to
2584   * TRANSFER.
2585   */
2586  rtems_bdbuf_unlock_cache ();
2587
2588  /*
2589   * If there are buffers to transfer to the media transfer them.
2590   */
2591  if (!rtems_chain_is_empty (&transfer->bds))
2592  {
2593    if (worker)
2594    {
2595      rtems_status_code sc = rtems_event_send (worker->id,
2596                                               RTEMS_BDBUF_SWAPOUT_SYNC);
2597      if (sc != RTEMS_SUCCESSFUL)
2598        rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_SO_WAKE);
2599    }
2600    else
2601    {
2602      rtems_bdbuf_swapout_write (transfer);
2603    }
2604
2605    transfered_buffers = true;
2606  }
2607
2608  if (bdbuf_cache.sync_active && !transfered_buffers)
2609  {
2610    rtems_id sync_requester;
2611    rtems_bdbuf_lock_cache ();
2612    sync_requester = bdbuf_cache.sync_requester;
2613    bdbuf_cache.sync_active = false;
2614    bdbuf_cache.sync_requester = 0;
2615    rtems_bdbuf_unlock_cache ();
2616    if (sync_requester)
2617      rtems_event_send (sync_requester, RTEMS_BDBUF_TRANSFER_SYNC);
2618  }
2619
2620  return transfered_buffers;
2621}
2622
2623/**
2624 * Allocate the write request and initialise it for good measure.
2625 *
2626 * @return rtems_blkdev_request* The write reference memory.
2627 */
2628static rtems_blkdev_request*
2629rtems_bdbuf_swapout_writereq_alloc (void)
2630{
2631  /*
2632   * @note chrisj The rtems_blkdev_request and the array at the end is a hack.
2633   * I am disappointment at finding code like this in RTEMS. The request should
2634   * have been a rtems_chain_control. Simple, fast and less storage as the node
2635   * is already part of the buffer structure.
2636   */
2637  rtems_blkdev_request* write_req =
2638    malloc (sizeof (rtems_blkdev_request) +
2639            (bdbuf_config.max_write_blocks * sizeof (rtems_blkdev_sg_buffer)));
2640
2641  if (!write_req)
2642    rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_SO_NOMEM);
2643
2644  write_req->req = RTEMS_BLKDEV_REQ_WRITE;
2645  write_req->req_done = rtems_bdbuf_transfer_done;
2646  write_req->done_arg = write_req;
2647  write_req->io_task = rtems_task_self ();
2648
2649  return write_req;
2650}
2651
2652/**
2653 * The swapout worker thread body.
2654 *
2655 * @param arg A pointer to the worker thread's private data.
2656 * @return rtems_task Not used.
2657 */
2658static rtems_task
2659rtems_bdbuf_swapout_worker_task (rtems_task_argument arg)
2660{
2661  rtems_bdbuf_swapout_worker* worker = (rtems_bdbuf_swapout_worker*) arg;
2662
2663  while (worker->enabled)
2664  {
2665    rtems_bdbuf_wait_for_event (RTEMS_BDBUF_SWAPOUT_SYNC);
2666
2667    rtems_bdbuf_swapout_write (&worker->transfer);
2668
2669    rtems_bdbuf_lock_cache ();
2670
2671    rtems_chain_initialize_empty (&worker->transfer.bds);
2672    worker->transfer.dd = BDBUF_INVALID_DEV;
2673
2674    rtems_chain_append_unprotected (&bdbuf_cache.swapout_workers, &worker->link);
2675
2676    rtems_bdbuf_unlock_cache ();
2677  }
2678
2679  free (worker->transfer.write_req);
2680  free (worker);
2681
2682  rtems_task_delete (RTEMS_SELF);
2683}
2684
2685/**
2686 * Open the swapout worker threads.
2687 */
2688static void
2689rtems_bdbuf_swapout_workers_open (void)
2690{
2691  rtems_status_code sc;
2692  size_t            w;
2693
2694  rtems_bdbuf_lock_cache ();
2695
2696  for (w = 0; w < bdbuf_config.swapout_workers; w++)
2697  {
2698    rtems_bdbuf_swapout_worker* worker;
2699
2700    worker = malloc (sizeof (rtems_bdbuf_swapout_worker));
2701    if (!worker)
2702      rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_SO_NOMEM);
2703
2704    rtems_chain_append_unprotected (&bdbuf_cache.swapout_workers, &worker->link);
2705    worker->enabled = true;
2706    worker->transfer.write_req = rtems_bdbuf_swapout_writereq_alloc ();
2707
2708    rtems_chain_initialize_empty (&worker->transfer.bds);
2709    worker->transfer.dd = BDBUF_INVALID_DEV;
2710
2711    sc = rtems_bdbuf_create_task (rtems_build_name('B', 'D', 'o', 'a' + w),
2712                                  bdbuf_config.swapout_worker_priority,
2713                                  RTEMS_BDBUF_SWAPOUT_WORKER_TASK_PRIORITY_DEFAULT,
2714                                  rtems_bdbuf_swapout_worker_task,
2715                                  (rtems_task_argument) worker,
2716                                  &worker->id);
2717    if (sc != RTEMS_SUCCESSFUL)
2718      rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_SO_WK_CREATE);
2719  }
2720
2721  rtems_bdbuf_unlock_cache ();
2722}
2723
2724/**
2725 * Close the swapout worker threads.
2726 */
2727static void
2728rtems_bdbuf_swapout_workers_close (void)
2729{
2730  rtems_chain_node* node;
2731
2732  rtems_bdbuf_lock_cache ();
2733
2734  node = rtems_chain_first (&bdbuf_cache.swapout_workers);
2735  while (!rtems_chain_is_tail (&bdbuf_cache.swapout_workers, node))
2736  {
2737    rtems_bdbuf_swapout_worker* worker = (rtems_bdbuf_swapout_worker*) node;
2738    worker->enabled = false;
2739    rtems_event_send (worker->id, RTEMS_BDBUF_SWAPOUT_SYNC);
2740    node = rtems_chain_next (node);
2741  }
2742
2743  rtems_bdbuf_unlock_cache ();
2744}
2745
2746/**
2747 * Body of task which takes care on flushing modified buffers to the disk.
2748 *
2749 * @param arg A pointer to the global cache data. Use the global variable and
2750 *            not this.
2751 * @return rtems_task Not used.
2752 */
2753static rtems_task
2754rtems_bdbuf_swapout_task (rtems_task_argument arg)
2755{
2756  rtems_bdbuf_swapout_transfer transfer;
2757  uint32_t                     period_in_ticks;
2758  const uint32_t               period_in_msecs = bdbuf_config.swapout_period;
2759  uint32_t                     timer_delta;
2760
2761  transfer.write_req = rtems_bdbuf_swapout_writereq_alloc ();
2762  rtems_chain_initialize_empty (&transfer.bds);
2763  transfer.dd = BDBUF_INVALID_DEV;
2764  transfer.syncing = false;
2765
2766  /*
2767   * Localise the period.
2768   */
2769  period_in_ticks = RTEMS_MICROSECONDS_TO_TICKS (period_in_msecs * 1000);
2770
2771  /*
2772   * This is temporary. Needs to be changed to use the real time clock.
2773   */
2774  timer_delta = period_in_msecs;
2775
2776  /*
2777   * Create the worker threads.
2778   */
2779  rtems_bdbuf_swapout_workers_open ();
2780
2781  while (bdbuf_cache.swapout_enabled)
2782  {
2783    rtems_event_set   out;
2784    rtems_status_code sc;
2785
2786    /*
2787     * Only update the timers once in the processing cycle.
2788     */
2789    bool update_timers = true;
2790
2791    /*
2792     * If we write buffers to any disk perform a check again. We only write a
2793     * single device at a time and the cache may have more than one device's
2794     * buffers modified waiting to be written.
2795     */
2796    bool transfered_buffers;
2797
2798    do
2799    {
2800      transfered_buffers = false;
2801
2802      /*
2803       * Extact all the buffers we find for a specific device. The device is
2804       * the first one we find on a modified list. Process the sync queue of
2805       * buffers first.
2806       */
2807      if (rtems_bdbuf_swapout_processing (timer_delta,
2808                                          update_timers,
2809                                          &transfer))
2810      {
2811        transfered_buffers = true;
2812      }
2813
2814      /*
2815       * Only update the timers once.
2816       */
2817      update_timers = false;
2818    }
2819    while (transfered_buffers);
2820
2821    sc = rtems_event_receive (RTEMS_BDBUF_SWAPOUT_SYNC,
2822                              RTEMS_EVENT_ALL | RTEMS_WAIT,
2823                              period_in_ticks,
2824                              &out);
2825
2826    if ((sc != RTEMS_SUCCESSFUL) && (sc != RTEMS_TIMEOUT))
2827      rtems_fatal_error_occurred (BLKDEV_FATAL_BDBUF_SWAPOUT_RE);
2828  }
2829
2830  rtems_bdbuf_swapout_workers_close ();
2831
2832  free (transfer.write_req);
2833
2834  rtems_task_delete (RTEMS_SELF);
2835}
2836
2837static void
2838rtems_bdbuf_purge_list (rtems_chain_control *purge_list)
2839{
2840  bool wake_buffer_waiters = false;
2841  rtems_chain_node *node = NULL;
2842
2843  while ((node = rtems_chain_get_unprotected (purge_list)) != NULL)
2844  {
2845    rtems_bdbuf_buffer *bd = (rtems_bdbuf_buffer *) node;
2846
2847    if (bd->waiters == 0)
2848      wake_buffer_waiters = true;
2849
2850    rtems_bdbuf_discard_buffer (bd);
2851  }
2852
2853  if (wake_buffer_waiters)
2854    rtems_bdbuf_wake (&bdbuf_cache.buffer_waiters);
2855}
2856
2857static void
2858rtems_bdbuf_gather_for_purge (rtems_chain_control *purge_list,
2859                              const rtems_disk_device *dd)
2860{
2861  rtems_bdbuf_buffer *stack [RTEMS_BDBUF_AVL_MAX_HEIGHT];
2862  rtems_bdbuf_buffer **prev = stack;
2863  rtems_bdbuf_buffer *cur = bdbuf_cache.tree;
2864
2865  *prev = NULL;
2866
2867  while (cur != NULL)
2868  {
2869    if (cur->dd == dd)
2870    {
2871      switch (cur->state)
2872      {
2873        case RTEMS_BDBUF_STATE_FREE:
2874        case RTEMS_BDBUF_STATE_EMPTY:
2875        case RTEMS_BDBUF_STATE_ACCESS_PURGED:
2876        case RTEMS_BDBUF_STATE_TRANSFER_PURGED:
2877          break;
2878        case RTEMS_BDBUF_STATE_SYNC:
2879          rtems_bdbuf_wake (&bdbuf_cache.transfer_waiters);
2880          /* Fall through */
2881        case RTEMS_BDBUF_STATE_MODIFIED:
2882          rtems_bdbuf_group_release (cur);
2883          /* Fall through */
2884        case RTEMS_BDBUF_STATE_CACHED:
2885          rtems_chain_extract_unprotected (&cur->link);
2886          rtems_chain_append_unprotected (purge_list, &cur->link);
2887          break;
2888        case RTEMS_BDBUF_STATE_TRANSFER:
2889          rtems_bdbuf_set_state (cur, RTEMS_BDBUF_STATE_TRANSFER_PURGED);
2890          break;
2891        case RTEMS_BDBUF_STATE_ACCESS_CACHED:
2892        case RTEMS_BDBUF_STATE_ACCESS_EMPTY:
2893        case RTEMS_BDBUF_STATE_ACCESS_MODIFIED:
2894          rtems_bdbuf_set_state (cur, RTEMS_BDBUF_STATE_ACCESS_PURGED);
2895          break;
2896        default:
2897          rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_STATE_11);
2898      }
2899    }
2900
2901    if (cur->avl.left != NULL)
2902    {
2903      /* Left */
2904      ++prev;
2905      *prev = cur;
2906      cur = cur->avl.left;
2907    }
2908    else if (cur->avl.right != NULL)
2909    {
2910      /* Right */
2911      ++prev;
2912      *prev = cur;
2913      cur = cur->avl.right;
2914    }
2915    else
2916    {
2917      while (*prev != NULL
2918             && (cur == (*prev)->avl.right || (*prev)->avl.right == NULL))
2919      {
2920        /* Up */
2921        cur = *prev;
2922        --prev;
2923      }
2924      if (*prev != NULL)
2925        /* Right */
2926        cur = (*prev)->avl.right;
2927      else
2928        /* Finished */
2929        cur = NULL;
2930    }
2931  }
2932}
2933
2934void
2935rtems_bdbuf_purge_dev (rtems_disk_device *dd)
2936{
2937  rtems_chain_control purge_list;
2938
2939  rtems_chain_initialize_empty (&purge_list);
2940  rtems_bdbuf_lock_cache ();
2941  rtems_bdbuf_read_ahead_reset (dd);
2942  rtems_bdbuf_gather_for_purge (&purge_list, dd);
2943  rtems_bdbuf_purge_list (&purge_list);
2944  rtems_bdbuf_unlock_cache ();
2945}
2946
2947rtems_status_code
2948rtems_bdbuf_set_block_size (rtems_disk_device *dd, uint32_t block_size)
2949{
2950  rtems_status_code sc = RTEMS_SUCCESSFUL;
2951
2952  rtems_bdbuf_lock_cache ();
2953
2954  if (block_size > 0)
2955  {
2956    size_t bds_per_group = rtems_bdbuf_bds_per_group (block_size);
2957
2958    if (bds_per_group != 0)
2959    {
2960      int block_to_media_block_shift = 0;
2961      uint32_t media_blocks_per_block = block_size / dd->media_block_size;
2962      uint32_t one = 1;
2963
2964      while ((one << block_to_media_block_shift) < media_blocks_per_block)
2965      {
2966        ++block_to_media_block_shift;
2967      }
2968
2969      if ((dd->media_block_size << block_to_media_block_shift) != block_size)
2970        block_to_media_block_shift = -1;
2971
2972      dd->block_size = block_size;
2973      dd->block_count = dd->size / media_blocks_per_block;
2974      dd->media_blocks_per_block = media_blocks_per_block;
2975      dd->block_to_media_block_shift = block_to_media_block_shift;
2976      dd->bds_per_group = bds_per_group;
2977
2978      rtems_bdbuf_read_ahead_reset (dd);
2979    }
2980    else
2981    {
2982      sc = RTEMS_INVALID_NUMBER;
2983    }
2984  }
2985  else
2986  {
2987    sc = RTEMS_INVALID_NUMBER;
2988  }
2989
2990  rtems_bdbuf_unlock_cache ();
2991
2992  return sc;
2993}
2994
2995static rtems_task
2996rtems_bdbuf_read_ahead_task (rtems_task_argument arg)
2997{
2998  rtems_chain_control *chain = &bdbuf_cache.read_ahead_chain;
2999
3000  while (bdbuf_cache.read_ahead_enabled)
3001  {
3002    rtems_chain_node *node;
3003
3004    rtems_bdbuf_wait_for_event (RTEMS_BDBUF_READ_AHEAD_WAKE_UP);
3005    rtems_bdbuf_lock_cache ();
3006
3007    while ((node = rtems_chain_get_unprotected (chain)) != NULL)
3008    {
3009      rtems_disk_device *dd = (rtems_disk_device *)
3010        ((char *) node - offsetof (rtems_disk_device, read_ahead.node));
3011      rtems_blkdev_bnum block = dd->read_ahead.next;
3012      rtems_blkdev_bnum media_block = 0;
3013      rtems_status_code sc =
3014        rtems_bdbuf_get_media_block (dd, block, &media_block);
3015
3016      rtems_chain_set_off_chain (&dd->read_ahead.node);
3017
3018      if (sc == RTEMS_SUCCESSFUL)
3019      {
3020        rtems_bdbuf_buffer *bd =
3021          rtems_bdbuf_get_buffer_for_read_ahead (dd, media_block);
3022
3023        if (bd != NULL)
3024        {
3025          uint32_t transfer_count = dd->block_count - block;
3026          uint32_t max_transfer_count = bdbuf_config.max_read_ahead_blocks;
3027
3028          if (transfer_count >= max_transfer_count)
3029          {
3030            transfer_count = max_transfer_count;
3031            dd->read_ahead.trigger = block + transfer_count / 2;
3032            dd->read_ahead.next = block + transfer_count;
3033          }
3034          else
3035          {
3036            dd->read_ahead.trigger = RTEMS_DISK_READ_AHEAD_NO_TRIGGER;
3037          }
3038
3039          ++dd->stats.read_ahead_transfers;
3040          rtems_bdbuf_execute_read_request (dd, bd, transfer_count);
3041        }
3042      }
3043      else
3044      {
3045        dd->read_ahead.trigger = RTEMS_DISK_READ_AHEAD_NO_TRIGGER;
3046      }
3047    }
3048
3049    rtems_bdbuf_unlock_cache ();
3050  }
3051
3052  rtems_task_delete (RTEMS_SELF);
3053}
3054
3055void rtems_bdbuf_get_device_stats (const rtems_disk_device *dd,
3056                                   rtems_blkdev_stats      *stats)
3057{
3058  rtems_bdbuf_lock_cache ();
3059  *stats = dd->stats;
3060  rtems_bdbuf_unlock_cache ();
3061}
3062
3063void rtems_bdbuf_reset_device_stats (rtems_disk_device *dd)
3064{
3065  rtems_bdbuf_lock_cache ();
3066  memset (&dd->stats, 0, sizeof(dd->stats));
3067  rtems_bdbuf_unlock_cache ();
3068}
Note: See TracBrowser for help on using the repository browser.