source: rtems/cpukit/libblock/src/bdbuf.c @ 5b29a08

4.115
Last change on this file since 5b29a08 was 5b29a08, checked in by Sebastian Huber <sebastian.huber@…>, on Jul 6, 2012 at 2:39:00 PM

libblock: Fix read-ahead task wake-up

In case the read-ahead task is not configured we must not send a wake-up
event. This would send the wake-up event to the executing task.

We must send the wake-up event only in case the request list changes
from empty to non-empty. Since otherwise we may send a false transfer
event.

  • Property mode set to 100644
File size: 83.6 KB
Line 
1/**
2 * @file
3 *
4 * @ingroup rtems_bdbuf
5 *
6 * Block device buffer management.
7 */
8
9/*
10 * Disk I/O buffering
11 * Buffer managment
12 *
13 * Copyright (C) 2001 OKTET Ltd., St.-Peterburg, Russia
14 * Author: Andrey G. Ivanov <Andrey.Ivanov@oktet.ru>
15 *         Victor V. Vengerov <vvv@oktet.ru>
16 *         Alexander Kukuta <kam@oktet.ru>
17 *
18 * Copyright (C) 2008,2009 Chris Johns <chrisj@rtems.org>
19 *    Rewritten to remove score mutex access. Fixes many performance
20 *    issues.
21 *
22 * Copyright (c) 2009-2012 embedded brains GmbH.
23 *
24 * @(#) bdbuf.c,v 1.14 2004/04/17 08:15:17 ralf Exp
25 */
26
27/**
28 * Set to 1 to enable debug tracing.
29 */
30#define RTEMS_BDBUF_TRACE 0
31
32#if HAVE_CONFIG_H
33#include "config.h"
34#endif
35#include <limits.h>
36#include <errno.h>
37#include <stdio.h>
38#include <string.h>
39#include <inttypes.h>
40
41#include <rtems.h>
42#include <rtems/error.h>
43#include <rtems/malloc.h>
44
45#include "rtems/bdbuf.h"
46
47#define BDBUF_INVALID_DEV NULL
48
49/*
50 * Simpler label for this file.
51 */
52#define bdbuf_config rtems_bdbuf_configuration
53
54/**
55 * A swapout transfer transaction data. This data is passed to a worked thread
56 * to handle the write phase of the transfer.
57 */
58typedef struct rtems_bdbuf_swapout_transfer
59{
60  rtems_chain_control   bds;         /**< The transfer list of BDs. */
61  rtems_disk_device    *dd;          /**< The device the transfer is for. */
62  bool                  syncing;     /**< The data is a sync'ing. */
63  rtems_blkdev_request* write_req;   /**< The write request array. */
64} rtems_bdbuf_swapout_transfer;
65
66/**
67 * Swapout worker thread. These are available to take processing from the
68 * main swapout thread and handle the I/O operation.
69 */
70typedef struct rtems_bdbuf_swapout_worker
71{
72  rtems_chain_node             link;     /**< The threads sit on a chain when
73                                          * idle. */
74  rtems_id                     id;       /**< The id of the task so we can wake
75                                          * it. */
76  bool                         enabled;  /**< The worker is enabled. */
77  rtems_bdbuf_swapout_transfer transfer; /**< The transfer data for this
78                                          * thread. */
79} rtems_bdbuf_swapout_worker;
80
81/**
82 * Buffer waiters synchronization.
83 */
84typedef struct rtems_bdbuf_waiters {
85  unsigned count;
86  rtems_id sema;
87} rtems_bdbuf_waiters;
88
89/**
90 * The BD buffer cache.
91 */
92typedef struct rtems_bdbuf_cache
93{
94  rtems_id            swapout;           /**< Swapout task ID */
95  bool                swapout_enabled;   /**< Swapout is only running if
96                                          * enabled. Set to false to kill the
97                                          * swap out task. It deletes itself. */
98  rtems_chain_control swapout_workers;   /**< The work threads for the swapout
99                                          * task. */
100
101  rtems_bdbuf_buffer* bds;               /**< Pointer to table of buffer
102                                          * descriptors. */
103  void*               buffers;           /**< The buffer's memory. */
104  size_t              buffer_min_count;  /**< Number of minimum size buffers
105                                          * that fit the buffer memory. */
106  size_t              max_bds_per_group; /**< The number of BDs of minimum
107                                          * buffer size that fit in a group. */
108  uint32_t            flags;             /**< Configuration flags. */
109
110  rtems_id            lock;              /**< The cache lock. It locks all
111                                          * cache data, BD and lists. */
112  rtems_id            sync_lock;         /**< Sync calls block writes. */
113  bool                sync_active;       /**< True if a sync is active. */
114  rtems_id            sync_requester;    /**< The sync requester. */
115  rtems_disk_device  *sync_device;       /**< The device to sync and
116                                          * BDBUF_INVALID_DEV not a device
117                                          * sync. */
118
119  rtems_bdbuf_buffer* tree;              /**< Buffer descriptor lookup AVL tree
120                                          * root. There is only one. */
121  rtems_chain_control lru;               /**< Least recently used list */
122  rtems_chain_control modified;          /**< Modified buffers list */
123  rtems_chain_control sync;              /**< Buffers to sync list */
124
125  rtems_bdbuf_waiters access_waiters;    /**< Wait for a buffer in
126                                          * ACCESS_CACHED, ACCESS_MODIFIED or
127                                          * ACCESS_EMPTY
128                                          * state. */
129  rtems_bdbuf_waiters transfer_waiters;  /**< Wait for a buffer in TRANSFER
130                                          * state. */
131  rtems_bdbuf_waiters buffer_waiters;    /**< Wait for a buffer and no one is
132                                          * available. */
133
134  size_t              group_count;       /**< The number of groups. */
135  rtems_bdbuf_group*  groups;            /**< The groups. */
136  rtems_id            read_ahead_task;   /**< Read-ahead task */
137  rtems_chain_control read_ahead_chain;  /**< Read-ahead request chain */
138  bool                read_ahead_enabled; /**< Read-ahead enabled */
139
140  bool                initialised;       /**< Initialised state. */
141} rtems_bdbuf_cache;
142
143/**
144 * Fatal errors
145 */
146#define RTEMS_BLKDEV_FATAL_ERROR(n) \
147  (((uint32_t)'B' << 24) | ((uint32_t)(n) & (uint32_t)0x00FFFFFF))
148
149#define RTEMS_BLKDEV_FATAL_BDBUF_STATE_11      RTEMS_BLKDEV_FATAL_ERROR(1)
150#define RTEMS_BLKDEV_FATAL_BDBUF_STATE_4       RTEMS_BLKDEV_FATAL_ERROR(2)
151#define RTEMS_BLKDEV_FATAL_BDBUF_STATE_5       RTEMS_BLKDEV_FATAL_ERROR(3)
152#define RTEMS_BLKDEV_FATAL_BDBUF_STATE_6       RTEMS_BLKDEV_FATAL_ERROR(4)
153#define RTEMS_BLKDEV_FATAL_BDBUF_STATE_7       RTEMS_BLKDEV_FATAL_ERROR(5)
154#define RTEMS_BLKDEV_FATAL_BDBUF_STATE_8       RTEMS_BLKDEV_FATAL_ERROR(6)
155#define RTEMS_BLKDEV_FATAL_BDBUF_STATE_9       RTEMS_BLKDEV_FATAL_ERROR(7)
156#define RTEMS_BLKDEV_FATAL_BDBUF_STATE_10      RTEMS_BLKDEV_FATAL_ERROR(8)
157#define RTEMS_BLKDEV_FATAL_BDBUF_TREE_RM       RTEMS_BLKDEV_FATAL_ERROR(9)
158#define RTEMS_BLKDEV_FATAL_BDBUF_SWAPOUT       RTEMS_BLKDEV_FATAL_ERROR(10)
159
160/*
161 * The lock/unlock fatal errors occur in case the bdbuf is not initialized with
162 * rtems_bdbuf_init().  General system corruption like stack overflow etc. may
163 * also trigger these fatal errors.
164 */
165#define RTEMS_BLKDEV_FATAL_BDBUF_SYNC_LOCK     RTEMS_BLKDEV_FATAL_ERROR(11)
166#define RTEMS_BLKDEV_FATAL_BDBUF_SYNC_UNLOCK   RTEMS_BLKDEV_FATAL_ERROR(12)
167#define RTEMS_BLKDEV_FATAL_BDBUF_CACHE_LOCK    RTEMS_BLKDEV_FATAL_ERROR(13)
168#define RTEMS_BLKDEV_FATAL_BDBUF_CACHE_UNLOCK  RTEMS_BLKDEV_FATAL_ERROR(14)
169
170#define RTEMS_BLKDEV_FATAL_BDBUF_PREEMPT_DIS   RTEMS_BLKDEV_FATAL_ERROR(15)
171#define RTEMS_BLKDEV_FATAL_BDBUF_CACHE_WAIT_2  RTEMS_BLKDEV_FATAL_ERROR(16)
172#define RTEMS_BLKDEV_FATAL_BDBUF_PREEMPT_RST   RTEMS_BLKDEV_FATAL_ERROR(17)
173#define RTEMS_BLKDEV_FATAL_BDBUF_CACHE_WAIT_TO RTEMS_BLKDEV_FATAL_ERROR(18)
174#define RTEMS_BLKDEV_FATAL_BDBUF_CACHE_WAKE    RTEMS_BLKDEV_FATAL_ERROR(19)
175#define RTEMS_BLKDEV_FATAL_BDBUF_SO_WAKE       RTEMS_BLKDEV_FATAL_ERROR(20)
176#define RTEMS_BLKDEV_FATAL_BDBUF_SO_NOMEM      RTEMS_BLKDEV_FATAL_ERROR(21)
177#define RTEMS_BLKDEV_FATAL_BDBUF_SO_WK_CREATE  RTEMS_BLKDEV_FATAL_ERROR(22)
178#define BLKDEV_FATAL_BDBUF_SWAPOUT_RE          RTEMS_BLKDEV_FATAL_ERROR(24)
179#define BLKDEV_FATAL_BDBUF_SWAPOUT_TS          RTEMS_BLKDEV_FATAL_ERROR(25)
180#define RTEMS_BLKDEV_FATAL_BDBUF_WAIT_EVNT     RTEMS_BLKDEV_FATAL_ERROR(26)
181#define RTEMS_BLKDEV_FATAL_BDBUF_RECYCLE       RTEMS_BLKDEV_FATAL_ERROR(27)
182#define RTEMS_BLKDEV_FATAL_BDBUF_STATE_0       RTEMS_BLKDEV_FATAL_ERROR(28)
183#define RTEMS_BLKDEV_FATAL_BDBUF_STATE_1       RTEMS_BLKDEV_FATAL_ERROR(29)
184#define RTEMS_BLKDEV_FATAL_BDBUF_STATE_2       RTEMS_BLKDEV_FATAL_ERROR(30)
185#define RTEMS_BLKDEV_FATAL_BDBUF_RA_WAKE_UP    RTEMS_BLKDEV_FATAL_ERROR(31)
186
187/**
188 * The events used in this code. These should be system events rather than
189 * application events.
190 */
191#define RTEMS_BDBUF_TRANSFER_SYNC  RTEMS_EVENT_1
192#define RTEMS_BDBUF_SWAPOUT_SYNC   RTEMS_EVENT_2
193#define RTEMS_BDBUF_READ_AHEAD_WAKE_UP RTEMS_EVENT_1
194
195/**
196 * Lock semaphore attributes. This is used for locking type mutexes.
197 *
198 * @warning Priority inheritance is on.
199 */
200#define RTEMS_BDBUF_CACHE_LOCK_ATTRIBS \
201  (RTEMS_PRIORITY | RTEMS_BINARY_SEMAPHORE | \
202   RTEMS_INHERIT_PRIORITY | RTEMS_NO_PRIORITY_CEILING | RTEMS_LOCAL)
203
204/**
205 * Waiter semaphore attributes.
206 *
207 * @warning Do not configure as inherit priority. If a driver is in the driver
208 *          initialisation table this locked semaphore will have the IDLE task
209 *          as the holder and a blocking task will raise the priority of the
210 *          IDLE task which can cause unsual side effects.
211 */
212#define RTEMS_BDBUF_CACHE_WAITER_ATTRIBS \
213  (RTEMS_PRIORITY | RTEMS_SIMPLE_BINARY_SEMAPHORE | \
214   RTEMS_NO_INHERIT_PRIORITY | RTEMS_NO_PRIORITY_CEILING | RTEMS_LOCAL)
215
216/**
217 * Waiter timeout. Set to non-zero to find some info on a waiter that is
218 * waiting too long.
219 */
220#define RTEMS_BDBUF_WAIT_TIMEOUT RTEMS_NO_TIMEOUT
221#if !defined (RTEMS_BDBUF_WAIT_TIMEOUT)
222#define RTEMS_BDBUF_WAIT_TIMEOUT \
223  (TOD_MICROSECONDS_TO_TICKS (20000000))
224#endif
225
226static rtems_task rtems_bdbuf_swapout_task(rtems_task_argument arg);
227
228static rtems_task rtems_bdbuf_read_ahead_task(rtems_task_argument arg);
229
230/**
231 * The Buffer Descriptor cache.
232 */
233static rtems_bdbuf_cache bdbuf_cache;
234
235#if RTEMS_BDBUF_TRACE
236/**
237 * If true output the trace message.
238 */
239bool rtems_bdbuf_tracer;
240
241/**
242 * Return the number of items on the list.
243 *
244 * @param list The chain control.
245 * @return uint32_t The number of items on the list.
246 */
247uint32_t
248rtems_bdbuf_list_count (rtems_chain_control* list)
249{
250  rtems_chain_node* node = rtems_chain_first (list);
251  uint32_t          count = 0;
252  while (!rtems_chain_is_tail (list, node))
253  {
254    count++;
255    node = rtems_chain_next (node);
256  }
257  return count;
258}
259
260/**
261 * Show the usage for the bdbuf cache.
262 */
263void
264rtems_bdbuf_show_usage (void)
265{
266  uint32_t group;
267  uint32_t total = 0;
268  uint32_t val;
269
270  for (group = 0; group < bdbuf_cache.group_count; group++)
271    total += bdbuf_cache.groups[group].users;
272  printf ("bdbuf:group users=%lu", total);
273  val = rtems_bdbuf_list_count (&bdbuf_cache.lru);
274  printf (", lru=%lu", val);
275  total = val;
276  val = rtems_bdbuf_list_count (&bdbuf_cache.modified);
277  printf (", mod=%lu", val);
278  total += val;
279  val = rtems_bdbuf_list_count (&bdbuf_cache.sync);
280  printf (", sync=%lu", val);
281  total += val;
282  printf (", total=%lu\n", total);
283}
284
285/**
286 * Show the users for a group of a bd.
287 *
288 * @param where A label to show the context of output.
289 * @param bd The bd to show the users of.
290 */
291void
292rtems_bdbuf_show_users (const char* where, rtems_bdbuf_buffer* bd)
293{
294  const char* states[] =
295    { "FR", "EM", "CH", "AC", "AM", "AE", "AP", "MD", "SY", "TR", "TP" };
296
297  printf ("bdbuf:users: %15s: [%" PRIu32 " (%s)] %td:%td = %" PRIu32 " %s\n",
298          where,
299          bd->block, states[bd->state],
300          bd->group - bdbuf_cache.groups,
301          bd - bdbuf_cache.bds,
302          bd->group->users,
303          bd->group->users > 8 ? "<<<<<<<" : "");
304}
305#else
306#define rtems_bdbuf_tracer (0)
307#define rtems_bdbuf_show_usage() ((void) 0)
308#define rtems_bdbuf_show_users(_w, _b) ((void) 0)
309#endif
310
311/**
312 * The default maximum height of 32 allows for AVL trees having between
313 * 5,704,880 and 4,294,967,295 nodes, depending on order of insertion.  You may
314 * change this compile-time constant as you wish.
315 */
316#ifndef RTEMS_BDBUF_AVL_MAX_HEIGHT
317#define RTEMS_BDBUF_AVL_MAX_HEIGHT (32)
318#endif
319
320static void
321rtems_bdbuf_fatal (rtems_bdbuf_buf_state state, uint32_t error)
322{
323  rtems_fatal_error_occurred ((((uint32_t) state) << 16) | error);
324}
325
326/**
327 * Searches for the node with specified dd/block.
328 *
329 * @param root pointer to the root node of the AVL-Tree
330 * @param dd disk device search key
331 * @param block block search key
332 * @retval NULL node with the specified dd/block is not found
333 * @return pointer to the node with specified dd/block
334 */
335static rtems_bdbuf_buffer *
336rtems_bdbuf_avl_search (rtems_bdbuf_buffer** root,
337                        const rtems_disk_device *dd,
338                        rtems_blkdev_bnum    block)
339{
340  rtems_bdbuf_buffer* p = *root;
341
342  while ((p != NULL) && ((p->dd != dd) || (p->block != block)))
343  {
344    if (((uintptr_t) p->dd < (uintptr_t) dd)
345        || ((p->dd == dd) && (p->block < block)))
346    {
347      p = p->avl.right;
348    }
349    else
350    {
351      p = p->avl.left;
352    }
353  }
354
355  return p;
356}
357
358/**
359 * Inserts the specified node to the AVl-Tree.
360 *
361 * @param root pointer to the root node of the AVL-Tree
362 * @param node Pointer to the node to add.
363 * @retval 0 The node added successfully
364 * @retval -1 An error occured
365 */
366static int
367rtems_bdbuf_avl_insert(rtems_bdbuf_buffer** root,
368                       rtems_bdbuf_buffer*  node)
369{
370  const rtems_disk_device *dd = node->dd;
371  rtems_blkdev_bnum block = node->block;
372
373  rtems_bdbuf_buffer*  p = *root;
374  rtems_bdbuf_buffer*  q;
375  rtems_bdbuf_buffer*  p1;
376  rtems_bdbuf_buffer*  p2;
377  rtems_bdbuf_buffer*  buf_stack[RTEMS_BDBUF_AVL_MAX_HEIGHT];
378  rtems_bdbuf_buffer** buf_prev = buf_stack;
379
380  bool modified = false;
381
382  if (p == NULL)
383  {
384    *root = node;
385    node->avl.left = NULL;
386    node->avl.right = NULL;
387    node->avl.bal = 0;
388    return 0;
389  }
390
391  while (p != NULL)
392  {
393    *buf_prev++ = p;
394
395    if (((uintptr_t) p->dd < (uintptr_t) dd)
396        || ((p->dd == dd) && (p->block < block)))
397    {
398      p->avl.cache = 1;
399      q = p->avl.right;
400      if (q == NULL)
401      {
402        q = node;
403        p->avl.right = q = node;
404        break;
405      }
406    }
407    else if ((p->dd != dd) || (p->block != block))
408    {
409      p->avl.cache = -1;
410      q = p->avl.left;
411      if (q == NULL)
412      {
413        q = node;
414        p->avl.left = q;
415        break;
416      }
417    }
418    else
419    {
420      return -1;
421    }
422
423    p = q;
424  }
425
426  q->avl.left = q->avl.right = NULL;
427  q->avl.bal = 0;
428  modified = true;
429  buf_prev--;
430
431  while (modified)
432  {
433    if (p->avl.cache == -1)
434    {
435      switch (p->avl.bal)
436      {
437        case 1:
438          p->avl.bal = 0;
439          modified = false;
440          break;
441
442        case 0:
443          p->avl.bal = -1;
444          break;
445
446        case -1:
447          p1 = p->avl.left;
448          if (p1->avl.bal == -1) /* simple LL-turn */
449          {
450            p->avl.left = p1->avl.right;
451            p1->avl.right = p;
452            p->avl.bal = 0;
453            p = p1;
454          }
455          else /* double LR-turn */
456          {
457            p2 = p1->avl.right;
458            p1->avl.right = p2->avl.left;
459            p2->avl.left = p1;
460            p->avl.left = p2->avl.right;
461            p2->avl.right = p;
462            if (p2->avl.bal == -1) p->avl.bal = +1; else p->avl.bal = 0;
463            if (p2->avl.bal == +1) p1->avl.bal = -1; else p1->avl.bal = 0;
464            p = p2;
465          }
466          p->avl.bal = 0;
467          modified = false;
468          break;
469
470        default:
471          break;
472      }
473    }
474    else
475    {
476      switch (p->avl.bal)
477      {
478        case -1:
479          p->avl.bal = 0;
480          modified = false;
481          break;
482
483        case 0:
484          p->avl.bal = 1;
485          break;
486
487        case 1:
488          p1 = p->avl.right;
489          if (p1->avl.bal == 1) /* simple RR-turn */
490          {
491            p->avl.right = p1->avl.left;
492            p1->avl.left = p;
493            p->avl.bal = 0;
494            p = p1;
495          }
496          else /* double RL-turn */
497          {
498            p2 = p1->avl.left;
499            p1->avl.left = p2->avl.right;
500            p2->avl.right = p1;
501            p->avl.right = p2->avl.left;
502            p2->avl.left = p;
503            if (p2->avl.bal == +1) p->avl.bal = -1; else p->avl.bal = 0;
504            if (p2->avl.bal == -1) p1->avl.bal = +1; else p1->avl.bal = 0;
505            p = p2;
506          }
507          p->avl.bal = 0;
508          modified = false;
509          break;
510
511        default:
512          break;
513      }
514    }
515    q = p;
516    if (buf_prev > buf_stack)
517    {
518      p = *--buf_prev;
519
520      if (p->avl.cache == -1)
521      {
522        p->avl.left = q;
523      }
524      else
525      {
526        p->avl.right = q;
527      }
528    }
529    else
530    {
531      *root = p;
532      break;
533    }
534  };
535
536  return 0;
537}
538
539
540/**
541 * Removes the node from the tree.
542 *
543 * @param root Pointer to pointer to the root node
544 * @param node Pointer to the node to remove
545 * @retval 0 Item removed
546 * @retval -1 No such item found
547 */
548static int
549rtems_bdbuf_avl_remove(rtems_bdbuf_buffer**      root,
550                       const rtems_bdbuf_buffer* node)
551{
552  const rtems_disk_device *dd = node->dd;
553  rtems_blkdev_bnum block = node->block;
554
555  rtems_bdbuf_buffer*  p = *root;
556  rtems_bdbuf_buffer*  q;
557  rtems_bdbuf_buffer*  r;
558  rtems_bdbuf_buffer*  s;
559  rtems_bdbuf_buffer*  p1;
560  rtems_bdbuf_buffer*  p2;
561  rtems_bdbuf_buffer*  buf_stack[RTEMS_BDBUF_AVL_MAX_HEIGHT];
562  rtems_bdbuf_buffer** buf_prev = buf_stack;
563
564  bool modified = false;
565
566  memset (buf_stack, 0, sizeof(buf_stack));
567
568  while (p != NULL)
569  {
570    *buf_prev++ = p;
571
572    if (((uintptr_t) p->dd < (uintptr_t) dd)
573        || ((p->dd == dd) && (p->block < block)))
574    {
575      p->avl.cache = 1;
576      p = p->avl.right;
577    }
578    else if ((p->dd != dd) || (p->block != block))
579    {
580      p->avl.cache = -1;
581      p = p->avl.left;
582    }
583    else
584    {
585      /* node found */
586      break;
587    }
588  }
589
590  if (p == NULL)
591  {
592    /* there is no such node */
593    return -1;
594  }
595
596  q = p;
597
598  buf_prev--;
599  if (buf_prev > buf_stack)
600  {
601    p = *(buf_prev - 1);
602  }
603  else
604  {
605    p = NULL;
606  }
607
608  /* at this moment q - is a node to delete, p is q's parent */
609  if (q->avl.right == NULL)
610  {
611    r = q->avl.left;
612    if (r != NULL)
613    {
614      r->avl.bal = 0;
615    }
616    q = r;
617  }
618  else
619  {
620    rtems_bdbuf_buffer **t;
621
622    r = q->avl.right;
623
624    if (r->avl.left == NULL)
625    {
626      r->avl.left = q->avl.left;
627      r->avl.bal = q->avl.bal;
628      r->avl.cache = 1;
629      *buf_prev++ = q = r;
630    }
631    else
632    {
633      t = buf_prev++;
634      s = r;
635
636      while (s->avl.left != NULL)
637      {
638        *buf_prev++ = r = s;
639        s = r->avl.left;
640        r->avl.cache = -1;
641      }
642
643      s->avl.left = q->avl.left;
644      r->avl.left = s->avl.right;
645      s->avl.right = q->avl.right;
646      s->avl.bal = q->avl.bal;
647      s->avl.cache = 1;
648
649      *t = q = s;
650    }
651  }
652
653  if (p != NULL)
654  {
655    if (p->avl.cache == -1)
656    {
657      p->avl.left = q;
658    }
659    else
660    {
661      p->avl.right = q;
662    }
663  }
664  else
665  {
666    *root = q;
667  }
668
669  modified = true;
670
671  while (modified)
672  {
673    if (buf_prev > buf_stack)
674    {
675      p = *--buf_prev;
676    }
677    else
678    {
679      break;
680    }
681
682    if (p->avl.cache == -1)
683    {
684      /* rebalance left branch */
685      switch (p->avl.bal)
686      {
687        case -1:
688          p->avl.bal = 0;
689          break;
690        case  0:
691          p->avl.bal = 1;
692          modified = false;
693          break;
694
695        case +1:
696          p1 = p->avl.right;
697
698          if (p1->avl.bal >= 0) /* simple RR-turn */
699          {
700            p->avl.right = p1->avl.left;
701            p1->avl.left = p;
702
703            if (p1->avl.bal == 0)
704            {
705              p1->avl.bal = -1;
706              modified = false;
707            }
708            else
709            {
710              p->avl.bal = 0;
711              p1->avl.bal = 0;
712            }
713            p = p1;
714          }
715          else /* double RL-turn */
716          {
717            p2 = p1->avl.left;
718
719            p1->avl.left = p2->avl.right;
720            p2->avl.right = p1;
721            p->avl.right = p2->avl.left;
722            p2->avl.left = p;
723
724            if (p2->avl.bal == +1) p->avl.bal = -1; else p->avl.bal = 0;
725            if (p2->avl.bal == -1) p1->avl.bal = 1; else p1->avl.bal = 0;
726
727            p = p2;
728            p2->avl.bal = 0;
729          }
730          break;
731
732        default:
733          break;
734      }
735    }
736    else
737    {
738      /* rebalance right branch */
739      switch (p->avl.bal)
740      {
741        case +1:
742          p->avl.bal = 0;
743          break;
744
745        case  0:
746          p->avl.bal = -1;
747          modified = false;
748          break;
749
750        case -1:
751          p1 = p->avl.left;
752
753          if (p1->avl.bal <= 0) /* simple LL-turn */
754          {
755            p->avl.left = p1->avl.right;
756            p1->avl.right = p;
757            if (p1->avl.bal == 0)
758            {
759              p1->avl.bal = 1;
760              modified = false;
761            }
762            else
763            {
764              p->avl.bal = 0;
765              p1->avl.bal = 0;
766            }
767            p = p1;
768          }
769          else /* double LR-turn */
770          {
771            p2 = p1->avl.right;
772
773            p1->avl.right = p2->avl.left;
774            p2->avl.left = p1;
775            p->avl.left = p2->avl.right;
776            p2->avl.right = p;
777
778            if (p2->avl.bal == -1) p->avl.bal = 1; else p->avl.bal = 0;
779            if (p2->avl.bal == +1) p1->avl.bal = -1; else p1->avl.bal = 0;
780
781            p = p2;
782            p2->avl.bal = 0;
783          }
784          break;
785
786        default:
787          break;
788      }
789    }
790
791    if (buf_prev > buf_stack)
792    {
793      q = *(buf_prev - 1);
794
795      if (q->avl.cache == -1)
796      {
797        q->avl.left = p;
798      }
799      else
800      {
801        q->avl.right = p;
802      }
803    }
804    else
805    {
806      *root = p;
807      break;
808    }
809
810  }
811
812  return 0;
813}
814
815static void
816rtems_bdbuf_set_state (rtems_bdbuf_buffer *bd, rtems_bdbuf_buf_state state)
817{
818  bd->state = state;
819}
820
821static rtems_blkdev_bnum
822rtems_bdbuf_media_block (const rtems_disk_device *dd, rtems_blkdev_bnum block)
823{
824  if (dd->block_to_media_block_shift >= 0)
825    return block << dd->block_to_media_block_shift;
826  else
827    /*
828     * Change the block number for the block size to the block number for the media
829     * block size. We have to use 64bit maths. There is no short cut here.
830     */
831    return (rtems_blkdev_bnum)
832      ((((uint64_t) block) * dd->block_size) / dd->media_block_size);
833}
834
835/**
836 * Lock the mutex. A single task can nest calls.
837 *
838 * @param lock The mutex to lock.
839 * @param fatal_error_code The error code if the call fails.
840 */
841static void
842rtems_bdbuf_lock (rtems_id lock, uint32_t fatal_error_code)
843{
844  rtems_status_code sc = rtems_semaphore_obtain (lock,
845                                                 RTEMS_WAIT,
846                                                 RTEMS_NO_TIMEOUT);
847  if (sc != RTEMS_SUCCESSFUL)
848    rtems_fatal_error_occurred (fatal_error_code);
849}
850
851/**
852 * Unlock the mutex.
853 *
854 * @param lock The mutex to unlock.
855 * @param fatal_error_code The error code if the call fails.
856 */
857static void
858rtems_bdbuf_unlock (rtems_id lock, uint32_t fatal_error_code)
859{
860  rtems_status_code sc = rtems_semaphore_release (lock);
861  if (sc != RTEMS_SUCCESSFUL)
862    rtems_fatal_error_occurred (fatal_error_code);
863}
864
865/**
866 * Lock the cache. A single task can nest calls.
867 */
868static void
869rtems_bdbuf_lock_cache (void)
870{
871  rtems_bdbuf_lock (bdbuf_cache.lock, RTEMS_BLKDEV_FATAL_BDBUF_CACHE_LOCK);
872}
873
874/**
875 * Unlock the cache.
876 */
877static void
878rtems_bdbuf_unlock_cache (void)
879{
880  rtems_bdbuf_unlock (bdbuf_cache.lock, RTEMS_BLKDEV_FATAL_BDBUF_CACHE_UNLOCK);
881}
882
883/**
884 * Lock the cache's sync. A single task can nest calls.
885 */
886static void
887rtems_bdbuf_lock_sync (void)
888{
889  rtems_bdbuf_lock (bdbuf_cache.sync_lock, RTEMS_BLKDEV_FATAL_BDBUF_SYNC_LOCK);
890}
891
892/**
893 * Unlock the cache's sync lock. Any blocked writers are woken.
894 */
895static void
896rtems_bdbuf_unlock_sync (void)
897{
898  rtems_bdbuf_unlock (bdbuf_cache.sync_lock,
899                      RTEMS_BLKDEV_FATAL_BDBUF_SYNC_UNLOCK);
900}
901
902static void
903rtems_bdbuf_group_obtain (rtems_bdbuf_buffer *bd)
904{
905  ++bd->group->users;
906}
907
908static void
909rtems_bdbuf_group_release (rtems_bdbuf_buffer *bd)
910{
911  --bd->group->users;
912}
913
914static rtems_mode
915rtems_bdbuf_disable_preemption (void)
916{
917  rtems_status_code sc = RTEMS_SUCCESSFUL;
918  rtems_mode prev_mode = 0;
919
920  sc = rtems_task_mode (RTEMS_NO_PREEMPT, RTEMS_PREEMPT_MASK, &prev_mode);
921  if (sc != RTEMS_SUCCESSFUL)
922    rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_PREEMPT_DIS);
923
924  return prev_mode;
925}
926
927static void
928rtems_bdbuf_restore_preemption (rtems_mode prev_mode)
929{
930  rtems_status_code sc = RTEMS_SUCCESSFUL;
931
932  sc = rtems_task_mode (prev_mode, RTEMS_ALL_MODE_MASKS, &prev_mode);
933  if (sc != RTEMS_SUCCESSFUL)
934    rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_PREEMPT_RST);
935}
936
937/**
938 * Wait until woken. Semaphores are used so a number of tasks can wait and can
939 * be woken at once. Task events would require we maintain a list of tasks to
940 * be woken and this would require storage and we do not know the number of
941 * tasks that could be waiting.
942 *
943 * While we have the cache locked we can try and claim the semaphore and
944 * therefore know when we release the lock to the cache we will block until the
945 * semaphore is released. This may even happen before we get to block.
946 *
947 * A counter is used to save the release call when no one is waiting.
948 *
949 * The function assumes the cache is locked on entry and it will be locked on
950 * exit.
951 */
952static void
953rtems_bdbuf_anonymous_wait (rtems_bdbuf_waiters *waiters)
954{
955  rtems_status_code sc;
956  rtems_mode        prev_mode;
957
958  /*
959   * Indicate we are waiting.
960   */
961  ++waiters->count;
962
963  /*
964   * Disable preemption then unlock the cache and block.  There is no POSIX
965   * condition variable in the core API so this is a work around.
966   *
967   * The issue is a task could preempt after the cache is unlocked because it is
968   * blocking or just hits that window, and before this task has blocked on the
969   * semaphore. If the preempting task flushes the queue this task will not see
970   * the flush and may block for ever or until another transaction flushes this
971   * semaphore.
972   */
973  prev_mode = rtems_bdbuf_disable_preemption ();
974
975  /*
976   * Unlock the cache, wait, and lock the cache when we return.
977   */
978  rtems_bdbuf_unlock_cache ();
979
980  sc = rtems_semaphore_obtain (waiters->sema, RTEMS_WAIT, RTEMS_BDBUF_WAIT_TIMEOUT);
981
982  if (sc == RTEMS_TIMEOUT)
983    rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_CACHE_WAIT_TO);
984
985  if (sc != RTEMS_UNSATISFIED)
986    rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_CACHE_WAIT_2);
987
988  rtems_bdbuf_lock_cache ();
989
990  rtems_bdbuf_restore_preemption (prev_mode);
991
992  --waiters->count;
993}
994
995static void
996rtems_bdbuf_wait (rtems_bdbuf_buffer *bd, rtems_bdbuf_waiters *waiters)
997{
998  rtems_bdbuf_group_obtain (bd);
999  ++bd->waiters;
1000  rtems_bdbuf_anonymous_wait (waiters);
1001  --bd->waiters;
1002  rtems_bdbuf_group_release (bd);
1003}
1004
1005/**
1006 * Wake a blocked resource. The resource has a counter that lets us know if
1007 * there are any waiters.
1008 */
1009static void
1010rtems_bdbuf_wake (const rtems_bdbuf_waiters *waiters)
1011{
1012  rtems_status_code sc = RTEMS_SUCCESSFUL;
1013
1014  if (waiters->count > 0)
1015  {
1016    sc = rtems_semaphore_flush (waiters->sema);
1017    if (sc != RTEMS_SUCCESSFUL)
1018      rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_CACHE_WAKE);
1019  }
1020}
1021
1022static void
1023rtems_bdbuf_wake_swapper (void)
1024{
1025  rtems_status_code sc = rtems_event_send (bdbuf_cache.swapout,
1026                                           RTEMS_BDBUF_SWAPOUT_SYNC);
1027  if (sc != RTEMS_SUCCESSFUL)
1028    rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_SO_WAKE);
1029}
1030
1031static bool
1032rtems_bdbuf_has_buffer_waiters (void)
1033{
1034  return bdbuf_cache.buffer_waiters.count;
1035}
1036
1037static void
1038rtems_bdbuf_remove_from_tree (rtems_bdbuf_buffer *bd)
1039{
1040  if (rtems_bdbuf_avl_remove (&bdbuf_cache.tree, bd) != 0)
1041    rtems_bdbuf_fatal (bd->state, RTEMS_BLKDEV_FATAL_BDBUF_TREE_RM);
1042}
1043
1044static void
1045rtems_bdbuf_remove_from_tree_and_lru_list (rtems_bdbuf_buffer *bd)
1046{
1047  switch (bd->state)
1048  {
1049    case RTEMS_BDBUF_STATE_FREE:
1050      break;
1051    case RTEMS_BDBUF_STATE_CACHED:
1052      rtems_bdbuf_remove_from_tree (bd);
1053      break;
1054    default:
1055      rtems_bdbuf_fatal (bd->state, RTEMS_BLKDEV_FATAL_BDBUF_STATE_10);
1056  }
1057
1058  rtems_chain_extract_unprotected (&bd->link);
1059}
1060
1061static void
1062rtems_bdbuf_make_free_and_add_to_lru_list (rtems_bdbuf_buffer *bd)
1063{
1064  rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_FREE);
1065  rtems_chain_prepend_unprotected (&bdbuf_cache.lru, &bd->link);
1066}
1067
1068static void
1069rtems_bdbuf_make_empty (rtems_bdbuf_buffer *bd)
1070{
1071  rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_EMPTY);
1072}
1073
1074static void
1075rtems_bdbuf_make_cached_and_add_to_lru_list (rtems_bdbuf_buffer *bd)
1076{
1077  rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_CACHED);
1078  rtems_chain_append_unprotected (&bdbuf_cache.lru, &bd->link);
1079}
1080
1081static void
1082rtems_bdbuf_discard_buffer (rtems_bdbuf_buffer *bd)
1083{
1084  rtems_bdbuf_make_empty (bd);
1085
1086  if (bd->waiters == 0)
1087  {
1088    rtems_bdbuf_remove_from_tree (bd);
1089    rtems_bdbuf_make_free_and_add_to_lru_list (bd);
1090  }
1091}
1092
1093static void
1094rtems_bdbuf_add_to_modified_list_after_access (rtems_bdbuf_buffer *bd)
1095{
1096  if (bdbuf_cache.sync_active && bdbuf_cache.sync_device == bd->dd)
1097  {
1098    rtems_bdbuf_unlock_cache ();
1099
1100    /*
1101     * Wait for the sync lock.
1102     */
1103    rtems_bdbuf_lock_sync ();
1104
1105    rtems_bdbuf_unlock_sync ();
1106    rtems_bdbuf_lock_cache ();
1107  }
1108
1109  /*
1110   * Only the first modified release sets the timer and any further user
1111   * accesses do not change the timer value which should move down. This
1112   * assumes the user's hold of the buffer is much less than the time on the
1113   * modified list. Resetting the timer on each access which could result in a
1114   * buffer never getting to 0 and never being forced onto disk. This raises a
1115   * difficult question. Is a snapshot of a block that is changing better than
1116   * nothing being written? We have tended to think we should hold changes for
1117   * only a specific period of time even if still changing and get onto disk
1118   * and letting the file system try and recover this position if it can.
1119   */
1120  if (bd->state == RTEMS_BDBUF_STATE_ACCESS_CACHED
1121        || bd->state == RTEMS_BDBUF_STATE_ACCESS_EMPTY)
1122    bd->hold_timer = bdbuf_config.swap_block_hold;
1123
1124  rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_MODIFIED);
1125  rtems_chain_append_unprotected (&bdbuf_cache.modified, &bd->link);
1126
1127  if (bd->waiters)
1128    rtems_bdbuf_wake (&bdbuf_cache.access_waiters);
1129  else if (rtems_bdbuf_has_buffer_waiters ())
1130    rtems_bdbuf_wake_swapper ();
1131}
1132
1133static void
1134rtems_bdbuf_add_to_lru_list_after_access (rtems_bdbuf_buffer *bd)
1135{
1136  rtems_bdbuf_group_release (bd);
1137  rtems_bdbuf_make_cached_and_add_to_lru_list (bd);
1138
1139  if (bd->waiters)
1140    rtems_bdbuf_wake (&bdbuf_cache.access_waiters);
1141  else
1142    rtems_bdbuf_wake (&bdbuf_cache.buffer_waiters);
1143}
1144
1145/**
1146 * Compute the number of BDs per group for a given buffer size.
1147 *
1148 * @param size The buffer size. It can be any size and we scale up.
1149 */
1150static size_t
1151rtems_bdbuf_bds_per_group (size_t size)
1152{
1153  size_t bufs_per_size;
1154  size_t bds_per_size;
1155
1156  if (size > bdbuf_config.buffer_max)
1157    return 0;
1158
1159  bufs_per_size = ((size - 1) / bdbuf_config.buffer_min) + 1;
1160
1161  for (bds_per_size = 1;
1162       bds_per_size < bufs_per_size;
1163       bds_per_size <<= 1)
1164    ;
1165
1166  return bdbuf_cache.max_bds_per_group / bds_per_size;
1167}
1168
1169static void
1170rtems_bdbuf_discard_buffer_after_access (rtems_bdbuf_buffer *bd)
1171{
1172  rtems_bdbuf_group_release (bd);
1173  rtems_bdbuf_discard_buffer (bd);
1174
1175  if (bd->waiters)
1176    rtems_bdbuf_wake (&bdbuf_cache.access_waiters);
1177  else
1178    rtems_bdbuf_wake (&bdbuf_cache.buffer_waiters);
1179}
1180
1181/**
1182 * Reallocate a group. The BDs currently allocated in the group are removed
1183 * from the ALV tree and any lists then the new BD's are prepended to the ready
1184 * list of the cache.
1185 *
1186 * @param group The group to reallocate.
1187 * @param new_bds_per_group The new count of BDs per group.
1188 * @return A buffer of this group.
1189 */
1190static rtems_bdbuf_buffer *
1191rtems_bdbuf_group_realloc (rtems_bdbuf_group* group, size_t new_bds_per_group)
1192{
1193  rtems_bdbuf_buffer* bd;
1194  size_t              b;
1195  size_t              bufs_per_bd;
1196
1197  if (rtems_bdbuf_tracer)
1198    printf ("bdbuf:realloc: %tu: %zd -> %zd\n",
1199            group - bdbuf_cache.groups, group->bds_per_group,
1200            new_bds_per_group);
1201
1202  bufs_per_bd = bdbuf_cache.max_bds_per_group / group->bds_per_group;
1203
1204  for (b = 0, bd = group->bdbuf;
1205       b < group->bds_per_group;
1206       b++, bd += bufs_per_bd)
1207    rtems_bdbuf_remove_from_tree_and_lru_list (bd);
1208
1209  group->bds_per_group = new_bds_per_group;
1210  bufs_per_bd = bdbuf_cache.max_bds_per_group / new_bds_per_group;
1211
1212  for (b = 1, bd = group->bdbuf + bufs_per_bd;
1213       b < group->bds_per_group;
1214       b++, bd += bufs_per_bd)
1215    rtems_bdbuf_make_free_and_add_to_lru_list (bd);
1216
1217  if (b > 1)
1218    rtems_bdbuf_wake (&bdbuf_cache.buffer_waiters);
1219
1220  return group->bdbuf;
1221}
1222
1223static void
1224rtems_bdbuf_setup_empty_buffer (rtems_bdbuf_buffer *bd,
1225                                rtems_disk_device  *dd,
1226                                rtems_blkdev_bnum   block)
1227{
1228  bd->dd        = dd ;
1229  bd->block     = block;
1230  bd->avl.left  = NULL;
1231  bd->avl.right = NULL;
1232  bd->waiters   = 0;
1233
1234  if (rtems_bdbuf_avl_insert (&bdbuf_cache.tree, bd) != 0)
1235    rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_RECYCLE);
1236
1237  rtems_bdbuf_make_empty (bd);
1238}
1239
1240static rtems_bdbuf_buffer *
1241rtems_bdbuf_get_buffer_from_lru_list (rtems_disk_device *dd,
1242                                      rtems_blkdev_bnum  block)
1243{
1244  rtems_chain_node *node = rtems_chain_first (&bdbuf_cache.lru);
1245
1246  while (!rtems_chain_is_tail (&bdbuf_cache.lru, node))
1247  {
1248    rtems_bdbuf_buffer *bd = (rtems_bdbuf_buffer *) node;
1249    rtems_bdbuf_buffer *empty_bd = NULL;
1250
1251    if (rtems_bdbuf_tracer)
1252      printf ("bdbuf:next-bd: %tu (%td:%" PRId32 ") %zd -> %zd\n",
1253              bd - bdbuf_cache.bds,
1254              bd->group - bdbuf_cache.groups, bd->group->users,
1255              bd->group->bds_per_group, dd->bds_per_group);
1256
1257    /*
1258     * If nobody waits for this BD, we may recycle it.
1259     */
1260    if (bd->waiters == 0)
1261    {
1262      if (bd->group->bds_per_group == dd->bds_per_group)
1263      {
1264        rtems_bdbuf_remove_from_tree_and_lru_list (bd);
1265
1266        empty_bd = bd;
1267      }
1268      else if (bd->group->users == 0)
1269        empty_bd = rtems_bdbuf_group_realloc (bd->group, dd->bds_per_group);
1270    }
1271
1272    if (empty_bd != NULL)
1273    {
1274      rtems_bdbuf_setup_empty_buffer (empty_bd, dd, block);
1275
1276      return empty_bd;
1277    }
1278
1279    node = rtems_chain_next (node);
1280  }
1281
1282  return NULL;
1283}
1284
1285static rtems_status_code
1286rtems_bdbuf_create_task(
1287  rtems_name name,
1288  rtems_task_priority priority,
1289  rtems_task_priority default_priority,
1290  rtems_task_entry entry,
1291  rtems_task_argument arg,
1292  rtems_id *id
1293)
1294{
1295  rtems_status_code sc;
1296  size_t stack_size = bdbuf_config.task_stack_size ?
1297    bdbuf_config.task_stack_size : RTEMS_BDBUF_TASK_STACK_SIZE_DEFAULT;
1298
1299  priority = priority != 0 ? priority : default_priority;
1300
1301  sc = rtems_task_create (name,
1302                          priority,
1303                          stack_size,
1304                          RTEMS_PREEMPT | RTEMS_NO_TIMESLICE | RTEMS_NO_ASR,
1305                          RTEMS_LOCAL | RTEMS_NO_FLOATING_POINT,
1306                          id);
1307
1308  if (sc == RTEMS_SUCCESSFUL)
1309    sc = rtems_task_start (*id, entry, arg);
1310
1311  return sc;
1312}
1313
1314/**
1315 * Initialise the cache.
1316 *
1317 * @return rtems_status_code The initialisation status.
1318 */
1319rtems_status_code
1320rtems_bdbuf_init (void)
1321{
1322  rtems_bdbuf_group*  group;
1323  rtems_bdbuf_buffer* bd;
1324  uint8_t*            buffer;
1325  size_t              b;
1326  size_t              cache_aligment;
1327  rtems_status_code   sc;
1328  rtems_mode          prev_mode;
1329
1330  if (rtems_bdbuf_tracer)
1331    printf ("bdbuf:init\n");
1332
1333  if (rtems_interrupt_is_in_progress())
1334    return RTEMS_CALLED_FROM_ISR;
1335
1336  /*
1337   * Check the configuration table values.
1338   */
1339  if ((bdbuf_config.buffer_max % bdbuf_config.buffer_min) != 0)
1340    return RTEMS_INVALID_NUMBER;
1341
1342  /*
1343   * We use a special variable to manage the initialisation incase we have
1344   * completing threads doing this. You may get errors if the another thread
1345   * makes a call and we have not finished initialisation.
1346   */
1347  prev_mode = rtems_bdbuf_disable_preemption ();
1348  if (bdbuf_cache.initialised)
1349  {
1350    rtems_bdbuf_restore_preemption (prev_mode);
1351    return RTEMS_RESOURCE_IN_USE;
1352  }
1353
1354  memset(&bdbuf_cache, 0, sizeof(bdbuf_cache));
1355  bdbuf_cache.initialised = true;
1356  rtems_bdbuf_restore_preemption (prev_mode);
1357
1358  /*
1359   * For unspecified cache alignments we use the CPU alignment.
1360   */
1361  cache_aligment = 32; /* FIXME rtems_cache_get_data_line_size() */
1362  if (cache_aligment <= 0)
1363    cache_aligment = CPU_ALIGNMENT;
1364
1365  bdbuf_cache.sync_device = BDBUF_INVALID_DEV;
1366
1367  rtems_chain_initialize_empty (&bdbuf_cache.swapout_workers);
1368  rtems_chain_initialize_empty (&bdbuf_cache.lru);
1369  rtems_chain_initialize_empty (&bdbuf_cache.modified);
1370  rtems_chain_initialize_empty (&bdbuf_cache.sync);
1371  rtems_chain_initialize_empty (&bdbuf_cache.read_ahead_chain);
1372
1373  /*
1374   * Create the locks for the cache.
1375   */
1376  sc = rtems_semaphore_create (rtems_build_name ('B', 'D', 'C', 'l'),
1377                               1, RTEMS_BDBUF_CACHE_LOCK_ATTRIBS, 0,
1378                               &bdbuf_cache.lock);
1379  if (sc != RTEMS_SUCCESSFUL)
1380    goto error;
1381
1382  rtems_bdbuf_lock_cache ();
1383
1384  sc = rtems_semaphore_create (rtems_build_name ('B', 'D', 'C', 's'),
1385                               1, RTEMS_BDBUF_CACHE_LOCK_ATTRIBS, 0,
1386                               &bdbuf_cache.sync_lock);
1387  if (sc != RTEMS_SUCCESSFUL)
1388    goto error;
1389
1390  sc = rtems_semaphore_create (rtems_build_name ('B', 'D', 'C', 'a'),
1391                               0, RTEMS_BDBUF_CACHE_WAITER_ATTRIBS, 0,
1392                               &bdbuf_cache.access_waiters.sema);
1393  if (sc != RTEMS_SUCCESSFUL)
1394    goto error;
1395
1396  sc = rtems_semaphore_create (rtems_build_name ('B', 'D', 'C', 't'),
1397                               0, RTEMS_BDBUF_CACHE_WAITER_ATTRIBS, 0,
1398                               &bdbuf_cache.transfer_waiters.sema);
1399  if (sc != RTEMS_SUCCESSFUL)
1400    goto error;
1401
1402  sc = rtems_semaphore_create (rtems_build_name ('B', 'D', 'C', 'b'),
1403                               0, RTEMS_BDBUF_CACHE_WAITER_ATTRIBS, 0,
1404                               &bdbuf_cache.buffer_waiters.sema);
1405  if (sc != RTEMS_SUCCESSFUL)
1406    goto error;
1407
1408  /*
1409   * Compute the various number of elements in the cache.
1410   */
1411  bdbuf_cache.buffer_min_count =
1412    bdbuf_config.size / bdbuf_config.buffer_min;
1413  bdbuf_cache.max_bds_per_group =
1414    bdbuf_config.buffer_max / bdbuf_config.buffer_min;
1415  bdbuf_cache.group_count =
1416    bdbuf_cache.buffer_min_count / bdbuf_cache.max_bds_per_group;
1417
1418  /*
1419   * Allocate the memory for the buffer descriptors.
1420   */
1421  bdbuf_cache.bds = calloc (sizeof (rtems_bdbuf_buffer),
1422                            bdbuf_cache.buffer_min_count);
1423  if (!bdbuf_cache.bds)
1424    goto error;
1425
1426  /*
1427   * Allocate the memory for the buffer descriptors.
1428   */
1429  bdbuf_cache.groups = calloc (sizeof (rtems_bdbuf_group),
1430                               bdbuf_cache.group_count);
1431  if (!bdbuf_cache.groups)
1432    goto error;
1433
1434  /*
1435   * Allocate memory for buffer memory. The buffer memory will be cache
1436   * aligned. It is possible to free the memory allocated by rtems_memalign()
1437   * with free(). Return 0 if allocated.
1438   *
1439   * The memory allocate allows a
1440   */
1441  if (rtems_memalign ((void **) &bdbuf_cache.buffers,
1442                      cache_aligment,
1443                      bdbuf_cache.buffer_min_count * bdbuf_config.buffer_min) != 0)
1444    goto error;
1445
1446  /*
1447   * The cache is empty after opening so we need to add all the buffers to it
1448   * and initialise the groups.
1449   */
1450  for (b = 0, group = bdbuf_cache.groups,
1451         bd = bdbuf_cache.bds, buffer = bdbuf_cache.buffers;
1452       b < bdbuf_cache.buffer_min_count;
1453       b++, bd++, buffer += bdbuf_config.buffer_min)
1454  {
1455    bd->dd    = BDBUF_INVALID_DEV;
1456    bd->group  = group;
1457    bd->buffer = buffer;
1458
1459    rtems_chain_append_unprotected (&bdbuf_cache.lru, &bd->link);
1460
1461    if ((b % bdbuf_cache.max_bds_per_group) ==
1462        (bdbuf_cache.max_bds_per_group - 1))
1463      group++;
1464  }
1465
1466  for (b = 0,
1467         group = bdbuf_cache.groups,
1468         bd = bdbuf_cache.bds;
1469       b < bdbuf_cache.group_count;
1470       b++,
1471         group++,
1472         bd += bdbuf_cache.max_bds_per_group)
1473  {
1474    group->bds_per_group = bdbuf_cache.max_bds_per_group;
1475    group->bdbuf = bd;
1476  }
1477
1478  /*
1479   * Create and start swapout task. This task will create and manage the worker
1480   * threads.
1481   */
1482  bdbuf_cache.swapout_enabled = true;
1483
1484  sc = rtems_bdbuf_create_task (rtems_build_name('B', 'S', 'W', 'P'),
1485                                bdbuf_config.swapout_priority,
1486                                RTEMS_BDBUF_SWAPOUT_TASK_PRIORITY_DEFAULT,
1487                                rtems_bdbuf_swapout_task,
1488                                0,
1489                                &bdbuf_cache.swapout);
1490  if (sc != RTEMS_SUCCESSFUL)
1491    goto error;
1492
1493  if (bdbuf_config.max_read_ahead_blocks > 0)
1494  {
1495    bdbuf_cache.read_ahead_enabled = true;
1496    sc = rtems_bdbuf_create_task (rtems_build_name('B', 'R', 'D', 'A'),
1497                                  bdbuf_config.read_ahead_priority,
1498                                  RTEMS_BDBUF_READ_AHEAD_TASK_PRIORITY_DEFAULT,
1499                                  rtems_bdbuf_read_ahead_task,
1500                                  0,
1501                                  &bdbuf_cache.read_ahead_task);
1502    if (sc != RTEMS_SUCCESSFUL)
1503      goto error;
1504  }
1505
1506  rtems_bdbuf_unlock_cache ();
1507
1508  return RTEMS_SUCCESSFUL;
1509
1510error:
1511
1512  if (bdbuf_cache.read_ahead_task != 0)
1513    rtems_task_delete (bdbuf_cache.read_ahead_task);
1514
1515  if (bdbuf_cache.swapout != 0)
1516    rtems_task_delete (bdbuf_cache.swapout);
1517
1518  free (bdbuf_cache.buffers);
1519  free (bdbuf_cache.groups);
1520  free (bdbuf_cache.bds);
1521
1522  rtems_semaphore_delete (bdbuf_cache.buffer_waiters.sema);
1523  rtems_semaphore_delete (bdbuf_cache.access_waiters.sema);
1524  rtems_semaphore_delete (bdbuf_cache.transfer_waiters.sema);
1525  rtems_semaphore_delete (bdbuf_cache.sync_lock);
1526
1527  if (bdbuf_cache.lock != 0)
1528  {
1529    rtems_bdbuf_unlock_cache ();
1530    rtems_semaphore_delete (bdbuf_cache.lock);
1531  }
1532
1533  bdbuf_cache.initialised = false;
1534
1535  return RTEMS_UNSATISFIED;
1536}
1537
1538static void
1539rtems_bdbuf_wait_for_event (rtems_event_set event)
1540{
1541  rtems_status_code sc = RTEMS_SUCCESSFUL;
1542  rtems_event_set   out = 0;
1543
1544  sc = rtems_event_receive (event,
1545                            RTEMS_EVENT_ALL | RTEMS_WAIT,
1546                            RTEMS_NO_TIMEOUT,
1547                            &out);
1548
1549  if (sc != RTEMS_SUCCESSFUL || out != event)
1550    rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_WAIT_EVNT);
1551}
1552
1553static void
1554rtems_bdbuf_wait_for_access (rtems_bdbuf_buffer *bd)
1555{
1556  while (true)
1557  {
1558    switch (bd->state)
1559    {
1560      case RTEMS_BDBUF_STATE_MODIFIED:
1561        rtems_bdbuf_group_release (bd);
1562        /* Fall through */
1563      case RTEMS_BDBUF_STATE_CACHED:
1564        rtems_chain_extract_unprotected (&bd->link);
1565        /* Fall through */
1566      case RTEMS_BDBUF_STATE_EMPTY:
1567        return;
1568      case RTEMS_BDBUF_STATE_ACCESS_CACHED:
1569      case RTEMS_BDBUF_STATE_ACCESS_EMPTY:
1570      case RTEMS_BDBUF_STATE_ACCESS_MODIFIED:
1571      case RTEMS_BDBUF_STATE_ACCESS_PURGED:
1572        rtems_bdbuf_wait (bd, &bdbuf_cache.access_waiters);
1573        break;
1574      case RTEMS_BDBUF_STATE_SYNC:
1575      case RTEMS_BDBUF_STATE_TRANSFER:
1576      case RTEMS_BDBUF_STATE_TRANSFER_PURGED:
1577        rtems_bdbuf_wait (bd, &bdbuf_cache.transfer_waiters);
1578        break;
1579      default:
1580        rtems_bdbuf_fatal (bd->state, RTEMS_BLKDEV_FATAL_BDBUF_STATE_7);
1581    }
1582  }
1583}
1584
1585static void
1586rtems_bdbuf_request_sync_for_modified_buffer (rtems_bdbuf_buffer *bd)
1587{
1588  rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_SYNC);
1589  rtems_chain_extract_unprotected (&bd->link);
1590  rtems_chain_append_unprotected (&bdbuf_cache.sync, &bd->link);
1591  rtems_bdbuf_wake_swapper ();
1592}
1593
1594/**
1595 * @brief Waits until the buffer is ready for recycling.
1596 *
1597 * @retval @c true Buffer is valid and may be recycled.
1598 * @retval @c false Buffer is invalid and has to searched again.
1599 */
1600static bool
1601rtems_bdbuf_wait_for_recycle (rtems_bdbuf_buffer *bd)
1602{
1603  while (true)
1604  {
1605    switch (bd->state)
1606    {
1607      case RTEMS_BDBUF_STATE_FREE:
1608        return true;
1609      case RTEMS_BDBUF_STATE_MODIFIED:
1610        rtems_bdbuf_request_sync_for_modified_buffer (bd);
1611        break;
1612      case RTEMS_BDBUF_STATE_CACHED:
1613      case RTEMS_BDBUF_STATE_EMPTY:
1614        if (bd->waiters == 0)
1615          return true;
1616        else
1617        {
1618          /*
1619           * It is essential that we wait here without a special wait count and
1620           * without the group in use.  Otherwise we could trigger a wait ping
1621           * pong with another recycle waiter.  The state of the buffer is
1622           * arbitrary afterwards.
1623           */
1624          rtems_bdbuf_anonymous_wait (&bdbuf_cache.buffer_waiters);
1625          return false;
1626        }
1627      case RTEMS_BDBUF_STATE_ACCESS_CACHED:
1628      case RTEMS_BDBUF_STATE_ACCESS_EMPTY:
1629      case RTEMS_BDBUF_STATE_ACCESS_MODIFIED:
1630      case RTEMS_BDBUF_STATE_ACCESS_PURGED:
1631        rtems_bdbuf_wait (bd, &bdbuf_cache.access_waiters);
1632        break;
1633      case RTEMS_BDBUF_STATE_SYNC:
1634      case RTEMS_BDBUF_STATE_TRANSFER:
1635      case RTEMS_BDBUF_STATE_TRANSFER_PURGED:
1636        rtems_bdbuf_wait (bd, &bdbuf_cache.transfer_waiters);
1637        break;
1638      default:
1639        rtems_bdbuf_fatal (bd->state, RTEMS_BLKDEV_FATAL_BDBUF_STATE_8);
1640    }
1641  }
1642}
1643
1644static void
1645rtems_bdbuf_wait_for_sync_done (rtems_bdbuf_buffer *bd)
1646{
1647  while (true)
1648  {
1649    switch (bd->state)
1650    {
1651      case RTEMS_BDBUF_STATE_CACHED:
1652      case RTEMS_BDBUF_STATE_EMPTY:
1653      case RTEMS_BDBUF_STATE_MODIFIED:
1654      case RTEMS_BDBUF_STATE_ACCESS_CACHED:
1655      case RTEMS_BDBUF_STATE_ACCESS_EMPTY:
1656      case RTEMS_BDBUF_STATE_ACCESS_MODIFIED:
1657      case RTEMS_BDBUF_STATE_ACCESS_PURGED:
1658        return;
1659      case RTEMS_BDBUF_STATE_SYNC:
1660      case RTEMS_BDBUF_STATE_TRANSFER:
1661      case RTEMS_BDBUF_STATE_TRANSFER_PURGED:
1662        rtems_bdbuf_wait (bd, &bdbuf_cache.transfer_waiters);
1663        break;
1664      default:
1665        rtems_bdbuf_fatal (bd->state, RTEMS_BLKDEV_FATAL_BDBUF_STATE_9);
1666    }
1667  }
1668}
1669
1670static void
1671rtems_bdbuf_wait_for_buffer (void)
1672{
1673  if (!rtems_chain_is_empty (&bdbuf_cache.modified))
1674    rtems_bdbuf_wake_swapper ();
1675
1676  rtems_bdbuf_anonymous_wait (&bdbuf_cache.buffer_waiters);
1677}
1678
1679static void
1680rtems_bdbuf_sync_after_access (rtems_bdbuf_buffer *bd)
1681{
1682  rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_SYNC);
1683
1684  rtems_chain_append_unprotected (&bdbuf_cache.sync, &bd->link);
1685
1686  if (bd->waiters)
1687    rtems_bdbuf_wake (&bdbuf_cache.access_waiters);
1688
1689  rtems_bdbuf_wake_swapper ();
1690  rtems_bdbuf_wait_for_sync_done (bd);
1691
1692  /*
1693   * We may have created a cached or empty buffer which may be recycled.
1694   */
1695  if (bd->waiters == 0
1696        && (bd->state == RTEMS_BDBUF_STATE_CACHED
1697          || bd->state == RTEMS_BDBUF_STATE_EMPTY))
1698  {
1699    if (bd->state == RTEMS_BDBUF_STATE_EMPTY)
1700    {
1701      rtems_bdbuf_remove_from_tree (bd);
1702      rtems_bdbuf_make_free_and_add_to_lru_list (bd);
1703    }
1704    rtems_bdbuf_wake (&bdbuf_cache.buffer_waiters);
1705  }
1706}
1707
1708static rtems_bdbuf_buffer *
1709rtems_bdbuf_get_buffer_for_read_ahead (rtems_disk_device *dd,
1710                                       rtems_blkdev_bnum  block)
1711{
1712  rtems_bdbuf_buffer *bd = NULL;
1713
1714  bd = rtems_bdbuf_avl_search (&bdbuf_cache.tree, dd, block);
1715
1716  if (bd == NULL)
1717  {
1718    bd = rtems_bdbuf_get_buffer_from_lru_list (dd, block);
1719
1720    if (bd != NULL)
1721      rtems_bdbuf_group_obtain (bd);
1722  }
1723  else
1724    /*
1725     * The buffer is in the cache.  So it is already available or in use, and
1726     * thus no need for a read ahead.
1727     */
1728    bd = NULL;
1729
1730  return bd;
1731}
1732
1733static rtems_bdbuf_buffer *
1734rtems_bdbuf_get_buffer_for_access (rtems_disk_device *dd,
1735                                   rtems_blkdev_bnum  block)
1736{
1737  rtems_bdbuf_buffer *bd = NULL;
1738
1739  do
1740  {
1741    bd = rtems_bdbuf_avl_search (&bdbuf_cache.tree, dd, block);
1742
1743    if (bd != NULL)
1744    {
1745      if (bd->group->bds_per_group != dd->bds_per_group)
1746      {
1747        if (rtems_bdbuf_wait_for_recycle (bd))
1748        {
1749          rtems_bdbuf_remove_from_tree_and_lru_list (bd);
1750          rtems_bdbuf_make_free_and_add_to_lru_list (bd);
1751          rtems_bdbuf_wake (&bdbuf_cache.buffer_waiters);
1752        }
1753        bd = NULL;
1754      }
1755    }
1756    else
1757    {
1758      bd = rtems_bdbuf_get_buffer_from_lru_list (dd, block);
1759
1760      if (bd == NULL)
1761        rtems_bdbuf_wait_for_buffer ();
1762    }
1763  }
1764  while (bd == NULL);
1765
1766  rtems_bdbuf_wait_for_access (bd);
1767  rtems_bdbuf_group_obtain (bd);
1768
1769  return bd;
1770}
1771
1772static rtems_status_code
1773rtems_bdbuf_get_media_block (const rtems_disk_device *dd,
1774                             rtems_blkdev_bnum        block,
1775                             rtems_blkdev_bnum       *media_block_ptr)
1776{
1777  rtems_status_code sc = RTEMS_SUCCESSFUL;
1778
1779  if (block < dd->block_count)
1780  {
1781    /*
1782     * Compute the media block number. Drivers work with media block number not
1783     * the block number a BD may have as this depends on the block size set by
1784     * the user.
1785     */
1786    *media_block_ptr = rtems_bdbuf_media_block (dd, block) + dd->start;
1787  }
1788  else
1789  {
1790    sc = RTEMS_INVALID_ID;
1791  }
1792
1793  return sc;
1794}
1795
1796rtems_status_code
1797rtems_bdbuf_get (rtems_disk_device   *dd,
1798                 rtems_blkdev_bnum    block,
1799                 rtems_bdbuf_buffer **bd_ptr)
1800{
1801  rtems_status_code   sc = RTEMS_SUCCESSFUL;
1802  rtems_bdbuf_buffer *bd = NULL;
1803  rtems_blkdev_bnum   media_block;
1804
1805  rtems_bdbuf_lock_cache ();
1806
1807  sc = rtems_bdbuf_get_media_block (dd, block, &media_block);
1808  if (sc == RTEMS_SUCCESSFUL)
1809  {
1810    /*
1811     * Print the block index relative to the physical disk.
1812     */
1813    if (rtems_bdbuf_tracer)
1814      printf ("bdbuf:get: %" PRIu32 " (%" PRIu32 ") (dev = %08x)\n",
1815              media_block, block, (unsigned) dd->dev);
1816
1817    bd = rtems_bdbuf_get_buffer_for_access (dd, media_block);
1818
1819    switch (bd->state)
1820    {
1821      case RTEMS_BDBUF_STATE_CACHED:
1822        rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_ACCESS_CACHED);
1823        break;
1824      case RTEMS_BDBUF_STATE_EMPTY:
1825        rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_ACCESS_EMPTY);
1826        break;
1827      case RTEMS_BDBUF_STATE_MODIFIED:
1828        /*
1829         * To get a modified buffer could be considered a bug in the caller
1830         * because you should not be getting an already modified buffer but
1831         * user may have modified a byte in a block then decided to seek the
1832         * start and write the whole block and the file system will have no
1833         * record of this so just gets the block to fill.
1834         */
1835        rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_ACCESS_MODIFIED);
1836        break;
1837      default:
1838        rtems_bdbuf_fatal (bd->state, RTEMS_BLKDEV_FATAL_BDBUF_STATE_2);
1839        break;
1840    }
1841
1842    if (rtems_bdbuf_tracer)
1843    {
1844      rtems_bdbuf_show_users ("get", bd);
1845      rtems_bdbuf_show_usage ();
1846    }
1847  }
1848
1849  rtems_bdbuf_unlock_cache ();
1850
1851  *bd_ptr = bd;
1852
1853  return sc;
1854}
1855
1856/**
1857 * Call back handler called by the low level driver when the transfer has
1858 * completed. This function may be invoked from interrupt handler.
1859 *
1860 * @param arg Arbitrary argument specified in block device request
1861 *            structure (in this case - pointer to the appropriate
1862 *            block device request structure).
1863 * @param status I/O completion status
1864 */
1865static void
1866rtems_bdbuf_transfer_done (void* arg, rtems_status_code status)
1867{
1868  rtems_blkdev_request* req = (rtems_blkdev_request*) arg;
1869
1870  req->status = status;
1871
1872  rtems_event_send (req->io_task, RTEMS_BDBUF_TRANSFER_SYNC);
1873}
1874
1875static rtems_status_code
1876rtems_bdbuf_execute_transfer_request (rtems_disk_device    *dd,
1877                                      rtems_blkdev_request *req,
1878                                      bool                  cache_locked)
1879{
1880  rtems_status_code sc = RTEMS_SUCCESSFUL;
1881  int result = 0;
1882  uint32_t transfer_index = 0;
1883  bool wake_transfer_waiters = false;
1884  bool wake_buffer_waiters = false;
1885
1886  if (cache_locked)
1887    rtems_bdbuf_unlock_cache ();
1888
1889  result = dd->ioctl (dd->phys_dev, RTEMS_BLKIO_REQUEST, req);
1890
1891  if (result == 0)
1892  {
1893    rtems_bdbuf_wait_for_event (RTEMS_BDBUF_TRANSFER_SYNC);
1894    sc = req->status;
1895  }
1896  else
1897    sc = RTEMS_IO_ERROR;
1898
1899  rtems_bdbuf_lock_cache ();
1900
1901  /* Statistics */
1902  if (req->req == RTEMS_BLKDEV_REQ_READ)
1903  {
1904    dd->stats.read_blocks += req->bufnum;
1905    if (sc != RTEMS_SUCCESSFUL)
1906      ++dd->stats.read_errors;
1907  }
1908  else
1909  {
1910    dd->stats.write_blocks += req->bufnum;
1911    ++dd->stats.write_transfers;
1912    if (sc != RTEMS_SUCCESSFUL)
1913      ++dd->stats.write_errors;
1914  }
1915
1916  for (transfer_index = 0; transfer_index < req->bufnum; ++transfer_index)
1917  {
1918    rtems_bdbuf_buffer *bd = req->bufs [transfer_index].user;
1919    bool waiters = bd->waiters;
1920
1921    if (waiters)
1922      wake_transfer_waiters = true;
1923    else
1924      wake_buffer_waiters = true;
1925
1926    rtems_bdbuf_group_release (bd);
1927
1928    if (sc == RTEMS_SUCCESSFUL && bd->state == RTEMS_BDBUF_STATE_TRANSFER)
1929      rtems_bdbuf_make_cached_and_add_to_lru_list (bd);
1930    else
1931      rtems_bdbuf_discard_buffer (bd);
1932
1933    if (rtems_bdbuf_tracer)
1934      rtems_bdbuf_show_users ("transfer", bd);
1935  }
1936
1937  if (wake_transfer_waiters)
1938    rtems_bdbuf_wake (&bdbuf_cache.transfer_waiters);
1939
1940  if (wake_buffer_waiters)
1941    rtems_bdbuf_wake (&bdbuf_cache.buffer_waiters);
1942
1943  if (!cache_locked)
1944    rtems_bdbuf_unlock_cache ();
1945
1946  if (sc == RTEMS_SUCCESSFUL || sc == RTEMS_UNSATISFIED)
1947    return sc;
1948  else
1949    return RTEMS_IO_ERROR;
1950}
1951
1952static rtems_status_code
1953rtems_bdbuf_execute_read_request (rtems_disk_device  *dd,
1954                                  rtems_bdbuf_buffer *bd,
1955                                  uint32_t            transfer_count)
1956{
1957  rtems_blkdev_request *req = NULL;
1958  rtems_blkdev_bnum media_block = bd->block;
1959  uint32_t media_blocks_per_block = dd->media_blocks_per_block;
1960  uint32_t block_size = dd->block_size;
1961  uint32_t transfer_index = 1;
1962
1963  /*
1964   * TODO: This type of request structure is wrong and should be removed.
1965   */
1966#define bdbuf_alloc(size) __builtin_alloca (size)
1967
1968  req = bdbuf_alloc (sizeof (rtems_blkdev_request) +
1969                     sizeof (rtems_blkdev_sg_buffer) * transfer_count);
1970
1971  req->req = RTEMS_BLKDEV_REQ_READ;
1972  req->req_done = rtems_bdbuf_transfer_done;
1973  req->done_arg = req;
1974  req->io_task = rtems_task_self ();
1975  req->status = RTEMS_RESOURCE_IN_USE;
1976  req->bufnum = 0;
1977
1978  rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_TRANSFER);
1979
1980  req->bufs [0].user   = bd;
1981  req->bufs [0].block  = media_block;
1982  req->bufs [0].length = block_size;
1983  req->bufs [0].buffer = bd->buffer;
1984
1985  if (rtems_bdbuf_tracer)
1986    rtems_bdbuf_show_users ("read", bd);
1987
1988  while (transfer_index < transfer_count)
1989  {
1990    media_block += media_blocks_per_block;
1991
1992    bd = rtems_bdbuf_get_buffer_for_read_ahead (dd, media_block);
1993
1994    if (bd == NULL)
1995      break;
1996
1997    rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_TRANSFER);
1998
1999    req->bufs [transfer_index].user   = bd;
2000    req->bufs [transfer_index].block  = media_block;
2001    req->bufs [transfer_index].length = block_size;
2002    req->bufs [transfer_index].buffer = bd->buffer;
2003
2004    if (rtems_bdbuf_tracer)
2005      rtems_bdbuf_show_users ("read", bd);
2006
2007    ++transfer_index;
2008  }
2009
2010  req->bufnum = transfer_index;
2011
2012  return rtems_bdbuf_execute_transfer_request (dd, req, true);
2013}
2014
2015static bool
2016rtems_bdbuf_is_read_ahead_active (const rtems_disk_device *dd)
2017{
2018  return !rtems_chain_is_node_off_chain (&dd->read_ahead.node);
2019}
2020
2021static void
2022rtems_bdbuf_read_ahead_cancel (rtems_disk_device *dd)
2023{
2024  if (rtems_bdbuf_is_read_ahead_active (dd))
2025  {
2026    rtems_chain_extract_unprotected (&dd->read_ahead.node);
2027    rtems_chain_set_off_chain (&dd->read_ahead.node);
2028  }
2029}
2030
2031static void
2032rtems_bdbuf_read_ahead_reset (rtems_disk_device *dd)
2033{
2034  rtems_bdbuf_read_ahead_cancel (dd);
2035  dd->read_ahead.trigger = RTEMS_DISK_READ_AHEAD_NO_TRIGGER;
2036}
2037
2038static void
2039rtems_bdbuf_check_read_ahead_trigger (rtems_disk_device *dd,
2040                                      rtems_blkdev_bnum  block)
2041{
2042  if (bdbuf_cache.read_ahead_task != 0
2043      && dd->read_ahead.trigger == block
2044      && !rtems_bdbuf_is_read_ahead_active (dd))
2045  {
2046    rtems_status_code sc;
2047    rtems_chain_control *chain = &bdbuf_cache.read_ahead_chain;
2048
2049    if (rtems_chain_is_empty (chain))
2050    {
2051      sc = rtems_event_send (bdbuf_cache.read_ahead_task,
2052                             RTEMS_BDBUF_READ_AHEAD_WAKE_UP);
2053      if (sc != RTEMS_SUCCESSFUL)
2054        rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_RA_WAKE_UP);
2055    }
2056
2057    rtems_chain_append_unprotected (chain, &dd->read_ahead.node);
2058  }
2059}
2060
2061static void
2062rtems_bdbuf_set_read_ahead_trigger (rtems_disk_device *dd,
2063                                    rtems_blkdev_bnum  block)
2064{
2065  if (dd->read_ahead.trigger != block)
2066  {
2067    rtems_bdbuf_read_ahead_cancel (dd);
2068    dd->read_ahead.trigger = block + 1;
2069    dd->read_ahead.next = block + 2;
2070  }
2071}
2072
2073rtems_status_code
2074rtems_bdbuf_read (rtems_disk_device   *dd,
2075                  rtems_blkdev_bnum    block,
2076                  rtems_bdbuf_buffer **bd_ptr)
2077{
2078  rtems_status_code     sc = RTEMS_SUCCESSFUL;
2079  rtems_bdbuf_buffer   *bd = NULL;
2080  rtems_blkdev_bnum     media_block;
2081
2082  rtems_bdbuf_lock_cache ();
2083
2084  sc = rtems_bdbuf_get_media_block (dd, block, &media_block);
2085  if (sc == RTEMS_SUCCESSFUL)
2086  {
2087    if (rtems_bdbuf_tracer)
2088      printf ("bdbuf:read: %" PRIu32 " (%" PRIu32 ") (dev = %08x)\n",
2089              media_block + dd->start, block, (unsigned) dd->dev);
2090
2091    bd = rtems_bdbuf_get_buffer_for_access (dd, media_block);
2092    switch (bd->state)
2093    {
2094      case RTEMS_BDBUF_STATE_CACHED:
2095        ++dd->stats.read_hits;
2096        rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_ACCESS_CACHED);
2097        break;
2098      case RTEMS_BDBUF_STATE_MODIFIED:
2099        ++dd->stats.read_hits;
2100        rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_ACCESS_MODIFIED);
2101        break;
2102      case RTEMS_BDBUF_STATE_EMPTY:
2103        ++dd->stats.read_misses;
2104        rtems_bdbuf_set_read_ahead_trigger (dd, block);
2105        sc = rtems_bdbuf_execute_read_request (dd, bd, 1);
2106        if (sc == RTEMS_SUCCESSFUL)
2107        {
2108          rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_ACCESS_CACHED);
2109          rtems_chain_extract_unprotected (&bd->link);
2110          rtems_bdbuf_group_obtain (bd);
2111        }
2112        else
2113        {
2114          bd = NULL;
2115        }
2116        break;
2117      default:
2118        rtems_bdbuf_fatal (bd->state, RTEMS_BLKDEV_FATAL_BDBUF_STATE_4);
2119        break;
2120    }
2121
2122    rtems_bdbuf_check_read_ahead_trigger (dd, block);
2123  }
2124
2125  rtems_bdbuf_unlock_cache ();
2126
2127  *bd_ptr = bd;
2128
2129  return sc;
2130}
2131
2132static rtems_status_code
2133rtems_bdbuf_check_bd_and_lock_cache (rtems_bdbuf_buffer *bd, const char *kind)
2134{
2135  if (bd == NULL)
2136    return RTEMS_INVALID_ADDRESS;
2137  if (rtems_bdbuf_tracer)
2138  {
2139    printf ("bdbuf:%s: %" PRIu32 "\n", kind, bd->block);
2140    rtems_bdbuf_show_users (kind, bd);
2141  }
2142  rtems_bdbuf_lock_cache();
2143
2144  return RTEMS_SUCCESSFUL;
2145}
2146
2147rtems_status_code
2148rtems_bdbuf_release (rtems_bdbuf_buffer *bd)
2149{
2150  rtems_status_code sc = RTEMS_SUCCESSFUL;
2151
2152  sc = rtems_bdbuf_check_bd_and_lock_cache (bd, "release");
2153  if (sc != RTEMS_SUCCESSFUL)
2154    return sc;
2155
2156  switch (bd->state)
2157  {
2158    case RTEMS_BDBUF_STATE_ACCESS_CACHED:
2159      rtems_bdbuf_add_to_lru_list_after_access (bd);
2160      break;
2161    case RTEMS_BDBUF_STATE_ACCESS_EMPTY:
2162    case RTEMS_BDBUF_STATE_ACCESS_PURGED:
2163      rtems_bdbuf_discard_buffer_after_access (bd);
2164      break;
2165    case RTEMS_BDBUF_STATE_ACCESS_MODIFIED:
2166      rtems_bdbuf_add_to_modified_list_after_access (bd);
2167      break;
2168    default:
2169      rtems_bdbuf_fatal (bd->state, RTEMS_BLKDEV_FATAL_BDBUF_STATE_0);
2170      break;
2171  }
2172
2173  if (rtems_bdbuf_tracer)
2174    rtems_bdbuf_show_usage ();
2175
2176  rtems_bdbuf_unlock_cache ();
2177
2178  return RTEMS_SUCCESSFUL;
2179}
2180
2181rtems_status_code
2182rtems_bdbuf_release_modified (rtems_bdbuf_buffer *bd)
2183{
2184  rtems_status_code sc = RTEMS_SUCCESSFUL;
2185
2186  sc = rtems_bdbuf_check_bd_and_lock_cache (bd, "release modified");
2187  if (sc != RTEMS_SUCCESSFUL)
2188    return sc;
2189
2190  switch (bd->state)
2191  {
2192    case RTEMS_BDBUF_STATE_ACCESS_CACHED:
2193    case RTEMS_BDBUF_STATE_ACCESS_EMPTY:
2194    case RTEMS_BDBUF_STATE_ACCESS_MODIFIED:
2195      rtems_bdbuf_add_to_modified_list_after_access (bd);
2196      break;
2197    case RTEMS_BDBUF_STATE_ACCESS_PURGED:
2198      rtems_bdbuf_discard_buffer_after_access (bd);
2199      break;
2200    default:
2201      rtems_bdbuf_fatal (bd->state, RTEMS_BLKDEV_FATAL_BDBUF_STATE_6);
2202      break;
2203  }
2204
2205  if (rtems_bdbuf_tracer)
2206    rtems_bdbuf_show_usage ();
2207
2208  rtems_bdbuf_unlock_cache ();
2209
2210  return RTEMS_SUCCESSFUL;
2211}
2212
2213rtems_status_code
2214rtems_bdbuf_sync (rtems_bdbuf_buffer *bd)
2215{
2216  rtems_status_code sc = RTEMS_SUCCESSFUL;
2217
2218  sc = rtems_bdbuf_check_bd_and_lock_cache (bd, "sync");
2219  if (sc != RTEMS_SUCCESSFUL)
2220    return sc;
2221
2222  switch (bd->state)
2223  {
2224    case RTEMS_BDBUF_STATE_ACCESS_CACHED:
2225    case RTEMS_BDBUF_STATE_ACCESS_EMPTY:
2226    case RTEMS_BDBUF_STATE_ACCESS_MODIFIED:
2227      rtems_bdbuf_sync_after_access (bd);
2228      break;
2229    case RTEMS_BDBUF_STATE_ACCESS_PURGED:
2230      rtems_bdbuf_discard_buffer_after_access (bd);
2231      break;
2232    default:
2233      rtems_bdbuf_fatal (bd->state, RTEMS_BLKDEV_FATAL_BDBUF_STATE_5);
2234      break;
2235  }
2236
2237  if (rtems_bdbuf_tracer)
2238    rtems_bdbuf_show_usage ();
2239
2240  rtems_bdbuf_unlock_cache ();
2241
2242  return RTEMS_SUCCESSFUL;
2243}
2244
2245rtems_status_code
2246rtems_bdbuf_syncdev (rtems_disk_device *dd)
2247{
2248  if (rtems_bdbuf_tracer)
2249    printf ("bdbuf:syncdev: %08x\n", (unsigned) dd->dev);
2250
2251  /*
2252   * Take the sync lock before locking the cache. Once we have the sync lock we
2253   * can lock the cache. If another thread has the sync lock it will cause this
2254   * thread to block until it owns the sync lock then it can own the cache. The
2255   * sync lock can only be obtained with the cache unlocked.
2256   */
2257  rtems_bdbuf_lock_sync ();
2258  rtems_bdbuf_lock_cache ();
2259
2260  /*
2261   * Set the cache to have a sync active for a specific device and let the swap
2262   * out task know the id of the requester to wake when done.
2263   *
2264   * The swap out task will negate the sync active flag when no more buffers
2265   * for the device are held on the "modified for sync" queues.
2266   */
2267  bdbuf_cache.sync_active    = true;
2268  bdbuf_cache.sync_requester = rtems_task_self ();
2269  bdbuf_cache.sync_device    = dd;
2270
2271  rtems_bdbuf_wake_swapper ();
2272  rtems_bdbuf_unlock_cache ();
2273  rtems_bdbuf_wait_for_event (RTEMS_BDBUF_TRANSFER_SYNC);
2274  rtems_bdbuf_unlock_sync ();
2275
2276  return RTEMS_SUCCESSFUL;
2277}
2278
2279/**
2280 * Swapout transfer to the driver. The driver will break this I/O into groups
2281 * of consecutive write requests is multiple consecutive buffers are required
2282 * by the driver. The cache is not locked.
2283 *
2284 * @param transfer The transfer transaction.
2285 */
2286static void
2287rtems_bdbuf_swapout_write (rtems_bdbuf_swapout_transfer* transfer)
2288{
2289  rtems_chain_node *node;
2290
2291  if (rtems_bdbuf_tracer)
2292    printf ("bdbuf:swapout transfer: %08x\n", (unsigned) transfer->dd->dev);
2293
2294  /*
2295   * If there are buffers to transfer to the media transfer them.
2296   */
2297  if (!rtems_chain_is_empty (&transfer->bds))
2298  {
2299    /*
2300     * The last block number used when the driver only supports
2301     * continuous blocks in a single request.
2302     */
2303    uint32_t last_block = 0;
2304
2305    rtems_disk_device *dd = transfer->dd;
2306    uint32_t media_blocks_per_block = dd->media_blocks_per_block;
2307    bool need_continuous_blocks =
2308      (dd->phys_dev->capabilities & RTEMS_BLKDEV_CAP_MULTISECTOR_CONT) != 0;
2309
2310    /*
2311     * Take as many buffers as configured and pass to the driver. Note, the
2312     * API to the drivers has an array of buffers and if a chain was passed
2313     * we could have just passed the list. If the driver API is updated it
2314     * should be possible to make this change with little effect in this
2315     * code. The array that is passed is broken in design and should be
2316     * removed. Merging members of a struct into the first member is
2317     * trouble waiting to happen.
2318     */
2319    transfer->write_req->status = RTEMS_RESOURCE_IN_USE;
2320    transfer->write_req->bufnum = 0;
2321
2322    while ((node = rtems_chain_get_unprotected(&transfer->bds)) != NULL)
2323    {
2324      rtems_bdbuf_buffer* bd = (rtems_bdbuf_buffer*) node;
2325      bool                write = false;
2326
2327      /*
2328       * If the device only accepts sequential buffers and this is not the
2329       * first buffer (the first is always sequential, and the buffer is not
2330       * sequential then put the buffer back on the transfer chain and write
2331       * the committed buffers.
2332       */
2333
2334      if (rtems_bdbuf_tracer)
2335        printf ("bdbuf:swapout write: bd:%" PRIu32 ", bufnum:%" PRIu32 " mode:%s\n",
2336                bd->block, transfer->write_req->bufnum,
2337                need_continuous_blocks ? "MULTI" : "SCAT");
2338
2339      if (need_continuous_blocks && transfer->write_req->bufnum &&
2340          bd->block != last_block + media_blocks_per_block)
2341      {
2342        rtems_chain_prepend_unprotected (&transfer->bds, &bd->link);
2343        write = true;
2344      }
2345      else
2346      {
2347        rtems_blkdev_sg_buffer* buf;
2348        buf = &transfer->write_req->bufs[transfer->write_req->bufnum];
2349        transfer->write_req->bufnum++;
2350        buf->user   = bd;
2351        buf->block  = bd->block;
2352        buf->length = dd->block_size;
2353        buf->buffer = bd->buffer;
2354        last_block  = bd->block;
2355      }
2356
2357      /*
2358       * Perform the transfer if there are no more buffers, or the transfer
2359       * size has reached the configured max. value.
2360       */
2361
2362      if (rtems_chain_is_empty (&transfer->bds) ||
2363          (transfer->write_req->bufnum >= bdbuf_config.max_write_blocks))
2364        write = true;
2365
2366      if (write)
2367      {
2368        rtems_bdbuf_execute_transfer_request (dd, transfer->write_req, false);
2369
2370        transfer->write_req->status = RTEMS_RESOURCE_IN_USE;
2371        transfer->write_req->bufnum = 0;
2372      }
2373    }
2374
2375    /*
2376     * If sync'ing and the deivce is capability of handling a sync IO control
2377     * call perform the call.
2378     */
2379    if (transfer->syncing &&
2380        (dd->phys_dev->capabilities & RTEMS_BLKDEV_CAP_SYNC))
2381    {
2382      /* int result = */ dd->ioctl (dd->phys_dev, RTEMS_BLKDEV_REQ_SYNC, NULL);
2383      /* How should the error be handled ? */
2384    }
2385  }
2386}
2387
2388/**
2389 * Process the modified list of buffers. There is a sync or modified list that
2390 * needs to be handled so we have a common function to do the work.
2391 *
2392 * @param dd_ptr Pointer to the device to handle. If BDBUF_INVALID_DEV no
2393 * device is selected so select the device of the first buffer to be written to
2394 * disk.
2395 * @param chain The modified chain to process.
2396 * @param transfer The chain to append buffers to be written too.
2397 * @param sync_active If true this is a sync operation so expire all timers.
2398 * @param update_timers If true update the timers.
2399 * @param timer_delta It update_timers is true update the timers by this
2400 *                    amount.
2401 */
2402static void
2403rtems_bdbuf_swapout_modified_processing (rtems_disk_device  **dd_ptr,
2404                                         rtems_chain_control* chain,
2405                                         rtems_chain_control* transfer,
2406                                         bool                 sync_active,
2407                                         bool                 update_timers,
2408                                         uint32_t             timer_delta)
2409{
2410  if (!rtems_chain_is_empty (chain))
2411  {
2412    rtems_chain_node* node = rtems_chain_head (chain);
2413    bool              sync_all;
2414   
2415    node = node->next;
2416
2417    /*
2418     * A sync active with no valid dev means sync all.
2419     */
2420    if (sync_active && (*dd_ptr == BDBUF_INVALID_DEV))
2421      sync_all = true;
2422    else
2423      sync_all = false;
2424   
2425    while (!rtems_chain_is_tail (chain, node))
2426    {
2427      rtems_bdbuf_buffer* bd = (rtems_bdbuf_buffer*) node;
2428
2429      /*
2430       * Check if the buffer's hold timer has reached 0. If a sync is active
2431       * or someone waits for a buffer written force all the timers to 0.
2432       *
2433       * @note Lots of sync requests will skew this timer. It should be based
2434       *       on TOD to be accurate. Does it matter ?
2435       */
2436      if (sync_all || (sync_active && (*dd_ptr == bd->dd))
2437          || rtems_bdbuf_has_buffer_waiters ())
2438        bd->hold_timer = 0;
2439
2440      if (bd->hold_timer)
2441      {
2442        if (update_timers)
2443        {
2444          if (bd->hold_timer > timer_delta)
2445            bd->hold_timer -= timer_delta;
2446          else
2447            bd->hold_timer = 0;
2448        }
2449
2450        if (bd->hold_timer)
2451        {
2452          node = node->next;
2453          continue;
2454        }
2455      }
2456
2457      /*
2458       * This assumes we can set it to BDBUF_INVALID_DEV which is just an
2459       * assumption. Cannot use the transfer list being empty the sync dev
2460       * calls sets the dev to use.
2461       */
2462      if (*dd_ptr == BDBUF_INVALID_DEV)
2463        *dd_ptr = bd->dd;
2464
2465      if (bd->dd == *dd_ptr)
2466      {
2467        rtems_chain_node* next_node = node->next;
2468        rtems_chain_node* tnode = rtems_chain_tail (transfer);
2469
2470        /*
2471         * The blocks on the transfer list are sorted in block order. This
2472         * means multi-block transfers for drivers that require consecutive
2473         * blocks perform better with sorted blocks and for real disks it may
2474         * help lower head movement.
2475         */
2476
2477        rtems_bdbuf_set_state (bd, RTEMS_BDBUF_STATE_TRANSFER);
2478
2479        rtems_chain_extract_unprotected (node);
2480
2481        tnode = tnode->previous;
2482
2483        while (node && !rtems_chain_is_head (transfer, tnode))
2484        {
2485          rtems_bdbuf_buffer* tbd = (rtems_bdbuf_buffer*) tnode;
2486
2487          if (bd->block > tbd->block)
2488          {
2489            rtems_chain_insert_unprotected (tnode, node);
2490            node = NULL;
2491          }
2492          else
2493            tnode = tnode->previous;
2494        }
2495
2496        if (node)
2497          rtems_chain_prepend_unprotected (transfer, node);
2498
2499        node = next_node;
2500      }
2501      else
2502      {
2503        node = node->next;
2504      }
2505    }
2506  }
2507}
2508
2509/**
2510 * Process the cache's modified buffers. Check the sync list first then the
2511 * modified list extracting the buffers suitable to be written to disk. We have
2512 * a device at a time. The task level loop will repeat this operation while
2513 * there are buffers to be written. If the transfer fails place the buffers
2514 * back on the modified list and try again later. The cache is unlocked while
2515 * the buffers are being written to disk.
2516 *
2517 * @param timer_delta It update_timers is true update the timers by this
2518 *                    amount.
2519 * @param update_timers If true update the timers.
2520 * @param transfer The transfer transaction data.
2521 *
2522 * @retval true Buffers where written to disk so scan again.
2523 * @retval false No buffers where written to disk.
2524 */
2525static bool
2526rtems_bdbuf_swapout_processing (unsigned long                 timer_delta,
2527                                bool                          update_timers,
2528                                rtems_bdbuf_swapout_transfer* transfer)
2529{
2530  rtems_bdbuf_swapout_worker* worker;
2531  bool                        transfered_buffers = false;
2532
2533  rtems_bdbuf_lock_cache ();
2534
2535  /*
2536   * If a sync is active do not use a worker because the current code does not
2537   * cleaning up after. We need to know the buffers have been written when
2538   * syncing to release sync lock and currently worker threads do not return to
2539   * here. We do not know the worker is the last in a sequence of sync writes
2540   * until after we have it running so we do not know to tell it to release the
2541   * lock. The simplest solution is to get the main swap out task perform all
2542   * sync operations.
2543   */
2544  if (bdbuf_cache.sync_active)
2545    worker = NULL;
2546  else
2547  {
2548    worker = (rtems_bdbuf_swapout_worker*)
2549      rtems_chain_get_unprotected (&bdbuf_cache.swapout_workers);
2550    if (worker)
2551      transfer = &worker->transfer;
2552  }
2553
2554  rtems_chain_initialize_empty (&transfer->bds);
2555  transfer->dd = BDBUF_INVALID_DEV;
2556  transfer->syncing = bdbuf_cache.sync_active;
2557 
2558  /*
2559   * When the sync is for a device limit the sync to that device. If the sync
2560   * is for a buffer handle process the devices in the order on the sync
2561   * list. This means the dev is BDBUF_INVALID_DEV.
2562   */
2563  if (bdbuf_cache.sync_active)
2564    transfer->dd = bdbuf_cache.sync_device;
2565   
2566  /*
2567   * If we have any buffers in the sync queue move them to the modified
2568   * list. The first sync buffer will select the device we use.
2569   */
2570  rtems_bdbuf_swapout_modified_processing (&transfer->dd,
2571                                           &bdbuf_cache.sync,
2572                                           &transfer->bds,
2573                                           true, false,
2574                                           timer_delta);
2575
2576  /*
2577   * Process the cache's modified list.
2578   */
2579  rtems_bdbuf_swapout_modified_processing (&transfer->dd,
2580                                           &bdbuf_cache.modified,
2581                                           &transfer->bds,
2582                                           bdbuf_cache.sync_active,
2583                                           update_timers,
2584                                           timer_delta);
2585
2586  /*
2587   * We have all the buffers that have been modified for this device so the
2588   * cache can be unlocked because the state of each buffer has been set to
2589   * TRANSFER.
2590   */
2591  rtems_bdbuf_unlock_cache ();
2592
2593  /*
2594   * If there are buffers to transfer to the media transfer them.
2595   */
2596  if (!rtems_chain_is_empty (&transfer->bds))
2597  {
2598    if (worker)
2599    {
2600      rtems_status_code sc = rtems_event_send (worker->id,
2601                                               RTEMS_BDBUF_SWAPOUT_SYNC);
2602      if (sc != RTEMS_SUCCESSFUL)
2603        rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_SO_WAKE);
2604    }
2605    else
2606    {
2607      rtems_bdbuf_swapout_write (transfer);
2608    }
2609
2610    transfered_buffers = true;
2611  }
2612
2613  if (bdbuf_cache.sync_active && !transfered_buffers)
2614  {
2615    rtems_id sync_requester;
2616    rtems_bdbuf_lock_cache ();
2617    sync_requester = bdbuf_cache.sync_requester;
2618    bdbuf_cache.sync_active = false;
2619    bdbuf_cache.sync_requester = 0;
2620    rtems_bdbuf_unlock_cache ();
2621    if (sync_requester)
2622      rtems_event_send (sync_requester, RTEMS_BDBUF_TRANSFER_SYNC);
2623  }
2624
2625  return transfered_buffers;
2626}
2627
2628/**
2629 * Allocate the write request and initialise it for good measure.
2630 *
2631 * @return rtems_blkdev_request* The write reference memory.
2632 */
2633static rtems_blkdev_request*
2634rtems_bdbuf_swapout_writereq_alloc (void)
2635{
2636  /*
2637   * @note chrisj The rtems_blkdev_request and the array at the end is a hack.
2638   * I am disappointment at finding code like this in RTEMS. The request should
2639   * have been a rtems_chain_control. Simple, fast and less storage as the node
2640   * is already part of the buffer structure.
2641   */
2642  rtems_blkdev_request* write_req =
2643    malloc (sizeof (rtems_blkdev_request) +
2644            (bdbuf_config.max_write_blocks * sizeof (rtems_blkdev_sg_buffer)));
2645
2646  if (!write_req)
2647    rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_SO_NOMEM);
2648
2649  write_req->req = RTEMS_BLKDEV_REQ_WRITE;
2650  write_req->req_done = rtems_bdbuf_transfer_done;
2651  write_req->done_arg = write_req;
2652  write_req->io_task = rtems_task_self ();
2653
2654  return write_req;
2655}
2656
2657/**
2658 * The swapout worker thread body.
2659 *
2660 * @param arg A pointer to the worker thread's private data.
2661 * @return rtems_task Not used.
2662 */
2663static rtems_task
2664rtems_bdbuf_swapout_worker_task (rtems_task_argument arg)
2665{
2666  rtems_bdbuf_swapout_worker* worker = (rtems_bdbuf_swapout_worker*) arg;
2667
2668  while (worker->enabled)
2669  {
2670    rtems_bdbuf_wait_for_event (RTEMS_BDBUF_SWAPOUT_SYNC);
2671
2672    rtems_bdbuf_swapout_write (&worker->transfer);
2673
2674    rtems_bdbuf_lock_cache ();
2675
2676    rtems_chain_initialize_empty (&worker->transfer.bds);
2677    worker->transfer.dd = BDBUF_INVALID_DEV;
2678
2679    rtems_chain_append_unprotected (&bdbuf_cache.swapout_workers, &worker->link);
2680
2681    rtems_bdbuf_unlock_cache ();
2682  }
2683
2684  free (worker->transfer.write_req);
2685  free (worker);
2686
2687  rtems_task_delete (RTEMS_SELF);
2688}
2689
2690/**
2691 * Open the swapout worker threads.
2692 */
2693static void
2694rtems_bdbuf_swapout_workers_open (void)
2695{
2696  rtems_status_code sc;
2697  size_t            w;
2698
2699  rtems_bdbuf_lock_cache ();
2700
2701  for (w = 0; w < bdbuf_config.swapout_workers; w++)
2702  {
2703    rtems_bdbuf_swapout_worker* worker;
2704
2705    worker = malloc (sizeof (rtems_bdbuf_swapout_worker));
2706    if (!worker)
2707      rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_SO_NOMEM);
2708
2709    rtems_chain_append_unprotected (&bdbuf_cache.swapout_workers, &worker->link);
2710    worker->enabled = true;
2711    worker->transfer.write_req = rtems_bdbuf_swapout_writereq_alloc ();
2712
2713    rtems_chain_initialize_empty (&worker->transfer.bds);
2714    worker->transfer.dd = BDBUF_INVALID_DEV;
2715
2716    sc = rtems_bdbuf_create_task (rtems_build_name('B', 'D', 'o', 'a' + w),
2717                                  bdbuf_config.swapout_worker_priority,
2718                                  RTEMS_BDBUF_SWAPOUT_WORKER_TASK_PRIORITY_DEFAULT,
2719                                  rtems_bdbuf_swapout_worker_task,
2720                                  (rtems_task_argument) worker,
2721                                  &worker->id);
2722    if (sc != RTEMS_SUCCESSFUL)
2723      rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_SO_WK_CREATE);
2724  }
2725
2726  rtems_bdbuf_unlock_cache ();
2727}
2728
2729/**
2730 * Close the swapout worker threads.
2731 */
2732static void
2733rtems_bdbuf_swapout_workers_close (void)
2734{
2735  rtems_chain_node* node;
2736
2737  rtems_bdbuf_lock_cache ();
2738
2739  node = rtems_chain_first (&bdbuf_cache.swapout_workers);
2740  while (!rtems_chain_is_tail (&bdbuf_cache.swapout_workers, node))
2741  {
2742    rtems_bdbuf_swapout_worker* worker = (rtems_bdbuf_swapout_worker*) node;
2743    worker->enabled = false;
2744    rtems_event_send (worker->id, RTEMS_BDBUF_SWAPOUT_SYNC);
2745    node = rtems_chain_next (node);
2746  }
2747
2748  rtems_bdbuf_unlock_cache ();
2749}
2750
2751/**
2752 * Body of task which takes care on flushing modified buffers to the disk.
2753 *
2754 * @param arg A pointer to the global cache data. Use the global variable and
2755 *            not this.
2756 * @return rtems_task Not used.
2757 */
2758static rtems_task
2759rtems_bdbuf_swapout_task (rtems_task_argument arg)
2760{
2761  rtems_bdbuf_swapout_transfer transfer;
2762  uint32_t                     period_in_ticks;
2763  const uint32_t               period_in_msecs = bdbuf_config.swapout_period;
2764  uint32_t                     timer_delta;
2765
2766  transfer.write_req = rtems_bdbuf_swapout_writereq_alloc ();
2767  rtems_chain_initialize_empty (&transfer.bds);
2768  transfer.dd = BDBUF_INVALID_DEV;
2769  transfer.syncing = false;
2770
2771  /*
2772   * Localise the period.
2773   */
2774  period_in_ticks = RTEMS_MICROSECONDS_TO_TICKS (period_in_msecs * 1000);
2775
2776  /*
2777   * This is temporary. Needs to be changed to use the real time clock.
2778   */
2779  timer_delta = period_in_msecs;
2780
2781  /*
2782   * Create the worker threads.
2783   */
2784  rtems_bdbuf_swapout_workers_open ();
2785
2786  while (bdbuf_cache.swapout_enabled)
2787  {
2788    rtems_event_set   out;
2789    rtems_status_code sc;
2790
2791    /*
2792     * Only update the timers once in the processing cycle.
2793     */
2794    bool update_timers = true;
2795
2796    /*
2797     * If we write buffers to any disk perform a check again. We only write a
2798     * single device at a time and the cache may have more than one device's
2799     * buffers modified waiting to be written.
2800     */
2801    bool transfered_buffers;
2802
2803    do
2804    {
2805      transfered_buffers = false;
2806
2807      /*
2808       * Extact all the buffers we find for a specific device. The device is
2809       * the first one we find on a modified list. Process the sync queue of
2810       * buffers first.
2811       */
2812      if (rtems_bdbuf_swapout_processing (timer_delta,
2813                                          update_timers,
2814                                          &transfer))
2815      {
2816        transfered_buffers = true;
2817      }
2818
2819      /*
2820       * Only update the timers once.
2821       */
2822      update_timers = false;
2823    }
2824    while (transfered_buffers);
2825
2826    sc = rtems_event_receive (RTEMS_BDBUF_SWAPOUT_SYNC,
2827                              RTEMS_EVENT_ALL | RTEMS_WAIT,
2828                              period_in_ticks,
2829                              &out);
2830
2831    if ((sc != RTEMS_SUCCESSFUL) && (sc != RTEMS_TIMEOUT))
2832      rtems_fatal_error_occurred (BLKDEV_FATAL_BDBUF_SWAPOUT_RE);
2833  }
2834
2835  rtems_bdbuf_swapout_workers_close ();
2836
2837  free (transfer.write_req);
2838
2839  rtems_task_delete (RTEMS_SELF);
2840}
2841
2842static void
2843rtems_bdbuf_purge_list (rtems_chain_control *purge_list)
2844{
2845  bool wake_buffer_waiters = false;
2846  rtems_chain_node *node = NULL;
2847
2848  while ((node = rtems_chain_get_unprotected (purge_list)) != NULL)
2849  {
2850    rtems_bdbuf_buffer *bd = (rtems_bdbuf_buffer *) node;
2851
2852    if (bd->waiters == 0)
2853      wake_buffer_waiters = true;
2854
2855    rtems_bdbuf_discard_buffer (bd);
2856  }
2857
2858  if (wake_buffer_waiters)
2859    rtems_bdbuf_wake (&bdbuf_cache.buffer_waiters);
2860}
2861
2862static void
2863rtems_bdbuf_gather_for_purge (rtems_chain_control *purge_list,
2864                              const rtems_disk_device *dd)
2865{
2866  rtems_bdbuf_buffer *stack [RTEMS_BDBUF_AVL_MAX_HEIGHT];
2867  rtems_bdbuf_buffer **prev = stack;
2868  rtems_bdbuf_buffer *cur = bdbuf_cache.tree;
2869
2870  *prev = NULL;
2871
2872  while (cur != NULL)
2873  {
2874    if (cur->dd == dd)
2875    {
2876      switch (cur->state)
2877      {
2878        case RTEMS_BDBUF_STATE_FREE:
2879        case RTEMS_BDBUF_STATE_EMPTY:
2880        case RTEMS_BDBUF_STATE_ACCESS_PURGED:
2881        case RTEMS_BDBUF_STATE_TRANSFER_PURGED:
2882          break;
2883        case RTEMS_BDBUF_STATE_SYNC:
2884          rtems_bdbuf_wake (&bdbuf_cache.transfer_waiters);
2885          /* Fall through */
2886        case RTEMS_BDBUF_STATE_MODIFIED:
2887          rtems_bdbuf_group_release (cur);
2888          /* Fall through */
2889        case RTEMS_BDBUF_STATE_CACHED:
2890          rtems_chain_extract_unprotected (&cur->link);
2891          rtems_chain_append_unprotected (purge_list, &cur->link);
2892          break;
2893        case RTEMS_BDBUF_STATE_TRANSFER:
2894          rtems_bdbuf_set_state (cur, RTEMS_BDBUF_STATE_TRANSFER_PURGED);
2895          break;
2896        case RTEMS_BDBUF_STATE_ACCESS_CACHED:
2897        case RTEMS_BDBUF_STATE_ACCESS_EMPTY:
2898        case RTEMS_BDBUF_STATE_ACCESS_MODIFIED:
2899          rtems_bdbuf_set_state (cur, RTEMS_BDBUF_STATE_ACCESS_PURGED);
2900          break;
2901        default:
2902          rtems_fatal_error_occurred (RTEMS_BLKDEV_FATAL_BDBUF_STATE_11);
2903      }
2904    }
2905
2906    if (cur->avl.left != NULL)
2907    {
2908      /* Left */
2909      ++prev;
2910      *prev = cur;
2911      cur = cur->avl.left;
2912    }
2913    else if (cur->avl.right != NULL)
2914    {
2915      /* Right */
2916      ++prev;
2917      *prev = cur;
2918      cur = cur->avl.right;
2919    }
2920    else
2921    {
2922      while (*prev != NULL
2923             && (cur == (*prev)->avl.right || (*prev)->avl.right == NULL))
2924      {
2925        /* Up */
2926        cur = *prev;
2927        --prev;
2928      }
2929      if (*prev != NULL)
2930        /* Right */
2931        cur = (*prev)->avl.right;
2932      else
2933        /* Finished */
2934        cur = NULL;
2935    }
2936  }
2937}
2938
2939void
2940rtems_bdbuf_purge_dev (rtems_disk_device *dd)
2941{
2942  rtems_chain_control purge_list;
2943
2944  rtems_chain_initialize_empty (&purge_list);
2945  rtems_bdbuf_lock_cache ();
2946  rtems_bdbuf_read_ahead_reset (dd);
2947  rtems_bdbuf_gather_for_purge (&purge_list, dd);
2948  rtems_bdbuf_purge_list (&purge_list);
2949  rtems_bdbuf_unlock_cache ();
2950}
2951
2952rtems_status_code
2953rtems_bdbuf_set_block_size (rtems_disk_device *dd, uint32_t block_size)
2954{
2955  rtems_status_code sc = RTEMS_SUCCESSFUL;
2956
2957  rtems_bdbuf_lock_cache ();
2958
2959  if (block_size > 0)
2960  {
2961    size_t bds_per_group = rtems_bdbuf_bds_per_group (block_size);
2962
2963    if (bds_per_group != 0)
2964    {
2965      int block_to_media_block_shift = 0;
2966      uint32_t media_blocks_per_block = block_size / dd->media_block_size;
2967      uint32_t one = 1;
2968
2969      while ((one << block_to_media_block_shift) < media_blocks_per_block)
2970      {
2971        ++block_to_media_block_shift;
2972      }
2973
2974      if ((dd->media_block_size << block_to_media_block_shift) != block_size)
2975        block_to_media_block_shift = -1;
2976
2977      dd->block_size = block_size;
2978      dd->block_count = dd->size / media_blocks_per_block;
2979      dd->media_blocks_per_block = media_blocks_per_block;
2980      dd->block_to_media_block_shift = block_to_media_block_shift;
2981      dd->bds_per_group = bds_per_group;
2982
2983      rtems_bdbuf_read_ahead_reset (dd);
2984    }
2985    else
2986    {
2987      sc = RTEMS_INVALID_NUMBER;
2988    }
2989  }
2990  else
2991  {
2992    sc = RTEMS_INVALID_NUMBER;
2993  }
2994
2995  rtems_bdbuf_unlock_cache ();
2996
2997  return sc;
2998}
2999
3000static rtems_task
3001rtems_bdbuf_read_ahead_task (rtems_task_argument arg)
3002{
3003  rtems_chain_control *chain = &bdbuf_cache.read_ahead_chain;
3004
3005  while (bdbuf_cache.read_ahead_enabled)
3006  {
3007    rtems_chain_node *node;
3008
3009    rtems_bdbuf_wait_for_event (RTEMS_BDBUF_READ_AHEAD_WAKE_UP);
3010    rtems_bdbuf_lock_cache ();
3011
3012    while ((node = rtems_chain_get_unprotected (chain)) != NULL)
3013    {
3014      rtems_disk_device *dd = (rtems_disk_device *)
3015        ((char *) node - offsetof (rtems_disk_device, read_ahead.node));
3016      rtems_blkdev_bnum block = dd->read_ahead.next;
3017      rtems_blkdev_bnum media_block = 0;
3018      rtems_status_code sc =
3019        rtems_bdbuf_get_media_block (dd, block, &media_block);
3020
3021      rtems_chain_set_off_chain (&dd->read_ahead.node);
3022
3023      if (sc == RTEMS_SUCCESSFUL)
3024      {
3025        rtems_bdbuf_buffer *bd =
3026          rtems_bdbuf_get_buffer_for_read_ahead (dd, media_block);
3027
3028        if (bd != NULL)
3029        {
3030          uint32_t transfer_count = dd->block_count - block;
3031          uint32_t max_transfer_count = bdbuf_config.max_read_ahead_blocks;
3032
3033          if (transfer_count >= max_transfer_count)
3034          {
3035            transfer_count = max_transfer_count;
3036            dd->read_ahead.trigger = block + transfer_count / 2;
3037            dd->read_ahead.next = block + transfer_count;
3038          }
3039          else
3040          {
3041            dd->read_ahead.trigger = RTEMS_DISK_READ_AHEAD_NO_TRIGGER;
3042          }
3043
3044          ++dd->stats.read_ahead_transfers;
3045          rtems_bdbuf_execute_read_request (dd, bd, transfer_count);
3046        }
3047      }
3048      else
3049      {
3050        dd->read_ahead.trigger = RTEMS_DISK_READ_AHEAD_NO_TRIGGER;
3051      }
3052    }
3053
3054    rtems_bdbuf_unlock_cache ();
3055  }
3056
3057  rtems_task_delete (RTEMS_SELF);
3058}
3059
3060void rtems_bdbuf_get_device_stats (const rtems_disk_device *dd,
3061                                   rtems_blkdev_stats      *stats)
3062{
3063  rtems_bdbuf_lock_cache ();
3064  *stats = dd->stats;
3065  rtems_bdbuf_unlock_cache ();
3066}
3067
3068void rtems_bdbuf_reset_device_stats (rtems_disk_device *dd)
3069{
3070  rtems_bdbuf_lock_cache ();
3071  memset (&dd->stats, 0, sizeof(dd->stats));
3072  rtems_bdbuf_unlock_cache ();
3073}
Note: See TracBrowser for help on using the repository browser.