1 | /*- |
---|
2 | * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org> |
---|
3 | * Copyright (c) 2001 Jason Evans <jasone@freebsd.org> |
---|
4 | * All rights reserved. |
---|
5 | * |
---|
6 | * Redistribution and use in source and binary forms, with or without |
---|
7 | * modification, are permitted provided that the following conditions |
---|
8 | * are met: |
---|
9 | * 1. Redistributions of source code must retain the above copyright |
---|
10 | * notice(s), this list of conditions and the following disclaimer as |
---|
11 | * the first lines of this file unmodified other than the possible |
---|
12 | * addition of one or more copyright notices. |
---|
13 | * 2. Redistributions in binary form must reproduce the above copyright |
---|
14 | * notice(s), this list of conditions and the following disclaimer in the |
---|
15 | * documentation and/or other materials provided with the distribution. |
---|
16 | * |
---|
17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY |
---|
18 | * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
---|
19 | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
---|
20 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY |
---|
21 | * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
---|
22 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR |
---|
23 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
---|
24 | * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
---|
25 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
---|
26 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH |
---|
27 | * DAMAGE. |
---|
28 | * |
---|
29 | * $FreeBSD$ |
---|
30 | */ |
---|
31 | |
---|
32 | #ifndef _SYS_SX_H_ |
---|
33 | #define _SYS_SX_H_ |
---|
34 | |
---|
35 | #include <sys/_lock.h> |
---|
36 | #include <sys/_sx.h> |
---|
37 | |
---|
38 | #ifdef _KERNEL |
---|
39 | #include <sys/pcpu.h> |
---|
40 | #include <sys/lock_profile.h> |
---|
41 | #include <sys/lockstat.h> |
---|
42 | #include <machine/atomic.h> |
---|
43 | #endif |
---|
44 | |
---|
45 | #ifdef __rtems__ |
---|
46 | #define SX_NOINLINE 1 |
---|
47 | #define _sx_slock _bsd__sx_xlock |
---|
48 | #define _sx_try_slock _bsd__sx_try_xlock |
---|
49 | #define _sx_sunlock _bsd__sx_xunlock |
---|
50 | #endif /* __rtems__ */ |
---|
51 | /* |
---|
52 | * In general, the sx locks and rwlocks use very similar algorithms. |
---|
53 | * The main difference in the implementations is how threads are |
---|
54 | * blocked when a lock is unavailable. For this, sx locks use sleep |
---|
55 | * queues which do not support priority propagation, and rwlocks use |
---|
56 | * turnstiles which do. |
---|
57 | * |
---|
58 | * The sx_lock field consists of several fields. The low bit |
---|
59 | * indicates if the lock is locked with a shared or exclusive lock. A |
---|
60 | * value of 0 indicates an exclusive lock, and a value of 1 indicates |
---|
61 | * a shared lock. Bit 1 is a boolean indicating if there are any |
---|
62 | * threads waiting for a shared lock. Bit 2 is a boolean indicating |
---|
63 | * if there are any threads waiting for an exclusive lock. Bit 3 is a |
---|
64 | * boolean indicating if an exclusive lock is recursively held. The |
---|
65 | * rest of the variable's definition is dependent on the value of the |
---|
66 | * first bit. For an exclusive lock, it is a pointer to the thread |
---|
67 | * holding the lock, similar to the mtx_lock field of mutexes. For |
---|
68 | * shared locks, it is a count of read locks that are held. |
---|
69 | * |
---|
70 | * When the lock is not locked by any thread, it is encoded as a |
---|
71 | * shared lock with zero waiters. |
---|
72 | */ |
---|
73 | |
---|
74 | #define SX_LOCK_SHARED 0x01 |
---|
75 | #define SX_LOCK_SHARED_WAITERS 0x02 |
---|
76 | #define SX_LOCK_EXCLUSIVE_WAITERS 0x04 |
---|
77 | #define SX_LOCK_RECURSED 0x08 |
---|
78 | #define SX_LOCK_FLAGMASK \ |
---|
79 | (SX_LOCK_SHARED | SX_LOCK_SHARED_WAITERS | \ |
---|
80 | SX_LOCK_EXCLUSIVE_WAITERS | SX_LOCK_RECURSED) |
---|
81 | |
---|
82 | #define SX_OWNER(x) ((x) & ~SX_LOCK_FLAGMASK) |
---|
83 | #define SX_SHARERS_SHIFT 4 |
---|
84 | #define SX_SHARERS(x) (SX_OWNER(x) >> SX_SHARERS_SHIFT) |
---|
85 | #define SX_SHARERS_LOCK(x) \ |
---|
86 | ((x) << SX_SHARERS_SHIFT | SX_LOCK_SHARED) |
---|
87 | #define SX_ONE_SHARER (1 << SX_SHARERS_SHIFT) |
---|
88 | |
---|
89 | #define SX_LOCK_UNLOCKED SX_SHARERS_LOCK(0) |
---|
90 | #define SX_LOCK_DESTROYED \ |
---|
91 | (SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS) |
---|
92 | |
---|
93 | #ifdef _KERNEL |
---|
94 | |
---|
95 | /* |
---|
96 | * Function prototipes. Routines that start with an underscore are not part |
---|
97 | * of the public interface and are wrappered with a macro. |
---|
98 | */ |
---|
99 | void sx_sysinit(void *arg); |
---|
100 | #define sx_init(sx, desc) sx_init_flags((sx), (desc), 0) |
---|
101 | void sx_init_flags(struct sx *sx, const char *description, int opts); |
---|
102 | void sx_destroy(struct sx *sx); |
---|
103 | int _sx_slock(struct sx *sx, int opts, const char *file, int line); |
---|
104 | int _sx_xlock(struct sx *sx, int opts, const char *file, int line); |
---|
105 | int _sx_try_slock(struct sx *sx, const char *file, int line); |
---|
106 | int _sx_try_xlock(struct sx *sx, const char *file, int line); |
---|
107 | void _sx_sunlock(struct sx *sx, const char *file, int line); |
---|
108 | void _sx_xunlock(struct sx *sx, const char *file, int line); |
---|
109 | int _sx_try_upgrade(struct sx *sx, const char *file, int line); |
---|
110 | void _sx_downgrade(struct sx *sx, const char *file, int line); |
---|
111 | int _sx_xlock_hard(struct sx *sx, uintptr_t tid, int opts, |
---|
112 | const char *file, int line); |
---|
113 | int _sx_slock_hard(struct sx *sx, int opts, const char *file, int line); |
---|
114 | void _sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int |
---|
115 | line); |
---|
116 | void _sx_sunlock_hard(struct sx *sx, const char *file, int line); |
---|
117 | #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) |
---|
118 | void _sx_assert(struct sx *sx, int what, const char *file, int line); |
---|
119 | #endif |
---|
120 | #ifdef DDB |
---|
121 | int sx_chain(struct thread *td, struct thread **ownerp); |
---|
122 | #endif |
---|
123 | |
---|
124 | #define sx_downgrade_(sx, file, line) \ |
---|
125 | _sx_downgrade((sx), (file), (line)) |
---|
126 | #define sx_try_slock_(sx, file, line) \ |
---|
127 | _sx_try_slock((sx), (file), (line)) |
---|
128 | #define sx_try_xlock_(sx, file, line) \ |
---|
129 | _sx_try_xlock((sx), (file), (line)) |
---|
130 | #define sx_try_upgrade_(sx, file, line) \ |
---|
131 | _sx_try_upgrade((sx), (file), (line)) |
---|
132 | |
---|
133 | struct sx_args { |
---|
134 | struct sx *sa_sx; |
---|
135 | const char *sa_desc; |
---|
136 | int sa_flags; |
---|
137 | }; |
---|
138 | |
---|
139 | #define SX_SYSINIT_FLAGS(name, sxa, desc, flags) \ |
---|
140 | static struct sx_args name##_args = { \ |
---|
141 | (sxa), \ |
---|
142 | (desc), \ |
---|
143 | (flags) \ |
---|
144 | }; \ |
---|
145 | SYSINIT(name##_sx_sysinit, SI_SUB_LOCK, SI_ORDER_MIDDLE, \ |
---|
146 | sx_sysinit, &name##_args); \ |
---|
147 | SYSUNINIT(name##_sx_sysuninit, SI_SUB_LOCK, SI_ORDER_MIDDLE, \ |
---|
148 | sx_destroy, (sxa)) |
---|
149 | |
---|
150 | #define SX_SYSINIT(name, sxa, desc) SX_SYSINIT_FLAGS(name, sxa, desc, 0) |
---|
151 | |
---|
152 | #ifndef __rtems__ |
---|
153 | /* |
---|
154 | * Full lock operations that are suitable to be inlined in non-debug kernels. |
---|
155 | * If the lock can't be acquired or released trivially then the work is |
---|
156 | * deferred to 'tougher' functions. |
---|
157 | */ |
---|
158 | |
---|
159 | /* Acquire an exclusive lock. */ |
---|
160 | static __inline int |
---|
161 | __sx_xlock(struct sx *sx, struct thread *td, int opts, const char *file, |
---|
162 | int line) |
---|
163 | { |
---|
164 | uintptr_t tid = (uintptr_t)td; |
---|
165 | int error = 0; |
---|
166 | |
---|
167 | if (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) |
---|
168 | error = _sx_xlock_hard(sx, tid, opts, file, line); |
---|
169 | else |
---|
170 | LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_XLOCK_ACQUIRE, |
---|
171 | sx, 0, 0, file, line); |
---|
172 | |
---|
173 | return (error); |
---|
174 | } |
---|
175 | |
---|
176 | /* Release an exclusive lock. */ |
---|
177 | static __inline void |
---|
178 | __sx_xunlock(struct sx *sx, struct thread *td, const char *file, int line) |
---|
179 | { |
---|
180 | uintptr_t tid = (uintptr_t)td; |
---|
181 | |
---|
182 | if (!atomic_cmpset_rel_ptr(&sx->sx_lock, tid, SX_LOCK_UNLOCKED)) |
---|
183 | _sx_xunlock_hard(sx, tid, file, line); |
---|
184 | } |
---|
185 | |
---|
186 | /* Acquire a shared lock. */ |
---|
187 | static __inline int |
---|
188 | __sx_slock(struct sx *sx, int opts, const char *file, int line) |
---|
189 | { |
---|
190 | uintptr_t x = sx->sx_lock; |
---|
191 | int error = 0; |
---|
192 | |
---|
193 | if (!(x & SX_LOCK_SHARED) || |
---|
194 | !atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER)) |
---|
195 | error = _sx_slock_hard(sx, opts, file, line); |
---|
196 | else |
---|
197 | LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_SLOCK_ACQUIRE, sx, 0, |
---|
198 | 0, file, line); |
---|
199 | |
---|
200 | return (error); |
---|
201 | } |
---|
202 | |
---|
203 | /* |
---|
204 | * Release a shared lock. We can just drop a single shared lock so |
---|
205 | * long as we aren't trying to drop the last shared lock when other |
---|
206 | * threads are waiting for an exclusive lock. This takes advantage of |
---|
207 | * the fact that an unlocked lock is encoded as a shared lock with a |
---|
208 | * count of 0. |
---|
209 | */ |
---|
210 | static __inline void |
---|
211 | __sx_sunlock(struct sx *sx, const char *file, int line) |
---|
212 | { |
---|
213 | uintptr_t x = sx->sx_lock; |
---|
214 | |
---|
215 | if (x == (SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS) || |
---|
216 | !atomic_cmpset_rel_ptr(&sx->sx_lock, x, x - SX_ONE_SHARER)) |
---|
217 | _sx_sunlock_hard(sx, file, line); |
---|
218 | } |
---|
219 | #endif /* __rtems__ */ |
---|
220 | |
---|
221 | /* |
---|
222 | * Public interface for lock operations. |
---|
223 | */ |
---|
224 | #ifndef LOCK_DEBUG |
---|
225 | #error "LOCK_DEBUG not defined, include <sys/lock.h> before <sys/sx.h>" |
---|
226 | #endif |
---|
227 | #if (LOCK_DEBUG > 0) || defined(SX_NOINLINE) |
---|
228 | #define sx_xlock_(sx, file, line) \ |
---|
229 | (void)_sx_xlock((sx), 0, (file), (line)) |
---|
230 | #define sx_xlock_sig_(sx, file, line) \ |
---|
231 | _sx_xlock((sx), SX_INTERRUPTIBLE, (file), (line)) |
---|
232 | #define sx_xunlock_(sx, file, line) \ |
---|
233 | _sx_xunlock((sx), (file), (line)) |
---|
234 | #define sx_slock_(sx, file, line) \ |
---|
235 | (void)_sx_slock((sx), 0, (file), (line)) |
---|
236 | #define sx_slock_sig_(sx, file, line) \ |
---|
237 | _sx_slock((sx), SX_INTERRUPTIBLE, (file) , (line)) |
---|
238 | #define sx_sunlock_(sx, file, line) \ |
---|
239 | _sx_sunlock((sx), (file), (line)) |
---|
240 | #else |
---|
241 | #define sx_xlock_(sx, file, line) \ |
---|
242 | (void)__sx_xlock((sx), curthread, 0, (file), (line)) |
---|
243 | #define sx_xlock_sig_(sx, file, line) \ |
---|
244 | __sx_xlock((sx), curthread, SX_INTERRUPTIBLE, (file), (line)) |
---|
245 | #define sx_xunlock_(sx, file, line) \ |
---|
246 | __sx_xunlock((sx), curthread, (file), (line)) |
---|
247 | #define sx_slock_(sx, file, line) \ |
---|
248 | (void)__sx_slock((sx), 0, (file), (line)) |
---|
249 | #define sx_slock_sig_(sx, file, line) \ |
---|
250 | __sx_slock((sx), SX_INTERRUPTIBLE, (file), (line)) |
---|
251 | #define sx_sunlock_(sx, file, line) \ |
---|
252 | __sx_sunlock((sx), (file), (line)) |
---|
253 | #endif /* LOCK_DEBUG > 0 || SX_NOINLINE */ |
---|
254 | #define sx_try_slock(sx) sx_try_slock_((sx), LOCK_FILE, LOCK_LINE) |
---|
255 | #define sx_try_xlock(sx) sx_try_xlock_((sx), LOCK_FILE, LOCK_LINE) |
---|
256 | #define sx_try_upgrade(sx) sx_try_upgrade_((sx), LOCK_FILE, LOCK_LINE) |
---|
257 | #define sx_downgrade(sx) sx_downgrade_((sx), LOCK_FILE, LOCK_LINE) |
---|
258 | #ifdef INVARIANTS |
---|
259 | #define sx_assert_(sx, what, file, line) \ |
---|
260 | _sx_assert((sx), (what), (file), (line)) |
---|
261 | #else |
---|
262 | #define sx_assert_(sx, what, file, line) (void)0 |
---|
263 | #endif |
---|
264 | |
---|
265 | #define sx_xlock(sx) sx_xlock_((sx), LOCK_FILE, LOCK_LINE) |
---|
266 | #define sx_xlock_sig(sx) sx_xlock_sig_((sx), LOCK_FILE, LOCK_LINE) |
---|
267 | #define sx_xunlock(sx) sx_xunlock_((sx), LOCK_FILE, LOCK_LINE) |
---|
268 | #define sx_slock(sx) sx_slock_((sx), LOCK_FILE, LOCK_LINE) |
---|
269 | #define sx_slock_sig(sx) sx_slock_sig_((sx), LOCK_FILE, LOCK_LINE) |
---|
270 | #define sx_sunlock(sx) sx_sunlock_((sx), LOCK_FILE, LOCK_LINE) |
---|
271 | #define sx_assert(sx, what) sx_assert_((sx), (what), __FILE__, __LINE__) |
---|
272 | |
---|
273 | /* |
---|
274 | * Return a pointer to the owning thread if the lock is exclusively |
---|
275 | * locked. |
---|
276 | */ |
---|
277 | #ifndef __rtems__ |
---|
278 | #define sx_xholder(sx) \ |
---|
279 | ((sx)->sx_lock & SX_LOCK_SHARED ? NULL : \ |
---|
280 | (struct thread *)SX_OWNER((sx)->sx_lock)) |
---|
281 | |
---|
282 | #define sx_xlocked(sx) \ |
---|
283 | (((sx)->sx_lock & ~(SX_LOCK_FLAGMASK & ~SX_LOCK_SHARED)) == \ |
---|
284 | (uintptr_t)curthread) |
---|
285 | #else /* __rtems__ */ |
---|
286 | int sx_xlocked(struct sx *sx); |
---|
287 | #endif /* __rtems__ */ |
---|
288 | |
---|
289 | #define sx_unlock_(sx, file, line) do { \ |
---|
290 | if (sx_xlocked(sx)) \ |
---|
291 | sx_xunlock_(sx, file, line); \ |
---|
292 | else \ |
---|
293 | sx_sunlock_(sx, file, line); \ |
---|
294 | } while (0) |
---|
295 | |
---|
296 | #define sx_unlock(sx) sx_unlock_((sx), LOCK_FILE, LOCK_LINE) |
---|
297 | |
---|
298 | #define sx_sleep(chan, sx, pri, wmesg, timo) \ |
---|
299 | _sleep((chan), &(sx)->lock_object, (pri), (wmesg), (timo)) |
---|
300 | |
---|
301 | /* |
---|
302 | * Options passed to sx_init_flags(). |
---|
303 | */ |
---|
304 | #define SX_DUPOK 0x01 |
---|
305 | #define SX_NOPROFILE 0x02 |
---|
306 | #define SX_NOWITNESS 0x04 |
---|
307 | #define SX_QUIET 0x08 |
---|
308 | #define SX_NOADAPTIVE 0x10 |
---|
309 | #define SX_RECURSE 0x20 |
---|
310 | |
---|
311 | /* |
---|
312 | * Options passed to sx_*lock_hard(). |
---|
313 | */ |
---|
314 | #define SX_INTERRUPTIBLE 0x40 |
---|
315 | |
---|
316 | #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) |
---|
317 | #define SA_LOCKED LA_LOCKED |
---|
318 | #define SA_SLOCKED LA_SLOCKED |
---|
319 | #define SA_XLOCKED LA_XLOCKED |
---|
320 | #define SA_UNLOCKED LA_UNLOCKED |
---|
321 | #define SA_RECURSED LA_RECURSED |
---|
322 | #define SA_NOTRECURSED LA_NOTRECURSED |
---|
323 | |
---|
324 | /* Backwards compatability. */ |
---|
325 | #define SX_LOCKED LA_LOCKED |
---|
326 | #define SX_SLOCKED LA_SLOCKED |
---|
327 | #define SX_XLOCKED LA_XLOCKED |
---|
328 | #define SX_UNLOCKED LA_UNLOCKED |
---|
329 | #define SX_RECURSED LA_RECURSED |
---|
330 | #define SX_NOTRECURSED LA_NOTRECURSED |
---|
331 | #endif |
---|
332 | |
---|
333 | #endif /* _KERNEL */ |
---|
334 | |
---|
335 | #endif /* !_SYS_SX_H_ */ |
---|