Context Navigation

ck_pr.h

6-freebsd-12

Last change on this file was 1af372a, checked in by Sebastian Huber <sebastian.huber@…>, on 09/14/18 at 12:04:09

ck: No hardware barriers in uniprocessor configs

Update #3472.

Property mode set to 100644

File size: 15.0 KB

Line
1	/*
2	* Copyright 2009-2015 Samy Al Bahra.
3	* Copyright 2013-2015 Olivier Houchard.
4	* All rights reserved.
5	*
6	* Redistribution and use in source and binary forms, with or without
7	* modification, are permitted provided that the following conditions
8	* are met:
9	* 1. Redistributions of source code must retain the above copyright
10	* notice, this list of conditions and the following disclaimer.
11	* 2. Redistributions in binary form must reproduce the above copyright
12	* notice, this list of conditions and the following disclaimer in the
13	* documentation and/or other materials provided with the distribution.
14	*
15	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25	* SUCH DAMAGE.
26	*/
27
28	#ifndef CK_PR_ARM_H
29	#define CK_PR_ARM_H
30
31	#ifndef CK_PR_H
32	#error Do not include this file directly, use ck_pr.h
33	#endif
34
35	#include <ck_cc.h>
36	#include <ck_md.h>
37
38	/*
39	* The following represent supported atomic operations.
40	* These operations may be emulated.
41	*/
42	#include "ck_f_pr.h"
43
44	/*
45	* Minimum interface requirement met.
46	*/
47	#define CK_F_PR
48
49	CK_CC_INLINE static void
50	ck_pr_stall(void)
51	{
52
53	__asm__ __volatile__("" ::: "memory");
54	return;
55	}
56
57	#if __ARM_ARCH >= 7 \|\| defined(__ARM_ARCH_7__) \|\| defined(__ARM_ARCH_7A__)
58	#ifdef RTEMS_SMP
59	#define CK_ISB __asm __volatile("isb" : : "r" (0) : "memory")
60	#define CK_DMB __asm __volatile("dmb" : : "r" (0) : "memory")
61	#define CK_DSB __asm __volatile("dsb" : : "r" (0) : "memory")
62	/* FreeBSD's toolchain doesn't accept dmb st, so use the opcode instead */
63	#if defined(__FreeBSD__) && !defined(__rtems__)
64	#define CK_DMB_ST __asm __volatile(".word 0xf57ff05e" : : "r" (0) : "memory")
65	#else
66	#define CK_DMB_ST __asm __volatile("dmb st" : : "r" (0) : "memory")
67	#endif /* __FreeBSD__ */
68	#else /* !RTEMS_SMP */
69	#define CK_ISB __asm __volatile("" : : "r" (0) : "memory")
70	#define CK_DMB __asm __volatile("" : : "r" (0) : "memory")
71	#define CK_DSB __asm __volatile("" : : "r" (0) : "memory")
72	#define CK_DMB_ST __asm __volatile("" : : "r" (0) : "memory")
73	#endif /* RTEMS_SMP */
74	#else
75	/* armv6 doesn't have dsb/dmb/isb, and no way to wait only for stores */
76	#define CK_ISB \
77	__asm __volatile("mcr p15, 0, %0, c7, c5, 4" : : "r" (0) : "memory")
78	#define CK_DSB \
79	__asm __volatile("mcr p15, 0, %0, c7, c10, 4" : : "r" (0) : "memory")
80	#define CK_DMB \
81	__asm __volatile("mcr p15, 0, %0, c7, c10, 5" : : "r" (0) : "memory")
82	#define CK_DMB_ST CK_DMB
83	#endif
84
85	#define CK_PR_FENCE(T, I) \
86	CK_CC_INLINE static void \
87	ck_pr_fence_strict_##T(void) \
88	{ \
89	I; \
90	}
91
92	CK_PR_FENCE(atomic, CK_DMB_ST)
93	CK_PR_FENCE(atomic_store, CK_DMB_ST)
94	CK_PR_FENCE(atomic_load, CK_DMB_ST)
95	CK_PR_FENCE(store_atomic, CK_DMB_ST)
96	CK_PR_FENCE(load_atomic, CK_DMB)
97	CK_PR_FENCE(store, CK_DMB_ST)
98	CK_PR_FENCE(store_load, CK_DMB)
99	CK_PR_FENCE(load, CK_DMB)
100	CK_PR_FENCE(load_store, CK_DMB)
101	CK_PR_FENCE(memory, CK_DMB)
102	CK_PR_FENCE(acquire, CK_DMB)
103	CK_PR_FENCE(release, CK_DMB)
104	CK_PR_FENCE(acqrel, CK_DMB)
105	CK_PR_FENCE(lock, CK_DMB)
106	CK_PR_FENCE(unlock, CK_DMB)
107
108	#undef CK_PR_FENCE
109
110	#undef CK_ISB
111	#undef CK_DSB
112	#undef CK_DMB
113	#undef CK_DMB_ST
114
115	#define CK_PR_LOAD(S, M, T, C, I) \
116	CK_CC_INLINE static T \
117	ck_pr_md_load_##S(const M *target) \
118	{ \
119	long r = 0; \
120	__asm__ __volatile__(I " %0, [%1];" \
121	: "=r" (r) \
122	: "r" (target) \
123	: "memory"); \
124	return ((T)r); \
125	}
126
127	CK_PR_LOAD(ptr, void, void *, uint32_t, "ldr")
128
129	#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I)
130
131	CK_PR_LOAD_S(32, uint32_t, "ldr")
132	CK_PR_LOAD_S(16, uint16_t, "ldrh")
133	CK_PR_LOAD_S(8, uint8_t, "ldrb")
134	CK_PR_LOAD_S(uint, unsigned int, "ldr")
135	CK_PR_LOAD_S(int, int, "ldr")
136	CK_PR_LOAD_S(short, short, "ldrh")
137	CK_PR_LOAD_S(char, char, "ldrb")
138
139	#undef CK_PR_LOAD_S
140	#undef CK_PR_LOAD
141
142	#if defined(__ARM_ARCH_7__) \|\| defined(__ARM_ARCH_7A__)
143
144	#define CK_PR_DOUBLE_LOAD(T, N) \
145	CK_CC_INLINE static T \
146	ck_pr_md_load_##N(const T *target) \
147	{ \
148	register T ret; \
149	\
150	__asm __volatile("ldrexd %0, [%1]" \
151	: "=&r" (ret) \
152	: "r" (target) \
153	: "memory", "cc"); \
154	return (ret); \
155	}
156
157	CK_PR_DOUBLE_LOAD(uint64_t, 64)
158	#ifndef CK_PR_DISABLE_DOUBLE
159	CK_PR_DOUBLE_LOAD(double, double)
160	#endif
161	#undef CK_PR_DOUBLE_LOAD
162	#endif
163
164	#define CK_PR_STORE(S, M, T, C, I) \
165	CK_CC_INLINE static void \
166	ck_pr_md_store_##S(M *target, T v) \
167	{ \
168	__asm__ __volatile__(I " %1, [%0]" \
169	: \
170	: "r" (target), \
171	"r" (v) \
172	: "memory"); \
173	return; \
174	}
175
176	CK_PR_STORE(ptr, void, const void *, uint32_t, "str")
177
178	#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I)
179
180	CK_PR_STORE_S(32, uint32_t, "str")
181	CK_PR_STORE_S(16, uint16_t, "strh")
182	CK_PR_STORE_S(8, uint8_t, "strb")
183	CK_PR_STORE_S(uint, unsigned int, "str")
184	CK_PR_STORE_S(int, int, "str")
185	CK_PR_STORE_S(short, short, "strh")
186	CK_PR_STORE_S(char, char, "strb")
187
188	#undef CK_PR_STORE_S
189	#undef CK_PR_STORE
190
191	#if defined(__ARM_ARCH_7__) \|\| defined(__ARM_ARCH_7A__)
192
193	#define CK_PR_DOUBLE_STORE(T, N) \
194	CK_CC_INLINE static void \
195	ck_pr_md_store_##N(const T *target, T value) \
196	{ \
197	T tmp; \
198	uint32_t flag; \
199	__asm __volatile("1: \n" \
200	"ldrexd %0, [%2]\n" \
201	"strexd %1, %3, [%2]\n" \
202	"teq %1, #0\n" \
203	"it ne \n" \
204	"bne 1b\n" \
205	: "=&r" (tmp), "=&r" (flag) \
206	: "r" (target), "r" (value) \
207	: "memory", "cc"); \
208	}
209
210	CK_PR_DOUBLE_STORE(uint64_t, 64)
211	#ifndef CK_PR_DISABLE_DOUBLE
212	CK_PR_DOUBLE_STORE(double, double)
213	#endif
214
215	#undef CK_PR_DOUBLE_STORE
216
217	#define CK_PR_DOUBLE_CAS_VALUE(T, N) \
218	CK_CC_INLINE static bool \
219	ck_pr_cas_##N##_value(T target, T compare, T set, T value) \
220	{ \
221	T previous; \
222	int tmp; \
223	\
224	__asm__ __volatile__("1:" \
225	"ldrexd %0, [%4];" \
226	"cmp %Q0, %Q2;" \
227	"ittt eq;" \
228	"cmpeq %R0, %R2;" \
229	"strexdeq %1, %3, [%4];" \
230	"cmpeq %1, #1;" \
231	"beq 1b;" \
232	:"=&r" (previous), "=&r" (tmp) \
233	: "r" (compare), "r" (set) , \
234	"r"(target) \
235	: "memory", "cc"); \
236	*value = previous; \
237	return (*value == compare); \
238	}
239
240	CK_PR_DOUBLE_CAS_VALUE(uint64_t, 64)
241	#ifndef CK_PR_DISABLE_DOUBLE
242	CK_PR_DOUBLE_CAS_VALUE(double, double)
243	#endif
244
245	#undef CK_PR_DOUBLE_CAS_VALUE
246
247	CK_CC_INLINE static bool
248	ck_pr_cas_ptr_2_value(void target, void compare, void set, void value)
249	{
250	uint32_t _compare = CK_CPP_CAST(uint32_t , compare);
251	uint32_t _set = CK_CPP_CAST(uint32_t , set);
252	uint64_t __compare = ((uint64_t)_compare[0]) \| ((uint64_t)_compare[1] << 32);
253	uint64_t __set = ((uint64_t)_set[0]) \| ((uint64_t)_set[1] << 32);
254
255	return (ck_pr_cas_64_value(CK_CPP_CAST(uint64_t *, target),
256	__compare,
257	__set,
258	CK_CPP_CAST(uint64_t *, value)));
259	}
260
261	#define CK_PR_DOUBLE_CAS(T, N) \
262	CK_CC_INLINE static bool \
263	ck_pr_cas_##N(T *target, T compare, T set) \
264	{ \
265	int ret; \
266	T tmp; \
267	\
268	__asm__ __volatile__("1:" \
269	"mov %0, #0;" \
270	"ldrexd %1, [%4];" \
271	"cmp %Q1, %Q2;" \
272	"itttt eq;" \
273	"cmpeq %R1, %R2;" \
274	"strexdeq %1, %3, [%4];" \
275	"moveq %0, #1;" \
276	"cmpeq %1, #1;" \
277	"beq 1b;" \
278	: "=&r" (ret), "=&r" (tmp) \
279	: "r" (compare), "r" (set) , \
280	"r"(target) \
281	: "memory", "cc"); \
282	\
283	return (ret); \
284	}
285
286	CK_PR_DOUBLE_CAS(uint64_t, 64)
287	#ifndef CK_PR_DISABLE_DOUBLE
288	CK_PR_DOUBLE_CAS(double, double)
289	#endif
290
291	CK_CC_INLINE static bool
292	ck_pr_cas_ptr_2(void target, void compare, void *set)
293	{
294	uint32_t _compare = CK_CPP_CAST(uint32_t , compare);
295	uint32_t _set = CK_CPP_CAST(uint32_t , set);
296	uint64_t __compare = ((uint64_t)_compare[0]) \| ((uint64_t)_compare[1] << 32);
297	uint64_t __set = ((uint64_t)_set[0]) \| ((uint64_t)_set[1] << 32);
298	return (ck_pr_cas_64(CK_CPP_CAST(uint64_t *, target),
299	__compare,
300	__set));
301	}
302
303	#endif
304
305	CK_CC_INLINE static bool
306	ck_pr_cas_ptr_value(void target, void compare, void set, void value)
307	{
308	void previous, tmp;
309	__asm__ __volatile__("1:"
310	"ldrex %0, [%2];"
311	"cmp %0, %4;"
312	"itt eq;"
313	"strexeq %1, %3, [%2];"
314	"cmpeq %1, #1;"
315	"beq 1b;"
316	: "=&r" (previous),
317	"=&r" (tmp)
318	: "r" (target),
319	"r" (set),
320	"r" (compare)
321	: "memory", "cc");
322	(void *)value = previous;
323	return (previous == compare);
324	}
325
326	CK_CC_INLINE static bool
327	ck_pr_cas_ptr(void target, void compare, void *set)
328	{
329	void previous, tmp;
330	__asm__ __volatile__("1:"
331	"ldrex %0, [%2];"
332	"cmp %0, %4;"
333	"itt eq;"
334	"strexeq %1, %3, [%2];"
335	"cmpeq %1, #1;"
336	"beq 1b;"
337	: "=&r" (previous),
338	"=&r" (tmp)
339	: "r" (target),
340	"r" (set),
341	"r" (compare)
342	: "memory", "cc");
343	return (previous == compare);
344	}
345
346	#define CK_PR_CAS(N, T, W) \
347	CK_CC_INLINE static bool \
348	ck_pr_cas_##N##_value(T target, T compare, T set, T value) \
349	{ \
350	T previous = 0, tmp = 0; \
351	__asm__ __volatile__("1:" \
352	"ldrex" W " %0, [%2];" \
353	"cmp %0, %4;" \
354	"itt eq;" \
355	"strex" W "eq %1, %3, [%2];" \
356	"cmpeq %1, #1;" \
357	"beq 1b;" \
358	/* \
359	* Using "+&" instead of "=&" to avoid bogus \
360	* clang warnings. \
361	*/ \
362	: "+&r" (previous), \
363	"+&r" (tmp) \
364	: "r" (target), \
365	"r" (set), \
366	"r" (compare) \
367	: "memory", "cc"); \
368	*value = previous; \
369	return (previous == compare); \
370	} \
371	CK_CC_INLINE static bool \
372	ck_pr_cas_##N(T *target, T compare, T set) \
373	{ \
374	T previous = 0, tmp = 0; \
375	__asm__ __volatile__("1:" \
376	"ldrex" W " %0, [%2];" \
377	"cmp %0, %4;" \
378	"itt eq;" \
379	"strex" W "eq %1, %3, [%2];" \
380	"cmpeq %1, #1;" \
381	"beq 1b;" \
382	: "+&r" (previous), \
383	"+&r" (tmp) \
384	: "r" (target), \
385	"r" (set), \
386	"r" (compare) \
387	: "memory", "cc"); \
388	return (previous == compare); \
389	}
390
391	CK_PR_CAS(32, uint32_t, "")
392	CK_PR_CAS(uint, unsigned int, "")
393	CK_PR_CAS(int, int, "")
394	CK_PR_CAS(16, uint16_t, "h")
395	CK_PR_CAS(8, uint8_t, "b")
396	CK_PR_CAS(short, short, "h")
397	CK_PR_CAS(char, char, "b")
398
399
400	#undef CK_PR_CAS
401
402	#define CK_PR_FAS(N, M, T, W) \
403	CK_CC_INLINE static T \
404	ck_pr_fas_##N(M *target, T v) \
405	{ \
406	T previous = 0; \
407	T tmp = 0; \
408	__asm__ __volatile__("1:" \
409	"ldrex" W " %0, [%2];" \
410	"strex" W " %1, %3, [%2];" \
411	"cmp %1, #0;" \
412	"bne 1b;" \
413	: "+&r" (previous), \
414	"+&r" (tmp) \
415	: "r" (target), \
416	"r" (v) \
417	: "memory", "cc"); \
418	return (previous); \
419	}
420
421	CK_PR_FAS(32, uint32_t, uint32_t, "")
422	CK_PR_FAS(ptr, void, void *, "")
423	CK_PR_FAS(int, int, int, "")
424	CK_PR_FAS(uint, unsigned int, unsigned int, "")
425	CK_PR_FAS(16, uint16_t, uint16_t, "h")
426	CK_PR_FAS(8, uint8_t, uint8_t, "b")
427	CK_PR_FAS(short, short, short, "h")
428	CK_PR_FAS(char, char, char, "b")
429
430
431	#undef CK_PR_FAS
432
433	#define CK_PR_UNARY(O, N, M, T, I, W) \
434	CK_CC_INLINE static void \
435	ck_pr_##O##_##N(M *target) \
436	{ \
437	T previous = 0; \
438	T tmp = 0; \
439	__asm__ __volatile__("1:" \
440	"ldrex" W " %0, [%2];" \
441	I ";" \
442	"strex" W " %1, %0, [%2];" \
443	"cmp %1, #0;" \
444	"bne 1b;" \
445	: "+&r" (previous), \
446	"+&r" (tmp) \
447	: "r" (target) \
448	: "memory", "cc"); \
449	return; \
450	}
451
452	CK_PR_UNARY(inc, ptr, void, void *, "add %0, %0, #1", "")
453	CK_PR_UNARY(dec, ptr, void, void *, "sub %0, %0, #1", "")
454	CK_PR_UNARY(not, ptr, void, void *, "mvn %0, %0", "")
455	CK_PR_UNARY(neg, ptr, void, void *, "neg %0, %0", "")
456
457	#define CK_PR_UNARY_S(S, T, W) \
458	CK_PR_UNARY(inc, S, T, T, "add %0, %0, #1", W) \
459	CK_PR_UNARY(dec, S, T, T, "sub %0, %0, #1", W) \
460	CK_PR_UNARY(not, S, T, T, "mvn %0, %0", W) \
461	CK_PR_UNARY(neg, S, T, T, "neg %0, %0", W) \
462
463	CK_PR_UNARY_S(32, uint32_t, "")
464	CK_PR_UNARY_S(uint, unsigned int, "")
465	CK_PR_UNARY_S(int, int, "")
466	CK_PR_UNARY_S(16, uint16_t, "h")
467	CK_PR_UNARY_S(8, uint8_t, "b")
468	CK_PR_UNARY_S(short, short, "h")
469	CK_PR_UNARY_S(char, char, "b")
470
471	#undef CK_PR_UNARY_S
472	#undef CK_PR_UNARY
473
474	#define CK_PR_BINARY(O, N, M, T, I, W) \
475	CK_CC_INLINE static void \
476	ck_pr_##O##_##N(M *target, T delta) \
477	{ \
478	T previous = 0; \
479	T tmp = 0; \
480	__asm__ __volatile__("1:" \
481	"ldrex" W " %0, [%2];" \
482	I " %0, %0, %3;" \
483	"strex" W " %1, %0, [%2];" \
484	"cmp %1, #0;" \
485	"bne 1b;" \
486	: "+&r" (previous), \
487	"+&r" (tmp) \
488	: "r" (target), \
489	"r" (delta) \
490	: "memory", "cc"); \
491	return; \
492	}
493
494	CK_PR_BINARY(and, ptr, void, uintptr_t, "and", "")
495	CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "")
496	CK_PR_BINARY(or, ptr, void, uintptr_t, "orr", "")
497	CK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "")
498	CK_PR_BINARY(xor, ptr, void, uintptr_t, "eor", "")
499
500	#define CK_PR_BINARY_S(S, T, W) \
501	CK_PR_BINARY(and, S, T, T, "and", W) \
502	CK_PR_BINARY(add, S, T, T, "add", W) \
503	CK_PR_BINARY(or, S, T, T, "orr", W) \
504	CK_PR_BINARY(sub, S, T, T, "sub", W) \
505	CK_PR_BINARY(xor, S, T, T, "eor", W)
506
507	CK_PR_BINARY_S(32, uint32_t, "")
508	CK_PR_BINARY_S(uint, unsigned int, "")
509	CK_PR_BINARY_S(int, int, "")
510	CK_PR_BINARY_S(16, uint16_t, "h")
511	CK_PR_BINARY_S(8, uint8_t, "b")
512	CK_PR_BINARY_S(short, short, "h")
513	CK_PR_BINARY_S(char, char, "b")
514
515	#undef CK_PR_BINARY_S
516	#undef CK_PR_BINARY
517
518	CK_CC_INLINE static void *
519	ck_pr_faa_ptr(void *target, uintptr_t delta)
520	{
521	uintptr_t previous, r, tmp;
522
523	__asm__ __volatile__("1:"
524	"ldrex %0, [%3];"
525	"add %1, %4, %0;"
526	"strex %2, %1, [%3];"
527	"cmp %2, #0;"
528	"bne 1b;"
529	: "=&r" (previous),
530	"=&r" (r),
531	"=&r" (tmp)
532	: "r" (target),
533	"r" (delta)
534	: "memory", "cc");
535
536	return (void *)(previous);
537	}
538
539	#define CK_PR_FAA(S, T, W) \
540	CK_CC_INLINE static T \
541	ck_pr_faa_##S(T *target, T delta) \
542	{ \
543	T previous = 0, r = 0, tmp = 0; \
544	__asm__ __volatile__("1:" \
545	"ldrex" W " %0, [%3];" \
546	"add %1, %4, %0;" \
547	"strex" W " %2, %1, [%3];" \
548	"cmp %2, #0;" \
549	"bne 1b;" \
550	: "+&r" (previous), \
551	"+&r" (r), \
552	"+&r" (tmp) \
553	: "r" (target), \
554	"r" (delta) \
555	: "memory", "cc"); \
556	return (previous); \
557	}
558
559	CK_PR_FAA(32, uint32_t, "")
560	CK_PR_FAA(uint, unsigned int, "")
561	CK_PR_FAA(int, int, "")
562	CK_PR_FAA(16, uint16_t, "h")
563	CK_PR_FAA(8, uint8_t, "b")
564	CK_PR_FAA(short, short, "h")
565	CK_PR_FAA(char, char, "b")
566
567	#undef CK_PR_FAA
568
569	#endif /* CK_PR_ARM_H */
570

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: rtems-libbsd/freebsd/sys/contrib/ck/include/gcc/arm/ck_pr.h

Download in other formats: