|
1 // Copyright (c) 2008-2009 Nokia Corporation and/or its subsidiary(-ies). |
|
2 // All rights reserved. |
|
3 // This component and the accompanying materials are made available |
|
4 // under the terms of the License "Eclipse Public License v1.0" |
|
5 // which accompanies this distribution, and is available |
|
6 // at the URL "http://www.eclipse.org/legal/epl-v10.html". |
|
7 // |
|
8 // Initial Contributors: |
|
9 // Nokia Corporation - initial contribution. |
|
10 // |
|
11 // Contributors: |
|
12 // |
|
13 // Description: |
|
14 // e32\common\arm\atomics.cia |
|
15 // |
|
16 // |
|
17 |
|
18 |
|
19 #include <cpudefs.h> |
|
20 #include <e32def.h> |
|
21 //#include <e32atomics.h> |
|
22 |
|
23 #if defined(__KERNEL_MODE__) |
|
24 #include "nk_cpu.h" |
|
25 #elif defined(__ATOMIC_USE_FAST_EXEC__) || defined(__ATOMIC64_USE_FAST_EXEC__) || defined(__ATOMIC64_USE_SLOW_EXEC__) |
|
26 #include <u32exec.h> |
|
27 #endif |
|
28 |
|
29 #define __concat__(a,b) a##b |
|
30 #define __concat3__(a,b,c) a##b##c |
|
31 #define __concat5__(a,b,c,d,e) a##b##c##d##e |
|
32 #define __fname__(type,order,size) __concat5__(__e32_atomic_,type,_,order,size) |
|
33 // __e32_atomic_##type##_##order##size |
|
34 |
|
35 #undef __BARRIERS_NEEDED__ |
|
36 #undef __AVOID_READ_SIDE_EFFECTS__ |
|
37 #ifdef __SMP__ |
|
38 #define __BARRIERS_NEEDED__ |
|
39 #else |
|
40 #ifdef __KERNEL_MODE__ |
|
41 // On non-SMP use interrupt disabling even on V6 and V6K just in case someone |
|
42 // has used the atomic operations on I/O addresses. |
|
43 #define __AVOID_READ_SIDE_EFFECTS__ |
|
44 #endif |
|
45 #endif |
|
46 |
|
47 #ifdef __BARRIERS_NEEDED__ |
|
48 #define __LOCAL_DATA_MEMORY_BARRIER__(reg) __DATA_MEMORY_BARRIER__(reg) |
|
49 #define __LOCAL_DATA_MEMORY_BARRIER_Z__(reg) __DATA_MEMORY_BARRIER_Z__(reg) |
|
50 #define __LOCAL_DATA_SYNC_BARRIER__(reg) __DATA_SYNC_BARRIER__(reg) |
|
51 #define __LOCAL_DATA_SYNC_BARRIER_Z__(reg) __DATA_SYNC_BARRIER_Z__(reg) |
|
52 #define __LOCAL_INST_SYNC_BARRIER__(reg) __INST_SYNC_BARRIER__(reg) |
|
53 #define __LOCAL_INST_SYNC_BARRIER_Z__(reg) __INST_SYNC_BARRIER_Z__(reg) |
|
54 #else // __BARRIERS_NEEDED__ |
|
55 #define __LOCAL_DATA_MEMORY_BARRIER__(reg) |
|
56 #define __LOCAL_DATA_MEMORY_BARRIER_Z__(reg) |
|
57 #define __LOCAL_DATA_SYNC_BARRIER__(reg) |
|
58 #define __LOCAL_DATA_SYNC_BARRIER_Z__(reg) |
|
59 #define __LOCAL_INST_SYNC_BARRIER__(reg) |
|
60 #define __LOCAL_INST_SYNC_BARRIER_Z__(reg) |
|
61 #endif // __BARRIERS_NEEDED__ |
|
62 |
|
63 #ifdef __CPU_ARM_HAS_CPS |
|
64 #define __DISABLE_INTERRUPTS__(keep,temp) asm("mrs "#keep ", cpsr"); CPSIDAIF |
|
65 #define __RESTORE_INTERRUPTS__(keep) asm("msr cpsr_c, "#keep ) // flags preserved |
|
66 #else |
|
67 #define __DISABLE_INTERRUPTS__(keep,temp) asm("mrs "#keep ", cpsr"); asm("orr "#temp ", "#keep ", #0xc0" ); asm("msr cpsr, "#temp ) |
|
68 #define __RESTORE_INTERRUPTS__(keep) asm("msr cpsr_c, "#keep ) // flags preserved |
|
69 #endif |
|
70 |
|
71 /****************************************************************************** |
|
72 * Barriers |
|
73 ******************************************************************************/ |
|
74 |
|
75 extern "C" EXPORT_C __NAKED__ void __e32_memory_barrier() |
|
76 { |
|
77 __LOCAL_DATA_MEMORY_BARRIER_Z__(r0); |
|
78 __JUMP(,lr); |
|
79 } |
|
80 |
|
81 /** Barrier guaranteeing completion as well as ordering |
|
82 |
|
83 */ |
|
84 #if defined(__KERNEL_MODE__) || defined(__CPU_ARM_SUPPORTS_USER_MODE_BARRIERS) |
|
85 extern "C" EXPORT_C __NAKED__ void __e32_io_completion_barrier() |
|
86 { |
|
87 __DATA_SYNC_BARRIER_Z__(r0); |
|
88 __JUMP(,lr); |
|
89 } |
|
90 #else |
|
91 extern "C" EXPORT_C __NAKED__ void __e32_io_completion_barrier() |
|
92 { |
|
93 asm("mov r0, sp "); |
|
94 asm("mov r1, #0 "); |
|
95 SLOW_EXEC2(EExecIMBRange); |
|
96 } |
|
97 #endif |
|
98 |
|
99 |
|
100 /****************************************************************************** |
|
101 * Miscellaneous utility functions |
|
102 ******************************************************************************/ |
|
103 |
|
104 /** Find the most significant 1 in a 32 bit word |
|
105 |
|
106 @param v The word to be scanned |
|
107 @return The bit number of the most significant 1 if v != 0 |
|
108 -1 if v == 0 |
|
109 */ |
|
110 extern "C" EXPORT_C __NAKED__ TInt __e32_find_ms1_32(TUint32 /*v*/) |
|
111 { |
|
112 #ifdef __CPU_ARM_HAS_CLZ |
|
113 CLZ( 1,0); // r1=31-MSB(r0), 32 if r0=0 |
|
114 asm("rsb r0, r1, #31 "); // r0=MSB(r0), -1 if r0=0 |
|
115 #else |
|
116 asm("movs r1, r0 "); |
|
117 asm("beq 0f "); |
|
118 asm("mov r0, #31 "); |
|
119 asm("cmp r1, #0x00010000 "); |
|
120 asm("movcc r1, r1, lsl #16 "); |
|
121 asm("subcc r0, r0, #16 "); |
|
122 asm("cmp r1, #0x01000000 "); |
|
123 asm("movcc r1, r1, lsl #8 "); |
|
124 asm("subcc r0, r0, #8 "); |
|
125 asm("cmp r1, #0x10000000 "); |
|
126 asm("movcc r1, r1, lsl #4 "); |
|
127 asm("subcc r0, r0, #4 "); |
|
128 asm("cmp r1, #0x40000000 "); |
|
129 asm("movcc r1, r1, lsl #2 "); |
|
130 asm("subcc r0, r0, #2 "); |
|
131 asm("cmp r1, #0x80000000 "); |
|
132 asm("subcc r0, r0, #1 "); |
|
133 __JUMP(, lr); |
|
134 asm("0: "); |
|
135 asm("mvn r0, #0 "); // if input zero, return -1 |
|
136 #endif |
|
137 __JUMP(, lr); |
|
138 } |
|
139 |
|
140 |
|
141 /** Find the least significant 1 in a 32 bit word |
|
142 |
|
143 @param v The word to be scanned |
|
144 @return The bit number of the least significant 1 if v != 0 |
|
145 -1 if v == 0 |
|
146 */ |
|
147 extern "C" EXPORT_C __NAKED__ TInt __e32_find_ls1_32(TUint32 /*v*/) |
|
148 { |
|
149 #ifdef __CPU_ARM_HAS_CLZ |
|
150 asm("subs r1, r0, #1 "); // r1 = arg - 1 |
|
151 asm("eorcs r0, r0, r1 "); // if arg=0, leave alone else mask upper bits |
|
152 CLZ( 1,0); // r1=31-MSB(r0), 32 if r0=0 |
|
153 asm("rsb r0, r1, #31 "); // r0=MSB(r0), -1 if r0=0 |
|
154 #else |
|
155 asm("movs r1, r0 "); |
|
156 asm("beq 0f "); |
|
157 asm("mov r0, #0 "); |
|
158 asm("movs r2, r1, lsl #16 "); |
|
159 asm("movne r1, r2 "); |
|
160 asm("addeq r0, r0, #16 "); |
|
161 asm("movs r2, r1, lsl #8 "); |
|
162 asm("movne r1, r2 "); |
|
163 asm("addeq r0, r0, #8 "); |
|
164 asm("movs r2, r1, lsl #4 "); |
|
165 asm("movne r1, r2 "); |
|
166 asm("addeq r0, r0, #4 "); |
|
167 asm("movs r2, r1, lsl #2 "); |
|
168 asm("movne r1, r2 "); |
|
169 asm("addeq r0, r0, #2 "); |
|
170 asm("movs r2, r1, lsl #1 "); |
|
171 asm("addeq r0, r0, #1 "); |
|
172 __JUMP(, lr); |
|
173 asm("0: "); |
|
174 asm("mvn r0, #0 "); // if input zero, return -1 |
|
175 #endif |
|
176 __JUMP(, lr); |
|
177 } |
|
178 |
|
179 |
|
180 /** Count the number of 1's in a 32 bit word |
|
181 |
|
182 @param v The word to be scanned |
|
183 @return The number of 1's |
|
184 */ |
|
185 extern "C" EXPORT_C __NAKED__ TInt __e32_bit_count_32(TUint32 /*v*/) |
|
186 { |
|
187 asm("mov r2, #0x0f "); // r2=0x0000000f |
|
188 asm("orr r2, r2, r2, lsl #8 "); // r2=0x00000f0f |
|
189 asm("orr r2, r2, r2, lsl #16 "); // r2=0x0f0f0f0f |
|
190 asm("eor r3, r2, r2, lsl #2 "); // r3=0x33333333 |
|
191 asm("eor ip, r3, r3, lsl #1 "); // ip=0x55555555 |
|
192 asm("bic r1, r0, ip "); // r1=odd bits of input |
|
193 asm("and r0, r0, ip "); // r0=even bits of input |
|
194 asm("add r0, r0, r1, lsr #1 "); // r0[2n:2n+1] = in[2n]+in[2n+1], 0<=n<=15 |
|
195 asm("bic r1, r0, r3 "); // r1 = r0[4n+2:4n+3] for 0<=n<=7, other bits 0 |
|
196 asm("and r0, r0, r3 "); // r0 = r0[4n:4n+1] for 0<=n<=7, other bits 0 |
|
197 asm("add r0, r0, r1, lsr #2 "); // r0 bits 4n:4n+3 = in[4n]+in[4n+1]+in[4n+2]+in[4n+3], 0<=n<=7 |
|
198 asm("add r0, r0, r0, lsr #4 "); // r0[8n:8n+3]=in[8n]+in[8n+1]+...+in[8n+7], 0<=n<=3 |
|
199 asm("and r0, r0, r2 "); // make sure other bits of r0 are zero |
|
200 asm("add r0, r0, r0, lsr #8 "); // r0[16n:16n+7]=in[16n]+in[16n+1]+...+in[16n+15], n=0,1 |
|
201 asm("add r0, r0, r0, lsr #16 "); // r0[0:7]=SUM{ in[n] : 0<=n<=31 } |
|
202 asm("and r0, r0, #0xff "); // mask other unwanted bits |
|
203 __JUMP(, lr); |
|
204 } |
|
205 |
|
206 |
|
207 /** Find the most significant 1 in a 64 bit word |
|
208 |
|
209 @param v The word to be scanned |
|
210 @return The bit number of the most significant 1 if v != 0 |
|
211 -1 if v == 0 |
|
212 */ |
|
213 extern "C" EXPORT_C __NAKED__ TInt __e32_find_ms1_64(TUint64 /*v*/) |
|
214 { |
|
215 /* On entry argument in R1:R0 */ |
|
216 #ifdef __CPU_ARM_HAS_CLZ |
|
217 CLZ( 2,1); // r2=31-MSB(r1), 32 if r1=0 |
|
218 asm("subs r2, r2, #32 "); // r2=-1-MSB(r1), 0 if r1=0 |
|
219 CLZcc(CC_EQ,2,0); // if r1=0, r2=31-MSB(r0), 32 if r0=0 |
|
220 asm("rsb r0, r2, #31 "); // if r1!=0, r0=32+MSB(r1) else if r0!=0 r0=MSB(r0) else r0=-1 |
|
221 #else |
|
222 asm("cmp r1, #1 "); // r1>=1 ? |
|
223 asm("movcs r0, #63 "); // if so r0=63 |
|
224 asm("movccs r1, r0 "); // else r1=r0, test for zero (C unaffected) |
|
225 asm("beq 0f "); |
|
226 asm("movcc r0, #31 "); // if r1=0 and r0!=0, r1=original r0 and r0=31 |
|
227 asm("cmp r1, #0x00010000 "); |
|
228 asm("movcc r1, r1, lsl #16 "); |
|
229 asm("subcc r0, r0, #16 "); |
|
230 asm("cmp r1, #0x01000000 "); |
|
231 asm("movcc r1, r1, lsl #8 "); |
|
232 asm("subcc r0, r0, #8 "); |
|
233 asm("cmp r1, #0x10000000 "); |
|
234 asm("movcc r1, r1, lsl #4 "); |
|
235 asm("subcc r0, r0, #4 "); |
|
236 asm("cmp r1, #0x40000000 "); |
|
237 asm("movcc r1, r1, lsl #2 "); |
|
238 asm("subcc r0, r0, #2 "); |
|
239 asm("cmp r1, #0x80000000 "); |
|
240 asm("subcc r0, r0, #1 "); |
|
241 __JUMP(, lr); |
|
242 asm("0: "); |
|
243 asm("mvn r0, #0 "); // if input zero, return -1 |
|
244 #endif |
|
245 __JUMP(, lr); |
|
246 } |
|
247 |
|
248 |
|
249 /** Find the least significant 1 in a 64 bit word |
|
250 |
|
251 @param v The word to be scanned |
|
252 @return The bit number of the least significant 1 if v != 0 |
|
253 -1 if v == 0 |
|
254 */ |
|
255 extern "C" EXPORT_C __NAKED__ TInt __e32_find_ls1_64(TUint64 /*v*/) |
|
256 { |
|
257 /* On entry argument in R1:R0 */ |
|
258 #ifdef __CPU_ARM_HAS_CLZ |
|
259 asm("subs r2, r0, #1 "); |
|
260 asm("sbcs r3, r1, #0 "); // r3:r2 = arg - 1 |
|
261 asm("eorcs r0, r0, r2 "); // if arg=0 leave alone else mask upper bits |
|
262 asm("eorcs r1, r1, r3 "); |
|
263 CLZ( 2,1); // r2=31-MSB(r1), 32 if r1=0 |
|
264 asm("subs r2, r2, #32 "); // r2=-1-MSB(r1), 0 if r1=0 |
|
265 CLZcc(CC_EQ,2,0); // if r1=0, r2=31-MSB(r0), 32 if r0=0 |
|
266 asm("rsb r0, r2, #31 "); // if r1!=0, r0=32+MSB(r1) else if r0!=0 r0=MSB(r0) else r0=-1 |
|
267 #else |
|
268 asm("cmp r0, #1 "); // LSW(arg) >= 1? |
|
269 asm("movcs r1, r0 "); // if so r1=r0 |
|
270 asm("movcs r0, #32 "); // and r0=32 |
|
271 asm("movcc r0, #0 "); // else r0=0 |
|
272 asm("cmpcc r1, #1 "); // and test if MSW(arg) >= 1 |
|
273 asm("bcc 0f "); // if not, return -1 |
|
274 asm("movs r2, r1, lsl #16 "); |
|
275 asm("movne r1, r2 "); |
|
276 asm("addeq r0, r0, #16 "); |
|
277 asm("movs r2, r1, lsl #8 "); |
|
278 asm("movne r1, r2 "); |
|
279 asm("addeq r0, r0, #8 "); |
|
280 asm("movs r2, r1, lsl #4 "); |
|
281 asm("movne r1, r2 "); |
|
282 asm("addeq r0, r0, #4 "); |
|
283 asm("movs r2, r1, lsl #2 "); |
|
284 asm("movne r1, r2 "); |
|
285 asm("addeq r0, r0, #2 "); |
|
286 asm("movs r2, r1, lsl #1 "); |
|
287 asm("addeq r0, r0, #1 "); |
|
288 __JUMP(, lr); |
|
289 asm("0: "); |
|
290 asm("mvn r0, #0 "); // if input zero, return -1 |
|
291 #endif |
|
292 __JUMP(, lr); |
|
293 } |
|
294 |
|
295 |
|
296 /** Count the number of 1's in a 64 bit word |
|
297 |
|
298 @param v The word to be scanned |
|
299 @return The number of 1's |
|
300 */ |
|
301 extern "C" EXPORT_C __NAKED__ TInt __e32_bit_count_64(TUint64 /*v*/) |
|
302 { |
|
303 /* On entry argument in R1:R0 */ |
|
304 asm("str r4, [sp, #-4]! "); |
|
305 asm("mov r2, #0x0f "); // r2=0x0000000f |
|
306 asm("orr r2, r2, r2, lsl #8 "); // r2=0x00000f0f |
|
307 asm("orr r2, r2, r2, lsl #16 "); // r2=0x0f0f0f0f |
|
308 asm("eor r3, r2, r2, lsl #2 "); // r3=0x33333333 |
|
309 asm("eor ip, r3, r3, lsl #1 "); // ip=0x55555555 |
|
310 |
|
311 asm("bic r4, r0, ip "); // r4=odd bits of input LSW |
|
312 asm("and r0, r0, ip "); // r0=even bits of input LSW |
|
313 asm("add r0, r0, r4, lsr #1 "); // r0[2n:2n+1] = in[2n]+in[2n+1], 0<=n<=15 |
|
314 asm("bic r4, r0, r3 "); // r4 = r0[4n+2:4n+3] for 0<=n<=7, other bits 0 |
|
315 asm("and r0, r0, r3 "); // r0 = r0[4n:4n+1] for 0<=n<=7, other bits 0 |
|
316 asm("add r0, r0, r4, lsr #2 "); // r0 bits 4n:4n+3 = in[4n]+in[4n+1]+in[4n+2]+in[4n+3], 0<=n<=7 |
|
317 |
|
318 asm("bic r4, r1, ip "); // r4=odd bits of input MSW |
|
319 asm("and r1, r1, ip "); // r1=even bits of input MSW |
|
320 asm("add r1, r1, r4, lsr #1 "); // r1[2n:2n+1] = in[2n+32]+in[2n+33], 0<=n<=15 |
|
321 asm("bic r4, r1, r3 "); // r4 = r1[4n+34:4n+35] for 0<=n<=7, other bits 0 |
|
322 asm("and r1, r1, r3 "); // r1 = r1[4n+32:4n+33] for 0<=n<=7, other bits 0 |
|
323 asm("add r1, r1, r4, lsr #2 "); // r1 bits 4n:4n+3 = in[4n+32]+in[4n+33]+in[4n+34]+in[4n+35], 0<=n<=7 |
|
324 asm("ldr r4, [sp], #4 "); |
|
325 |
|
326 asm("add r0, r0, r1 "); // r0 bits 4n:4n+3 = in[4n]+in[4n+1]+in[4n+2]+in[4n+3]+in[4n+32]+in[4n+33]+in[4n+34]+in[4n+35], 0<=n<=7 |
|
327 asm("bic r1, r0, r2 "); // odd nibbles only |
|
328 asm("and r0, r0, r2 "); // even nibbles only |
|
329 asm("add r0, r0, r1, lsr #4 "); // r0[8n:8n+7]=bit count of byte n of MSW + bit count of byte n of LSW |
|
330 asm("add r0, r0, r0, lsr #8 "); // r0[16n:16n+7]=bit count of hword n of MSW + bit count of hword n of LSW |
|
331 asm("add r0, r0, r0, lsr #16 "); // r0[0:7]=total bit count |
|
332 asm("and r0, r0, #0xff "); // mask other unwanted bits |
|
333 __JUMP(, lr); |
|
334 } |
|
335 |
|
336 |
|
337 |
|
338 /****************************************************************************** |
|
339 * 64 bit operations |
|
340 ******************************************************************************/ |
|
341 #define __DATA_SIZE__ 64 |
|
342 #if defined(__CPU_ARM_HAS_LDREX_STREX_V6K) && !defined(__AVOID_READ_SIDE_EFFECTS__) |
|
343 |
|
344 // Include LDREXD/STREXD-based 64 bit operations |
|
345 #define __OP_LOAD__ |
|
346 #include "atomic_64_v6k.h" |
|
347 #define __OP_STORE__ |
|
348 #include "atomic_64_v6k.h" |
|
349 #define __OP_SWP__ |
|
350 #include "atomic_64_v6k.h" |
|
351 #define __OP_CAS__ |
|
352 #include "atomic_64_v6k.h" |
|
353 #define __OP_ADD__ |
|
354 #include "atomic_64_v6k.h" |
|
355 #define __OP_AND__ |
|
356 #include "atomic_64_v6k.h" |
|
357 #define __OP_IOR__ |
|
358 #include "atomic_64_v6k.h" |
|
359 #define __OP_XOR__ |
|
360 #include "atomic_64_v6k.h" |
|
361 #define __OP_AXO__ |
|
362 #include "atomic_64_v6k.h" |
|
363 #define __OP_TAU__ |
|
364 #include "atomic_64_v6k.h" |
|
365 #define __OP_TAS__ |
|
366 #include "atomic_64_v6k.h" |
|
367 |
|
368 #else |
|
369 #ifdef __KERNEL_MODE__ |
|
370 |
|
371 // Include interrupt-disabling 64 bit operations |
|
372 #define __OP_LOAD__ |
|
373 #include "atomic_64_v6_v5.h" |
|
374 #define __OP_STORE__ |
|
375 #include "atomic_64_v6_v5.h" |
|
376 #define __OP_SWP__ |
|
377 #include "atomic_64_v6_v5.h" |
|
378 #define __OP_CAS__ |
|
379 #include "atomic_64_v6_v5.h" |
|
380 #define __OP_ADD__ |
|
381 #include "atomic_64_v6_v5.h" |
|
382 #define __OP_AND__ |
|
383 #include "atomic_64_v6_v5.h" |
|
384 #define __OP_IOR__ |
|
385 #include "atomic_64_v6_v5.h" |
|
386 #define __OP_XOR__ |
|
387 #include "atomic_64_v6_v5.h" |
|
388 #define __OP_AXO__ |
|
389 #include "atomic_64_v6_v5.h" |
|
390 #define __OP_TAU__ |
|
391 #include "atomic_64_v6_v5.h" |
|
392 #define __OP_TAS__ |
|
393 #include "atomic_64_v6_v5.h" |
|
394 |
|
395 #else |
|
396 |
|
397 // Include 64 bit operations using Exec calls |
|
398 #define __OP_LOAD__ |
|
399 #include "atomic_64_v6_v5.h" |
|
400 #define __OP_STORE__ |
|
401 #include "atomic_64_v6_v5.h" |
|
402 #define __OP_SWP__ |
|
403 #include "atomic_64_exec.h" |
|
404 #define __OP_CAS__ |
|
405 #include "atomic_64_exec.h" |
|
406 #define __OP_ADD__ |
|
407 #include "atomic_64_exec.h" |
|
408 #define __OP_AND__ |
|
409 #include "atomic_64_exec.h" |
|
410 #define __OP_IOR__ |
|
411 #include "atomic_64_exec.h" |
|
412 #define __OP_XOR__ |
|
413 #include "atomic_64_exec.h" |
|
414 #define __OP_AXO__ |
|
415 #include "atomic_64_exec.h" |
|
416 #define __OP_TAU__ |
|
417 #include "atomic_64_exec.h" |
|
418 #define __OP_TAS__ |
|
419 #include "atomic_64_exec.h" |
|
420 |
|
421 #endif |
|
422 #endif |
|
423 #undef __DATA_SIZE__ |
|
424 |
|
425 /****************************************************************************** |
|
426 * 8,16,32 bit load/store operations |
|
427 ******************************************************************************/ |
|
428 |
|
429 #define __DATA_SIZE__ 8 |
|
430 #define __OP_LOAD__ |
|
431 #include "atomic_32_v6.h" |
|
432 #define __OP_STORE__ |
|
433 #include "atomic_32_v6.h" |
|
434 #undef __DATA_SIZE__ |
|
435 |
|
436 #define __DATA_SIZE__ 16 |
|
437 #define __OP_LOAD__ |
|
438 #include "atomic_32_v6.h" |
|
439 #define __OP_STORE__ |
|
440 #include "atomic_32_v6.h" |
|
441 #undef __DATA_SIZE__ |
|
442 |
|
443 #define __DATA_SIZE__ 32 |
|
444 #define __OP_LOAD__ |
|
445 #include "atomic_32_v6.h" |
|
446 #define __OP_STORE__ |
|
447 #include "atomic_32_v6.h" |
|
448 #undef __DATA_SIZE__ |
|
449 |
|
450 /****************************************************************************** |
|
451 * 8,16,32 bit RMW operations |
|
452 ******************************************************************************/ |
|
453 |
|
454 #if defined(__CPU_ARM_HAS_LDREX_STREX_V6K) && !defined(__AVOID_READ_SIDE_EFFECTS__) |
|
455 // V6K - Use variants of LDREX/STREX for everything |
|
456 #define __ATOMIC_8_IMPL__ "atomic_32_v6.h" |
|
457 #define __ATOMIC_16_IMPL__ "atomic_32_v6.h" |
|
458 #define __ATOMIC_32_IMPL__ "atomic_32_v6.h" |
|
459 #elif defined(__CPU_ARM_HAS_LDREX_STREX) && !defined(__AVOID_READ_SIDE_EFFECTS__) |
|
460 // V6 - Use LDREX/STREX for 32 bit operations |
|
461 // Use LDREX/STREX with shifts/rotates for 8/16 bit operations |
|
462 #define __ATOMIC_8_IMPL__ "atomic_8_16_v6.h" |
|
463 #define __ATOMIC_16_IMPL__ "atomic_8_16_v6.h" |
|
464 #define __ATOMIC_32_IMPL__ "atomic_32_v6.h" |
|
465 #else |
|
466 // V5 - Use interrupt disabling kernel side, Exec calls user side |
|
467 #ifdef __KERNEL_MODE__ |
|
468 #define __ATOMIC_8_IMPL__ "atomic_8_16_32_irq.h" |
|
469 #define __ATOMIC_16_IMPL__ "atomic_8_16_32_irq.h" |
|
470 #define __ATOMIC_32_IMPL__ "atomic_8_16_32_irq.h" |
|
471 #else |
|
472 #define __ATOMIC_8_IMPL__ "atomic_8_16_32_exec.h" |
|
473 #define __ATOMIC_16_IMPL__ "atomic_8_16_32_exec.h" |
|
474 #define __ATOMIC_32_IMPL__ "atomic_8_16_32_exec.h" |
|
475 #endif |
|
476 #endif |
|
477 |
|
478 #define __DATA_SIZE__ 8 |
|
479 #define __OP_SWP__ |
|
480 #include __ATOMIC_8_IMPL__ |
|
481 #define __OP_CAS__ |
|
482 #include __ATOMIC_8_IMPL__ |
|
483 #define __OP_ADD__ |
|
484 #include __ATOMIC_8_IMPL__ |
|
485 #define __OP_AND__ |
|
486 #include __ATOMIC_8_IMPL__ |
|
487 #define __OP_IOR__ |
|
488 #include __ATOMIC_8_IMPL__ |
|
489 #define __OP_XOR__ |
|
490 #include __ATOMIC_8_IMPL__ |
|
491 #define __OP_AXO__ |
|
492 #include __ATOMIC_8_IMPL__ |
|
493 #define __OP_TAU__ |
|
494 #include __ATOMIC_8_IMPL__ |
|
495 #define __OP_TAS__ |
|
496 #include __ATOMIC_8_IMPL__ |
|
497 #undef __DATA_SIZE__ |
|
498 |
|
499 #define __DATA_SIZE__ 16 |
|
500 #define __OP_SWP__ |
|
501 #include __ATOMIC_16_IMPL__ |
|
502 #define __OP_CAS__ |
|
503 #include __ATOMIC_16_IMPL__ |
|
504 #define __OP_ADD__ |
|
505 #include __ATOMIC_16_IMPL__ |
|
506 #define __OP_AND__ |
|
507 #include __ATOMIC_16_IMPL__ |
|
508 #define __OP_IOR__ |
|
509 #include __ATOMIC_16_IMPL__ |
|
510 #define __OP_XOR__ |
|
511 #include __ATOMIC_16_IMPL__ |
|
512 #define __OP_AXO__ |
|
513 #include __ATOMIC_16_IMPL__ |
|
514 #define __OP_TAU__ |
|
515 #include __ATOMIC_16_IMPL__ |
|
516 #define __OP_TAS__ |
|
517 #include __ATOMIC_16_IMPL__ |
|
518 #undef __DATA_SIZE__ |
|
519 |
|
520 #define __DATA_SIZE__ 32 |
|
521 #define __OP_SWP__ |
|
522 #include __ATOMIC_32_IMPL__ |
|
523 #define __OP_CAS__ |
|
524 #include __ATOMIC_32_IMPL__ |
|
525 #define __OP_ADD__ |
|
526 #include __ATOMIC_32_IMPL__ |
|
527 #define __OP_AND__ |
|
528 #include __ATOMIC_32_IMPL__ |
|
529 #define __OP_IOR__ |
|
530 #include __ATOMIC_32_IMPL__ |
|
531 #define __OP_XOR__ |
|
532 #include __ATOMIC_32_IMPL__ |
|
533 #define __OP_AXO__ |
|
534 #include __ATOMIC_32_IMPL__ |
|
535 #define __OP_TAU__ |
|
536 #include __ATOMIC_32_IMPL__ |
|
537 #define __OP_TAS__ |
|
538 #include __ATOMIC_32_IMPL__ |
|
539 #undef __DATA_SIZE__ |
|
540 |