|
0
|
1 |
// Copyright (c) 2008-2009 Nokia Corporation and/or its subsidiary(-ies).
|
|
|
2 |
// All rights reserved.
|
|
|
3 |
// This component and the accompanying materials are made available
|
|
|
4 |
// under the terms of the License "Eclipse Public License v1.0"
|
|
|
5 |
// which accompanies this distribution, and is available
|
|
|
6 |
// at the URL "http://www.eclipse.org/legal/epl-v10.html".
|
|
|
7 |
//
|
|
|
8 |
// Initial Contributors:
|
|
|
9 |
// Nokia Corporation - initial contribution.
|
|
|
10 |
//
|
|
|
11 |
// Contributors:
|
|
|
12 |
//
|
|
|
13 |
// Description:
|
|
|
14 |
// e32\common\arm\atomics.cia
|
|
|
15 |
//
|
|
|
16 |
//
|
|
|
17 |
|
|
|
18 |
|
|
|
19 |
#include <cpudefs.h>
|
|
|
20 |
#include <e32def.h>
|
|
|
21 |
//#include <e32atomics.h>
|
|
|
22 |
|
|
|
23 |
#if defined(__KERNEL_MODE__)
|
|
|
24 |
#include "nk_cpu.h"
|
|
|
25 |
#elif defined(__ATOMIC_USE_FAST_EXEC__) || defined(__ATOMIC64_USE_FAST_EXEC__) || defined(__ATOMIC64_USE_SLOW_EXEC__)
|
|
|
26 |
#include <u32exec.h>
|
|
|
27 |
#endif
|
|
|
28 |
|
|
|
29 |
#define __concat__(a,b) a##b
|
|
|
30 |
#define __concat3__(a,b,c) a##b##c
|
|
|
31 |
#define __concat5__(a,b,c,d,e) a##b##c##d##e
|
|
|
32 |
#define __fname__(type,order,size) __concat5__(__e32_atomic_,type,_,order,size)
|
|
|
33 |
// __e32_atomic_##type##_##order##size
|
|
|
34 |
|
|
|
35 |
#undef __BARRIERS_NEEDED__
|
|
|
36 |
#undef __AVOID_READ_SIDE_EFFECTS__
|
|
|
37 |
#ifdef __SMP__
|
|
|
38 |
#define __BARRIERS_NEEDED__
|
|
|
39 |
#else
|
|
|
40 |
#ifdef __KERNEL_MODE__
|
|
|
41 |
// On non-SMP use interrupt disabling even on V6 and V6K just in case someone
|
|
|
42 |
// has used the atomic operations on I/O addresses.
|
|
|
43 |
#define __AVOID_READ_SIDE_EFFECTS__
|
|
|
44 |
#endif
|
|
|
45 |
#endif
|
|
|
46 |
|
|
|
47 |
#ifdef __BARRIERS_NEEDED__
|
|
|
48 |
#define __LOCAL_DATA_MEMORY_BARRIER__(reg) __DATA_MEMORY_BARRIER__(reg)
|
|
|
49 |
#define __LOCAL_DATA_MEMORY_BARRIER_Z__(reg) __DATA_MEMORY_BARRIER_Z__(reg)
|
|
|
50 |
#define __LOCAL_DATA_SYNC_BARRIER__(reg) __DATA_SYNC_BARRIER__(reg)
|
|
|
51 |
#define __LOCAL_DATA_SYNC_BARRIER_Z__(reg) __DATA_SYNC_BARRIER_Z__(reg)
|
|
|
52 |
#define __LOCAL_INST_SYNC_BARRIER__(reg) __INST_SYNC_BARRIER__(reg)
|
|
|
53 |
#define __LOCAL_INST_SYNC_BARRIER_Z__(reg) __INST_SYNC_BARRIER_Z__(reg)
|
|
|
54 |
#else // __BARRIERS_NEEDED__
|
|
|
55 |
#define __LOCAL_DATA_MEMORY_BARRIER__(reg)
|
|
|
56 |
#define __LOCAL_DATA_MEMORY_BARRIER_Z__(reg)
|
|
|
57 |
#define __LOCAL_DATA_SYNC_BARRIER__(reg)
|
|
|
58 |
#define __LOCAL_DATA_SYNC_BARRIER_Z__(reg)
|
|
|
59 |
#define __LOCAL_INST_SYNC_BARRIER__(reg)
|
|
|
60 |
#define __LOCAL_INST_SYNC_BARRIER_Z__(reg)
|
|
|
61 |
#endif // __BARRIERS_NEEDED__
|
|
|
62 |
|
|
|
63 |
#ifdef __CPU_ARM_HAS_CPS
|
|
|
64 |
#define __DISABLE_INTERRUPTS__(keep,temp) asm("mrs "#keep ", cpsr"); CPSIDAIF
|
|
|
65 |
#define __RESTORE_INTERRUPTS__(keep) asm("msr cpsr_c, "#keep ) // flags preserved
|
|
|
66 |
#else
|
|
|
67 |
#define __DISABLE_INTERRUPTS__(keep,temp) asm("mrs "#keep ", cpsr"); asm("orr "#temp ", "#keep ", #0xc0" ); asm("msr cpsr, "#temp )
|
|
|
68 |
#define __RESTORE_INTERRUPTS__(keep) asm("msr cpsr_c, "#keep ) // flags preserved
|
|
|
69 |
#endif
|
|
|
70 |
|
|
|
71 |
/******************************************************************************
|
|
|
72 |
* Barriers
|
|
|
73 |
******************************************************************************/
|
|
|
74 |
|
|
|
75 |
extern "C" EXPORT_C __NAKED__ void __e32_memory_barrier()
|
|
|
76 |
{
|
|
|
77 |
__LOCAL_DATA_MEMORY_BARRIER_Z__(r0);
|
|
|
78 |
__JUMP(,lr);
|
|
|
79 |
}
|
|
|
80 |
|
|
|
81 |
/** Barrier guaranteeing completion as well as ordering
|
|
|
82 |
|
|
|
83 |
*/
|
|
|
84 |
#if defined(__KERNEL_MODE__) || defined(__CPU_ARM_SUPPORTS_USER_MODE_BARRIERS)
|
|
|
85 |
extern "C" EXPORT_C __NAKED__ void __e32_io_completion_barrier()
|
|
|
86 |
{
|
|
|
87 |
__DATA_SYNC_BARRIER_Z__(r0);
|
|
|
88 |
__JUMP(,lr);
|
|
|
89 |
}
|
|
|
90 |
#else
|
|
|
91 |
extern "C" EXPORT_C __NAKED__ void __e32_io_completion_barrier()
|
|
|
92 |
{
|
|
|
93 |
asm("mov r0, sp ");
|
|
|
94 |
asm("mov r1, #0 ");
|
|
|
95 |
SLOW_EXEC2(EExecIMBRange);
|
|
|
96 |
}
|
|
|
97 |
#endif
|
|
|
98 |
|
|
|
99 |
|
|
|
100 |
/******************************************************************************
|
|
|
101 |
* Miscellaneous utility functions
|
|
|
102 |
******************************************************************************/
|
|
|
103 |
|
|
|
104 |
/** Find the most significant 1 in a 32 bit word
|
|
|
105 |
|
|
|
106 |
@param v The word to be scanned
|
|
|
107 |
@return The bit number of the most significant 1 if v != 0
|
|
|
108 |
-1 if v == 0
|
|
|
109 |
*/
|
|
|
110 |
extern "C" EXPORT_C __NAKED__ TInt __e32_find_ms1_32(TUint32 /*v*/)
|
|
|
111 |
{
|
|
|
112 |
#ifdef __CPU_ARM_HAS_CLZ
|
|
|
113 |
CLZ( 1,0); // r1=31-MSB(r0), 32 if r0=0
|
|
|
114 |
asm("rsb r0, r1, #31 "); // r0=MSB(r0), -1 if r0=0
|
|
|
115 |
#else
|
|
|
116 |
asm("movs r1, r0 ");
|
|
|
117 |
asm("beq 0f ");
|
|
|
118 |
asm("mov r0, #31 ");
|
|
|
119 |
asm("cmp r1, #0x00010000 ");
|
|
|
120 |
asm("movcc r1, r1, lsl #16 ");
|
|
|
121 |
asm("subcc r0, r0, #16 ");
|
|
|
122 |
asm("cmp r1, #0x01000000 ");
|
|
|
123 |
asm("movcc r1, r1, lsl #8 ");
|
|
|
124 |
asm("subcc r0, r0, #8 ");
|
|
|
125 |
asm("cmp r1, #0x10000000 ");
|
|
|
126 |
asm("movcc r1, r1, lsl #4 ");
|
|
|
127 |
asm("subcc r0, r0, #4 ");
|
|
|
128 |
asm("cmp r1, #0x40000000 ");
|
|
|
129 |
asm("movcc r1, r1, lsl #2 ");
|
|
|
130 |
asm("subcc r0, r0, #2 ");
|
|
|
131 |
asm("cmp r1, #0x80000000 ");
|
|
|
132 |
asm("subcc r0, r0, #1 ");
|
|
|
133 |
__JUMP(, lr);
|
|
|
134 |
asm("0: ");
|
|
|
135 |
asm("mvn r0, #0 "); // if input zero, return -1
|
|
|
136 |
#endif
|
|
|
137 |
__JUMP(, lr);
|
|
|
138 |
}
|
|
|
139 |
|
|
|
140 |
|
|
|
141 |
/** Find the least significant 1 in a 32 bit word
|
|
|
142 |
|
|
|
143 |
@param v The word to be scanned
|
|
|
144 |
@return The bit number of the least significant 1 if v != 0
|
|
|
145 |
-1 if v == 0
|
|
|
146 |
*/
|
|
|
147 |
extern "C" EXPORT_C __NAKED__ TInt __e32_find_ls1_32(TUint32 /*v*/)
|
|
|
148 |
{
|
|
|
149 |
#ifdef __CPU_ARM_HAS_CLZ
|
|
|
150 |
asm("subs r1, r0, #1 "); // r1 = arg - 1
|
|
|
151 |
asm("eorcs r0, r0, r1 "); // if arg=0, leave alone else mask upper bits
|
|
|
152 |
CLZ( 1,0); // r1=31-MSB(r0), 32 if r0=0
|
|
|
153 |
asm("rsb r0, r1, #31 "); // r0=MSB(r0), -1 if r0=0
|
|
|
154 |
#else
|
|
|
155 |
asm("movs r1, r0 ");
|
|
|
156 |
asm("beq 0f ");
|
|
|
157 |
asm("mov r0, #0 ");
|
|
|
158 |
asm("movs r2, r1, lsl #16 ");
|
|
|
159 |
asm("movne r1, r2 ");
|
|
|
160 |
asm("addeq r0, r0, #16 ");
|
|
|
161 |
asm("movs r2, r1, lsl #8 ");
|
|
|
162 |
asm("movne r1, r2 ");
|
|
|
163 |
asm("addeq r0, r0, #8 ");
|
|
|
164 |
asm("movs r2, r1, lsl #4 ");
|
|
|
165 |
asm("movne r1, r2 ");
|
|
|
166 |
asm("addeq r0, r0, #4 ");
|
|
|
167 |
asm("movs r2, r1, lsl #2 ");
|
|
|
168 |
asm("movne r1, r2 ");
|
|
|
169 |
asm("addeq r0, r0, #2 ");
|
|
|
170 |
asm("movs r2, r1, lsl #1 ");
|
|
|
171 |
asm("addeq r0, r0, #1 ");
|
|
|
172 |
__JUMP(, lr);
|
|
|
173 |
asm("0: ");
|
|
|
174 |
asm("mvn r0, #0 "); // if input zero, return -1
|
|
|
175 |
#endif
|
|
|
176 |
__JUMP(, lr);
|
|
|
177 |
}
|
|
|
178 |
|
|
|
179 |
|
|
|
180 |
/** Count the number of 1's in a 32 bit word
|
|
|
181 |
|
|
|
182 |
@param v The word to be scanned
|
|
|
183 |
@return The number of 1's
|
|
|
184 |
*/
|
|
|
185 |
extern "C" EXPORT_C __NAKED__ TInt __e32_bit_count_32(TUint32 /*v*/)
|
|
|
186 |
{
|
|
|
187 |
asm("mov r2, #0x0f "); // r2=0x0000000f
|
|
|
188 |
asm("orr r2, r2, r2, lsl #8 "); // r2=0x00000f0f
|
|
|
189 |
asm("orr r2, r2, r2, lsl #16 "); // r2=0x0f0f0f0f
|
|
|
190 |
asm("eor r3, r2, r2, lsl #2 "); // r3=0x33333333
|
|
|
191 |
asm("eor ip, r3, r3, lsl #1 "); // ip=0x55555555
|
|
|
192 |
asm("bic r1, r0, ip "); // r1=odd bits of input
|
|
|
193 |
asm("and r0, r0, ip "); // r0=even bits of input
|
|
|
194 |
asm("add r0, r0, r1, lsr #1 "); // r0[2n:2n+1] = in[2n]+in[2n+1], 0<=n<=15
|
|
|
195 |
asm("bic r1, r0, r3 "); // r1 = r0[4n+2:4n+3] for 0<=n<=7, other bits 0
|
|
|
196 |
asm("and r0, r0, r3 "); // r0 = r0[4n:4n+1] for 0<=n<=7, other bits 0
|
|
|
197 |
asm("add r0, r0, r1, lsr #2 "); // r0 bits 4n:4n+3 = in[4n]+in[4n+1]+in[4n+2]+in[4n+3], 0<=n<=7
|
|
|
198 |
asm("add r0, r0, r0, lsr #4 "); // r0[8n:8n+3]=in[8n]+in[8n+1]+...+in[8n+7], 0<=n<=3
|
|
|
199 |
asm("and r0, r0, r2 "); // make sure other bits of r0 are zero
|
|
|
200 |
asm("add r0, r0, r0, lsr #8 "); // r0[16n:16n+7]=in[16n]+in[16n+1]+...+in[16n+15], n=0,1
|
|
|
201 |
asm("add r0, r0, r0, lsr #16 "); // r0[0:7]=SUM{ in[n] : 0<=n<=31 }
|
|
|
202 |
asm("and r0, r0, #0xff "); // mask other unwanted bits
|
|
|
203 |
__JUMP(, lr);
|
|
|
204 |
}
|
|
|
205 |
|
|
|
206 |
|
|
|
207 |
/** Find the most significant 1 in a 64 bit word
|
|
|
208 |
|
|
|
209 |
@param v The word to be scanned
|
|
|
210 |
@return The bit number of the most significant 1 if v != 0
|
|
|
211 |
-1 if v == 0
|
|
|
212 |
*/
|
|
|
213 |
extern "C" EXPORT_C __NAKED__ TInt __e32_find_ms1_64(TUint64 /*v*/)
|
|
|
214 |
{
|
|
|
215 |
/* On entry argument in R1:R0 */
|
|
|
216 |
#ifdef __CPU_ARM_HAS_CLZ
|
|
|
217 |
CLZ( 2,1); // r2=31-MSB(r1), 32 if r1=0
|
|
|
218 |
asm("subs r2, r2, #32 "); // r2=-1-MSB(r1), 0 if r1=0
|
|
|
219 |
CLZcc(CC_EQ,2,0); // if r1=0, r2=31-MSB(r0), 32 if r0=0
|
|
|
220 |
asm("rsb r0, r2, #31 "); // if r1!=0, r0=32+MSB(r1) else if r0!=0 r0=MSB(r0) else r0=-1
|
|
|
221 |
#else
|
|
|
222 |
asm("cmp r1, #1 "); // r1>=1 ?
|
|
|
223 |
asm("movcs r0, #63 "); // if so r0=63
|
|
|
224 |
asm("movccs r1, r0 "); // else r1=r0, test for zero (C unaffected)
|
|
|
225 |
asm("beq 0f ");
|
|
|
226 |
asm("movcc r0, #31 "); // if r1=0 and r0!=0, r1=original r0 and r0=31
|
|
|
227 |
asm("cmp r1, #0x00010000 ");
|
|
|
228 |
asm("movcc r1, r1, lsl #16 ");
|
|
|
229 |
asm("subcc r0, r0, #16 ");
|
|
|
230 |
asm("cmp r1, #0x01000000 ");
|
|
|
231 |
asm("movcc r1, r1, lsl #8 ");
|
|
|
232 |
asm("subcc r0, r0, #8 ");
|
|
|
233 |
asm("cmp r1, #0x10000000 ");
|
|
|
234 |
asm("movcc r1, r1, lsl #4 ");
|
|
|
235 |
asm("subcc r0, r0, #4 ");
|
|
|
236 |
asm("cmp r1, #0x40000000 ");
|
|
|
237 |
asm("movcc r1, r1, lsl #2 ");
|
|
|
238 |
asm("subcc r0, r0, #2 ");
|
|
|
239 |
asm("cmp r1, #0x80000000 ");
|
|
|
240 |
asm("subcc r0, r0, #1 ");
|
|
|
241 |
__JUMP(, lr);
|
|
|
242 |
asm("0: ");
|
|
|
243 |
asm("mvn r0, #0 "); // if input zero, return -1
|
|
|
244 |
#endif
|
|
|
245 |
__JUMP(, lr);
|
|
|
246 |
}
|
|
|
247 |
|
|
|
248 |
|
|
|
249 |
/** Find the least significant 1 in a 64 bit word
|
|
|
250 |
|
|
|
251 |
@param v The word to be scanned
|
|
|
252 |
@return The bit number of the least significant 1 if v != 0
|
|
|
253 |
-1 if v == 0
|
|
|
254 |
*/
|
|
|
255 |
extern "C" EXPORT_C __NAKED__ TInt __e32_find_ls1_64(TUint64 /*v*/)
|
|
|
256 |
{
|
|
|
257 |
/* On entry argument in R1:R0 */
|
|
|
258 |
#ifdef __CPU_ARM_HAS_CLZ
|
|
|
259 |
asm("subs r2, r0, #1 ");
|
|
|
260 |
asm("sbcs r3, r1, #0 "); // r3:r2 = arg - 1
|
|
|
261 |
asm("eorcs r0, r0, r2 "); // if arg=0 leave alone else mask upper bits
|
|
|
262 |
asm("eorcs r1, r1, r3 ");
|
|
|
263 |
CLZ( 2,1); // r2=31-MSB(r1), 32 if r1=0
|
|
|
264 |
asm("subs r2, r2, #32 "); // r2=-1-MSB(r1), 0 if r1=0
|
|
|
265 |
CLZcc(CC_EQ,2,0); // if r1=0, r2=31-MSB(r0), 32 if r0=0
|
|
|
266 |
asm("rsb r0, r2, #31 "); // if r1!=0, r0=32+MSB(r1) else if r0!=0 r0=MSB(r0) else r0=-1
|
|
|
267 |
#else
|
|
|
268 |
asm("cmp r0, #1 "); // LSW(arg) >= 1?
|
|
|
269 |
asm("movcs r1, r0 "); // if so r1=r0
|
|
|
270 |
asm("movcs r0, #32 "); // and r0=32
|
|
|
271 |
asm("movcc r0, #0 "); // else r0=0
|
|
|
272 |
asm("cmpcc r1, #1 "); // and test if MSW(arg) >= 1
|
|
|
273 |
asm("bcc 0f "); // if not, return -1
|
|
|
274 |
asm("movs r2, r1, lsl #16 ");
|
|
|
275 |
asm("movne r1, r2 ");
|
|
|
276 |
asm("addeq r0, r0, #16 ");
|
|
|
277 |
asm("movs r2, r1, lsl #8 ");
|
|
|
278 |
asm("movne r1, r2 ");
|
|
|
279 |
asm("addeq r0, r0, #8 ");
|
|
|
280 |
asm("movs r2, r1, lsl #4 ");
|
|
|
281 |
asm("movne r1, r2 ");
|
|
|
282 |
asm("addeq r0, r0, #4 ");
|
|
|
283 |
asm("movs r2, r1, lsl #2 ");
|
|
|
284 |
asm("movne r1, r2 ");
|
|
|
285 |
asm("addeq r0, r0, #2 ");
|
|
|
286 |
asm("movs r2, r1, lsl #1 ");
|
|
|
287 |
asm("addeq r0, r0, #1 ");
|
|
|
288 |
__JUMP(, lr);
|
|
|
289 |
asm("0: ");
|
|
|
290 |
asm("mvn r0, #0 "); // if input zero, return -1
|
|
|
291 |
#endif
|
|
|
292 |
__JUMP(, lr);
|
|
|
293 |
}
|
|
|
294 |
|
|
|
295 |
|
|
|
296 |
/** Count the number of 1's in a 64 bit word
|
|
|
297 |
|
|
|
298 |
@param v The word to be scanned
|
|
|
299 |
@return The number of 1's
|
|
|
300 |
*/
|
|
|
301 |
extern "C" EXPORT_C __NAKED__ TInt __e32_bit_count_64(TUint64 /*v*/)
|
|
|
302 |
{
|
|
|
303 |
/* On entry argument in R1:R0 */
|
|
|
304 |
asm("str r4, [sp, #-4]! ");
|
|
|
305 |
asm("mov r2, #0x0f "); // r2=0x0000000f
|
|
|
306 |
asm("orr r2, r2, r2, lsl #8 "); // r2=0x00000f0f
|
|
|
307 |
asm("orr r2, r2, r2, lsl #16 "); // r2=0x0f0f0f0f
|
|
|
308 |
asm("eor r3, r2, r2, lsl #2 "); // r3=0x33333333
|
|
|
309 |
asm("eor ip, r3, r3, lsl #1 "); // ip=0x55555555
|
|
|
310 |
|
|
|
311 |
asm("bic r4, r0, ip "); // r4=odd bits of input LSW
|
|
|
312 |
asm("and r0, r0, ip "); // r0=even bits of input LSW
|
|
|
313 |
asm("add r0, r0, r4, lsr #1 "); // r0[2n:2n+1] = in[2n]+in[2n+1], 0<=n<=15
|
|
|
314 |
asm("bic r4, r0, r3 "); // r4 = r0[4n+2:4n+3] for 0<=n<=7, other bits 0
|
|
|
315 |
asm("and r0, r0, r3 "); // r0 = r0[4n:4n+1] for 0<=n<=7, other bits 0
|
|
|
316 |
asm("add r0, r0, r4, lsr #2 "); // r0 bits 4n:4n+3 = in[4n]+in[4n+1]+in[4n+2]+in[4n+3], 0<=n<=7
|
|
|
317 |
|
|
|
318 |
asm("bic r4, r1, ip "); // r4=odd bits of input MSW
|
|
|
319 |
asm("and r1, r1, ip "); // r1=even bits of input MSW
|
|
|
320 |
asm("add r1, r1, r4, lsr #1 "); // r1[2n:2n+1] = in[2n+32]+in[2n+33], 0<=n<=15
|
|
|
321 |
asm("bic r4, r1, r3 "); // r4 = r1[4n+34:4n+35] for 0<=n<=7, other bits 0
|
|
|
322 |
asm("and r1, r1, r3 "); // r1 = r1[4n+32:4n+33] for 0<=n<=7, other bits 0
|
|
|
323 |
asm("add r1, r1, r4, lsr #2 "); // r1 bits 4n:4n+3 = in[4n+32]+in[4n+33]+in[4n+34]+in[4n+35], 0<=n<=7
|
|
|
324 |
asm("ldr r4, [sp], #4 ");
|
|
|
325 |
|
|
|
326 |
asm("add r0, r0, r1 "); // r0 bits 4n:4n+3 = in[4n]+in[4n+1]+in[4n+2]+in[4n+3]+in[4n+32]+in[4n+33]+in[4n+34]+in[4n+35], 0<=n<=7
|
|
|
327 |
asm("bic r1, r0, r2 "); // odd nibbles only
|
|
|
328 |
asm("and r0, r0, r2 "); // even nibbles only
|
|
|
329 |
asm("add r0, r0, r1, lsr #4 "); // r0[8n:8n+7]=bit count of byte n of MSW + bit count of byte n of LSW
|
|
|
330 |
asm("add r0, r0, r0, lsr #8 "); // r0[16n:16n+7]=bit count of hword n of MSW + bit count of hword n of LSW
|
|
|
331 |
asm("add r0, r0, r0, lsr #16 "); // r0[0:7]=total bit count
|
|
|
332 |
asm("and r0, r0, #0xff "); // mask other unwanted bits
|
|
|
333 |
__JUMP(, lr);
|
|
|
334 |
}
|
|
|
335 |
|
|
|
336 |
|
|
|
337 |
|
|
|
338 |
/******************************************************************************
|
|
|
339 |
* 64 bit operations
|
|
|
340 |
******************************************************************************/
|
|
|
341 |
#define __DATA_SIZE__ 64
|
|
|
342 |
#if defined(__CPU_ARM_HAS_LDREX_STREX_V6K) && !defined(__AVOID_READ_SIDE_EFFECTS__)
|
|
|
343 |
|
|
|
344 |
// Include LDREXD/STREXD-based 64 bit operations
|
|
|
345 |
#define __OP_LOAD__
|
|
|
346 |
#include "atomic_64_v6k.h"
|
|
|
347 |
#define __OP_STORE__
|
|
|
348 |
#include "atomic_64_v6k.h"
|
|
|
349 |
#define __OP_SWP__
|
|
|
350 |
#include "atomic_64_v6k.h"
|
|
|
351 |
#define __OP_CAS__
|
|
|
352 |
#include "atomic_64_v6k.h"
|
|
|
353 |
#define __OP_ADD__
|
|
|
354 |
#include "atomic_64_v6k.h"
|
|
|
355 |
#define __OP_AND__
|
|
|
356 |
#include "atomic_64_v6k.h"
|
|
|
357 |
#define __OP_IOR__
|
|
|
358 |
#include "atomic_64_v6k.h"
|
|
|
359 |
#define __OP_XOR__
|
|
|
360 |
#include "atomic_64_v6k.h"
|
|
|
361 |
#define __OP_AXO__
|
|
|
362 |
#include "atomic_64_v6k.h"
|
|
|
363 |
#define __OP_TAU__
|
|
|
364 |
#include "atomic_64_v6k.h"
|
|
|
365 |
#define __OP_TAS__
|
|
|
366 |
#include "atomic_64_v6k.h"
|
|
|
367 |
|
|
|
368 |
#else
|
|
|
369 |
#ifdef __KERNEL_MODE__
|
|
|
370 |
|
|
|
371 |
// Include interrupt-disabling 64 bit operations
|
|
|
372 |
#define __OP_LOAD__
|
|
|
373 |
#include "atomic_64_v6_v5.h"
|
|
|
374 |
#define __OP_STORE__
|
|
|
375 |
#include "atomic_64_v6_v5.h"
|
|
|
376 |
#define __OP_SWP__
|
|
|
377 |
#include "atomic_64_v6_v5.h"
|
|
|
378 |
#define __OP_CAS__
|
|
|
379 |
#include "atomic_64_v6_v5.h"
|
|
|
380 |
#define __OP_ADD__
|
|
|
381 |
#include "atomic_64_v6_v5.h"
|
|
|
382 |
#define __OP_AND__
|
|
|
383 |
#include "atomic_64_v6_v5.h"
|
|
|
384 |
#define __OP_IOR__
|
|
|
385 |
#include "atomic_64_v6_v5.h"
|
|
|
386 |
#define __OP_XOR__
|
|
|
387 |
#include "atomic_64_v6_v5.h"
|
|
|
388 |
#define __OP_AXO__
|
|
|
389 |
#include "atomic_64_v6_v5.h"
|
|
|
390 |
#define __OP_TAU__
|
|
|
391 |
#include "atomic_64_v6_v5.h"
|
|
|
392 |
#define __OP_TAS__
|
|
|
393 |
#include "atomic_64_v6_v5.h"
|
|
|
394 |
|
|
|
395 |
#else
|
|
|
396 |
|
|
|
397 |
// Include 64 bit operations using Exec calls
|
|
|
398 |
#define __OP_LOAD__
|
|
|
399 |
#include "atomic_64_v6_v5.h"
|
|
|
400 |
#define __OP_STORE__
|
|
|
401 |
#include "atomic_64_v6_v5.h"
|
|
|
402 |
#define __OP_SWP__
|
|
|
403 |
#include "atomic_64_exec.h"
|
|
|
404 |
#define __OP_CAS__
|
|
|
405 |
#include "atomic_64_exec.h"
|
|
|
406 |
#define __OP_ADD__
|
|
|
407 |
#include "atomic_64_exec.h"
|
|
|
408 |
#define __OP_AND__
|
|
|
409 |
#include "atomic_64_exec.h"
|
|
|
410 |
#define __OP_IOR__
|
|
|
411 |
#include "atomic_64_exec.h"
|
|
|
412 |
#define __OP_XOR__
|
|
|
413 |
#include "atomic_64_exec.h"
|
|
|
414 |
#define __OP_AXO__
|
|
|
415 |
#include "atomic_64_exec.h"
|
|
|
416 |
#define __OP_TAU__
|
|
|
417 |
#include "atomic_64_exec.h"
|
|
|
418 |
#define __OP_TAS__
|
|
|
419 |
#include "atomic_64_exec.h"
|
|
|
420 |
|
|
|
421 |
#endif
|
|
|
422 |
#endif
|
|
|
423 |
#undef __DATA_SIZE__
|
|
|
424 |
|
|
|
425 |
/******************************************************************************
|
|
|
426 |
* 8,16,32 bit load/store operations
|
|
|
427 |
******************************************************************************/
|
|
|
428 |
|
|
|
429 |
#define __DATA_SIZE__ 8
|
|
|
430 |
#define __OP_LOAD__
|
|
|
431 |
#include "atomic_32_v6.h"
|
|
|
432 |
#define __OP_STORE__
|
|
|
433 |
#include "atomic_32_v6.h"
|
|
|
434 |
#undef __DATA_SIZE__
|
|
|
435 |
|
|
|
436 |
#define __DATA_SIZE__ 16
|
|
|
437 |
#define __OP_LOAD__
|
|
|
438 |
#include "atomic_32_v6.h"
|
|
|
439 |
#define __OP_STORE__
|
|
|
440 |
#include "atomic_32_v6.h"
|
|
|
441 |
#undef __DATA_SIZE__
|
|
|
442 |
|
|
|
443 |
#define __DATA_SIZE__ 32
|
|
|
444 |
#define __OP_LOAD__
|
|
|
445 |
#include "atomic_32_v6.h"
|
|
|
446 |
#define __OP_STORE__
|
|
|
447 |
#include "atomic_32_v6.h"
|
|
|
448 |
#undef __DATA_SIZE__
|
|
|
449 |
|
|
|
450 |
/******************************************************************************
|
|
|
451 |
* 8,16,32 bit RMW operations
|
|
|
452 |
******************************************************************************/
|
|
|
453 |
|
|
|
454 |
#if defined(__CPU_ARM_HAS_LDREX_STREX_V6K) && !defined(__AVOID_READ_SIDE_EFFECTS__)
|
|
|
455 |
// V6K - Use variants of LDREX/STREX for everything
|
|
|
456 |
#define __ATOMIC_8_IMPL__ "atomic_32_v6.h"
|
|
|
457 |
#define __ATOMIC_16_IMPL__ "atomic_32_v6.h"
|
|
|
458 |
#define __ATOMIC_32_IMPL__ "atomic_32_v6.h"
|
|
|
459 |
#elif defined(__CPU_ARM_HAS_LDREX_STREX) && !defined(__AVOID_READ_SIDE_EFFECTS__)
|
|
|
460 |
// V6 - Use LDREX/STREX for 32 bit operations
|
|
|
461 |
// Use LDREX/STREX with shifts/rotates for 8/16 bit operations
|
|
|
462 |
#define __ATOMIC_8_IMPL__ "atomic_8_16_v6.h"
|
|
|
463 |
#define __ATOMIC_16_IMPL__ "atomic_8_16_v6.h"
|
|
|
464 |
#define __ATOMIC_32_IMPL__ "atomic_32_v6.h"
|
|
|
465 |
#else
|
|
|
466 |
// V5 - Use interrupt disabling kernel side, Exec calls user side
|
|
|
467 |
#ifdef __KERNEL_MODE__
|
|
|
468 |
#define __ATOMIC_8_IMPL__ "atomic_8_16_32_irq.h"
|
|
|
469 |
#define __ATOMIC_16_IMPL__ "atomic_8_16_32_irq.h"
|
|
|
470 |
#define __ATOMIC_32_IMPL__ "atomic_8_16_32_irq.h"
|
|
|
471 |
#else
|
|
|
472 |
#define __ATOMIC_8_IMPL__ "atomic_8_16_32_exec.h"
|
|
|
473 |
#define __ATOMIC_16_IMPL__ "atomic_8_16_32_exec.h"
|
|
|
474 |
#define __ATOMIC_32_IMPL__ "atomic_8_16_32_exec.h"
|
|
|
475 |
#endif
|
|
|
476 |
#endif
|
|
|
477 |
|
|
|
478 |
#define __DATA_SIZE__ 8
|
|
|
479 |
#define __OP_SWP__
|
|
|
480 |
#include __ATOMIC_8_IMPL__
|
|
|
481 |
#define __OP_CAS__
|
|
|
482 |
#include __ATOMIC_8_IMPL__
|
|
|
483 |
#define __OP_ADD__
|
|
|
484 |
#include __ATOMIC_8_IMPL__
|
|
|
485 |
#define __OP_AND__
|
|
|
486 |
#include __ATOMIC_8_IMPL__
|
|
|
487 |
#define __OP_IOR__
|
|
|
488 |
#include __ATOMIC_8_IMPL__
|
|
|
489 |
#define __OP_XOR__
|
|
|
490 |
#include __ATOMIC_8_IMPL__
|
|
|
491 |
#define __OP_AXO__
|
|
|
492 |
#include __ATOMIC_8_IMPL__
|
|
|
493 |
#define __OP_TAU__
|
|
|
494 |
#include __ATOMIC_8_IMPL__
|
|
|
495 |
#define __OP_TAS__
|
|
|
496 |
#include __ATOMIC_8_IMPL__
|
|
|
497 |
#undef __DATA_SIZE__
|
|
|
498 |
|
|
|
499 |
#define __DATA_SIZE__ 16
|
|
|
500 |
#define __OP_SWP__
|
|
|
501 |
#include __ATOMIC_16_IMPL__
|
|
|
502 |
#define __OP_CAS__
|
|
|
503 |
#include __ATOMIC_16_IMPL__
|
|
|
504 |
#define __OP_ADD__
|
|
|
505 |
#include __ATOMIC_16_IMPL__
|
|
|
506 |
#define __OP_AND__
|
|
|
507 |
#include __ATOMIC_16_IMPL__
|
|
|
508 |
#define __OP_IOR__
|
|
|
509 |
#include __ATOMIC_16_IMPL__
|
|
|
510 |
#define __OP_XOR__
|
|
|
511 |
#include __ATOMIC_16_IMPL__
|
|
|
512 |
#define __OP_AXO__
|
|
|
513 |
#include __ATOMIC_16_IMPL__
|
|
|
514 |
#define __OP_TAU__
|
|
|
515 |
#include __ATOMIC_16_IMPL__
|
|
|
516 |
#define __OP_TAS__
|
|
|
517 |
#include __ATOMIC_16_IMPL__
|
|
|
518 |
#undef __DATA_SIZE__
|
|
|
519 |
|
|
|
520 |
#define __DATA_SIZE__ 32
|
|
|
521 |
#define __OP_SWP__
|
|
|
522 |
#include __ATOMIC_32_IMPL__
|
|
|
523 |
#define __OP_CAS__
|
|
|
524 |
#include __ATOMIC_32_IMPL__
|
|
|
525 |
#define __OP_ADD__
|
|
|
526 |
#include __ATOMIC_32_IMPL__
|
|
|
527 |
#define __OP_AND__
|
|
|
528 |
#include __ATOMIC_32_IMPL__
|
|
|
529 |
#define __OP_IOR__
|
|
|
530 |
#include __ATOMIC_32_IMPL__
|
|
|
531 |
#define __OP_XOR__
|
|
|
532 |
#include __ATOMIC_32_IMPL__
|
|
|
533 |
#define __OP_AXO__
|
|
|
534 |
#include __ATOMIC_32_IMPL__
|
|
|
535 |
#define __OP_TAU__
|
|
|
536 |
#include __ATOMIC_32_IMPL__
|
|
|
537 |
#define __OP_TAS__
|
|
|
538 |
#include __ATOMIC_32_IMPL__
|
|
|
539 |
#undef __DATA_SIZE__
|
|
|
540 |
|