1 // Copyright (c) 2008-2009 Nokia Corporation and/or its subsidiary(-ies).

     2 // All rights reserved.

     3 // This component and the accompanying materials are made available

     4 // under the terms of the License "Eclipse Public License v1.0"

     5 // which accompanies this distribution, and is available

     6 // at the URL "http://www.eclipse.org/legal/epl-v10.html".

     7 //

     8 // Initial Contributors:

     9 // Nokia Corporation - initial contribution.

    10 //

    11 // Contributors:

    12 //

    13 // Description:

    14 // e32\common\x86\atomics.cpp

    15 // 

    16 //

    17 

    18 #include <e32atomics.h>

    19 #include <cpudefs.h>

    20 

    21 /*

    22 Versions needed:

    23 	WINS/WINSCW		Use X86 locked operations. Assume Pentium or above CPU (CMPXCHG8B available)

    24 	X86				For Pentium and above use locked operations

    25 					For 486 use locked operations for 8, 16, 32 bit. For 64 bit must disable interrupts.

    26 					NOTE: 486 not supported at the moment

    27 	ARMv4/ARMv5		Must disable interrupts.

    28 	ARMv6			LDREX/STREX for 8, 16, 32 bit. For 64 bit must disable interrupts (maybe).

    29 	ARMv6K/ARMv7	LDREXB/LDREXH/LDREX/LDREXD

    30 

    31 Need both kernel side and user side versions

    32 */

    33 

    34 #if	defined(__SMP__) || !defined(__EPOC32__)

    35 #define	__BARRIERS_NEEDED__

    36 #define	__LOCK__	lock

    37 #else

    38 #define	__LOCK__

    39 #endif

    40 

    41 

    42 extern "C" {

    43 

    44 #undef	__TUintX__

    45 #undef	__TIntX__

    46 #undef	__fname__

    47 #undef	__A_REG__

    48 #undef	__C_REG__

    49 #undef	__D_REG__

    50 #define	__TUintX__		TUint32

    51 #define	__TIntX__		TInt32

    52 #define	__fname__(x)	x##32

    53 #define	__A_REG__		eax

    54 #define	__C_REG__		ecx

    55 #define	__D_REG__		edx

    56 #include "atomic_skeleton.h"

    57 

    58 #undef	__TUintX__

    59 #undef	__TIntX__

    60 #undef	__fname__

    61 #undef	__A_REG__

    62 #undef	__C_REG__

    63 #undef	__D_REG__

    64 #define	__TUintX__		TUint16

    65 #define	__TIntX__		TInt16

    66 #define	__fname__(x)	x##16

    67 #define	__A_REG__		ax

    68 #define	__C_REG__		cx

    69 #define	__D_REG__		dx

    70 #include "atomic_skeleton.h"

    71 

    72 #undef	__TUintX__

    73 #undef	__TIntX__

    74 #undef	__fname__

    75 #undef	__A_REG__

    76 #undef	__C_REG__

    77 #undef	__D_REG__

    78 #define	__TUintX__		TUint8

    79 #define	__TIntX__		TInt8

    80 #define	__fname__(x)	x##8

    81 #define	__A_REG__		al

    82 #define	__C_REG__		cl

    83 #define	__D_REG__		dl

    84 #include "atomic_skeleton.h"

    85 

    86 #undef	__TUintX__

    87 #undef	__TIntX__

    88 #undef	__fname__

    89 #undef	__A_REG__

    90 #undef	__C_REG__

    91 #undef	__D_REG__

    92 

    93 /** Full memory barrier for explicit memory accesses

    94 

    95 */

    96 EXPORT_C __NAKED__ void __e32_memory_barrier()

    97 	{

    98 #ifdef __BARRIERS_NEEDED__

    99 	_asm lock add dword ptr [esp], 0

   100 #endif

   101 	_asm ret

   102 	}

   103 

   104 

   105 /** Barrier guaranteeing completion as well as ordering

   106 

   107 */

   108 EXPORT_C __NAKED__ void __e32_io_completion_barrier()

   109 	{

   110 	_asm push ebx

   111 	_asm cpuid

   112 	_asm pop ebx

   113 	_asm ret

   114 	}

   115 

   116 

   117 /** Find the most significant 1 in a 32 bit word

   118 

   119 	@param	v	The word to be scanned

   120 	@return		The bit number of the most significant 1 if v != 0

   121 				-1 if v == 0

   122 */

   123 EXPORT_C __NAKED__ TInt __e32_find_ms1_32(TUint32 /*v*/)

   124 	{

   125 	_asm bsr eax, [esp+4]

   126 	_asm jnz short done

   127 	_asm mov eax, 0ffffffffh

   128 done:

   129 	_asm ret

   130 	}

   131 

   132 

   133 /** Find the least significant 1 in a 32 bit word

   134 

   135 	@param	v	The word to be scanned

   136 	@return		The bit number of the least significant 1 if v != 0

   137 				-1 if v == 0

   138 */

   139 EXPORT_C __NAKED__ TInt __e32_find_ls1_32(TUint32 /*v*/)

   140 	{

   141 	_asm bsf eax, [esp+4]

   142 	_asm jnz short done

   143 	_asm mov eax, 0ffffffffh

   144 done:

   145 	_asm ret

   146 	}

   147 

   148 

   149 /** Count the number of 1's in a 32 bit word

   150 

   151 	@param	v	The word to be scanned

   152 	@return		The number of 1's

   153 */

   154 EXPORT_C __NAKED__ TInt __e32_bit_count_32(TUint32 /*v*/)

   155 	{

   156 	_asm mov eax, [esp+4]

   157 	_asm mov edx, eax

   158 	_asm and eax, 0aaaaaaaah

   159 	_asm and edx, 055555555h

   160 	_asm shr eax, 1

   161 	_asm add eax, edx

   162 	_asm mov edx, eax

   163 	_asm and eax, 0cccccccch

   164 	_asm and edx, 033333333h

   165 	_asm shr eax, 2

   166 	_asm add eax, edx

   167 	_asm mov edx, eax

   168 	_asm shr eax, 4

   169 	_asm add eax, edx

   170 	_asm and eax, 00f0f0f0fh

   171 	_asm add al, ah

   172 	_asm mov dl, al

   173 	_asm shr eax, 16

   174 	_asm add al, ah

   175 	_asm xor ah, ah

   176 	_asm add al, dl

   177 	_asm ret

   178 	}

   179 

   180 

   181 /** Find the most significant 1 in a 64 bit word

   182 

   183 	@param	v	The word to be scanned

   184 	@return		The bit number of the most significant 1 if v != 0

   185 				-1 if v == 0

   186 */

   187 EXPORT_C __NAKED__ TInt __e32_find_ms1_64(TUint64 /*v*/)

   188 	{

   189 	_asm bsr eax, [esp+8]

   190 	_asm jnz short mswnz

   191 	_asm bsr eax, [esp+4]

   192 	_asm jnz short lswnz

   193 	_asm mov eax, 0ffffffffh

   194 mswnz:

   195 	_asm or eax, 32

   196 lswnz:

   197 	_asm ret

   198 	}

   199 

   200 

   201 /** Find the least significant 1 in a 64 bit word

   202 

   203 	@param	v	The word to be scanned

   204 	@return		The bit number of the least significant 1 if v != 0

   205 				-1 if v == 0

   206 */

   207 EXPORT_C __NAKED__ TInt __e32_find_ls1_64(TUint64 /*v*/)

   208 	{

   209 	_asm bsf eax, [esp+4]

   210 	_asm jnz short lswnz

   211 	_asm bsf eax, [esp+8]

   212 	_asm jnz short mswnz

   213 	_asm mov eax, 0ffffffffh

   214 mswnz:

   215 	_asm or eax, 32

   216 lswnz:

   217 	_asm ret

   218 	}

   219 

   220 

   221 /** Count the number of 1's in a 64 bit word

   222 

   223 	@param	v	The word to be scanned

   224 	@return		The number of 1's

   225 */

   226 EXPORT_C __NAKED__ TInt __e32_bit_count_64(TUint64 /*v*/)

   227 	{

   228 	_asm mov eax, [esp+4]

   229 	_asm mov edx, [esp+8]

   230 

   231 	_asm mov ecx, eax

   232 	_asm and eax, 0aaaaaaaah

   233 	_asm and ecx, 055555555h

   234 	_asm shr eax, 1

   235 	_asm add eax, ecx			/* 16 groups of 2 bits, count=0,1,2 */

   236 	_asm mov ecx, eax

   237 	_asm and eax, 0cccccccch

   238 	_asm and ecx, 033333333h

   239 	_asm shr eax, 2

   240 	_asm add ecx, eax			/* 8 groups of 4 bits, count=0...4 */

   241 

   242 	_asm mov eax, edx

   243 	_asm and eax, 0aaaaaaaah

   244 	_asm and edx, 055555555h

   245 	_asm shr eax, 1

   246 	_asm add eax, edx			/* 16 groups of 2 bits, count=0,1,2 */

   247 	_asm mov edx, eax

   248 	_asm and eax, 0cccccccch

   249 	_asm and edx, 033333333h

   250 	_asm shr eax, 2

   251 	_asm add eax, edx			/* 8 groups of 4 bits, count=0...4 */

   252 

   253 	_asm add eax, ecx			/* 8 groups of 4 bits, count=0...8 */

   254 	_asm mov edx, eax

   255 	_asm and eax, 0f0f0f0f0h

   256 	_asm and edx, 00f0f0f0fh

   257 	_asm shr eax, 4

   258 	_asm add eax, edx			/* 4 groups of 8 bits, count=0...16 */

   259 	_asm add al, ah

   260 	_asm mov dl, al

   261 	_asm shr eax, 16

   262 	_asm add al, ah

   263 	_asm xor ah, ah

   264 	_asm add al, dl

   265 	_asm ret

   266 	}

   267 

   268 

   269 

   270 

   271 /** Read a 64 bit word with acquire semantics

   272 

   273 	@param	a	Address of word to be read - must be a multiple of 8

   274 	@return		The value read

   275 */

   276 EXPORT_C __NAKED__ TUint64	__e32_atomic_load_acq64(const volatile TAny* /*a*/)

   277 	{

   278 	_asm push ebx

   279 	_asm push edi

   280 	_asm mov edi, [esp+12]

   281 	_asm mov eax, 0badbeefh

   282 	_asm mov edx, eax

   283 	_asm mov ebx, eax

   284 	_asm mov ecx, eax

   285 	_asm __LOCK__ cmpxchg8b [edi]

   286 	_asm pop edi

   287 	_asm pop ebx

   288 	_asm ret

   289 	}

   290 

   291 

   292 /** Write a 64 bit word with release semantics

   293 

   294 	@param	a	Address of word to be written - must be a multiple of 8

   295 	@param	v	The value to be written

   296 	@return		The value written

   297 */

   298 EXPORT_C __NAKED__ TUint64	__e32_atomic_store_rel64(volatile TAny* /*a*/, TUint64 /*v*/)

   299 	{

   300 	_asm push ebx

   301 	_asm push edi

   302 	_asm mov edi, [esp+12]

   303 	_asm mov ebx, [esp+16]

   304 	_asm mov ecx, [esp+20]

   305 	_asm mov eax, [edi]

   306 	_asm mov edx, [edi+4]

   307 	_asm retry:

   308 	_asm __LOCK__ cmpxchg8b [edi]

   309 	_asm jne short retry

   310 	_asm mov eax, ebx

   311 	_asm mov edx, ecx

   312 	_asm pop edi

   313 	_asm pop ebx

   314 	_asm ret

   315 	}

   316 

   317 

   318 /** Write a 64 bit word with full barrier semantics

   319 

   320 	@param	a	Address of word to be written - must be a multiple of 8

   321 	@param	v	The value to be written

   322 	@return		The value written

   323 */

   324 EXPORT_C __NAKED__ TUint64	__e32_atomic_store_ord64(volatile TAny* /*a*/, TUint64 /*v*/)

   325 	{

   326 	_asm jmp __e32_atomic_store_rel64

   327 	}

   328 

   329 

   330 /** Write a 64 bit word to memory and return the original value of the memory.

   331 	Relaxed ordering.

   332 

   333 	@param	a	Address of word to be written - must be a multiple of 8

   334 	@param	v	The value to be written

   335 	@return		The original value of *a

   336 */

   337 EXPORT_C __NAKED__ TUint64	__e32_atomic_swp_rlx64(volatile TAny* /*a*/, TUint64 /*v*/)

   338 	{

   339 	_asm jmp __e32_atomic_swp_ord64

   340 	}

   341 

   342 

   343 /** Write a 64 bit word to memory and return the original value of the memory.

   344 	Acquire semantics.

   345 

   346 	@param	a	Address of word to be written - must be a multiple of 8

   347 	@param	v	The value to be written

   348 	@return		The original value of *a

   349 */

   350 EXPORT_C __NAKED__ TUint64	__e32_atomic_swp_acq64(volatile TAny* /*a*/, TUint64 /*v*/)

   351 	{

   352 	_asm jmp __e32_atomic_swp_ord64

   353 	}

   354 

   355 

   356 /** Write a 64 bit word to memory and return the original value of the memory.

   357 	Release semantics.

   358 

   359 	@param	a	Address of word to be written - must be a multiple of 8

   360 	@param	v	The value to be written

   361 	@return		The original value of *a

   362 */

   363 EXPORT_C __NAKED__ TUint64	__e32_atomic_swp_rel64(volatile TAny* /*a*/, TUint64 /*v*/)

   364 	{

   365 	_asm jmp __e32_atomic_swp_ord64

   366 	}

   367 

   368 

   369 /** Write a 64 bit word to memory and return the original value of the memory.

   370 	Full barrier semantics.

   371 

   372 	@param	a	Address of word to be written - must be a multiple of 8

   373 	@param	v	The value to be written

   374 	@return		The original value of *a

   375 */

   376 EXPORT_C __NAKED__ TUint64	__e32_atomic_swp_ord64(volatile TAny* /*a*/, TUint64 /*v*/)

   377 	{

   378 	_asm push ebx

   379 	_asm push edi

   380 	_asm mov edi, [esp+12]

   381 	_asm mov ebx, [esp+16]

   382 	_asm mov ecx, [esp+20]

   383 	_asm mov eax, [edi]

   384 	_asm mov edx, [edi+4]

   385 	_asm retry:

   386 	_asm __LOCK__ cmpxchg8b [edi]

   387 	_asm jne short retry

   388 	_asm pop edi

   389 	_asm pop ebx

   390 	_asm ret

   391 	}

   392 

   393 

   394 /** 64 bit compare and swap, relaxed ordering.

   395 

   396 	Atomically performs the following operation:

   397 		if (*a == *q)	{ *a = v; return TRUE; }

   398 		else			{ *q = *a; return FALSE; }

   399 

   400 	@param	a	Address of word to be written - must be a multiple of 8

   401 	@param	q	Address of location containing expected value

   402 	@param	v	The new value to be written if the old value is as expected

   403 	@return		TRUE if *a was updated, FALSE otherwise

   404 */

   405 EXPORT_C __NAKED__ TBool		__e32_atomic_cas_rlx64(volatile TAny* /*a*/, TUint64* /*q*/, TUint64 /*v*/)

   406 	{

   407 	_asm jmp __e32_atomic_cas_ord64

   408 	}

   409 

   410 

   411 /** 64 bit compare and swap, acquire semantics.

   412 

   413 	Atomically performs the following operation:

   414 		if (*a == *q)	{ *a = v; return TRUE; }

   415 		else			{ *q = *a; return FALSE; }

   416 

   417 	@param	a	Address of word to be written - must be a multiple of 8

   418 	@param	q	Address of location containing expected value

   419 	@param	v	The new value to be written if the old value is as expected

   420 	@return		TRUE if *a was updated, FALSE otherwise

   421 */

   422 EXPORT_C __NAKED__ TBool		__e32_atomic_cas_acq64(volatile TAny* /*a*/, TUint64* /*q*/, TUint64 /*v*/)

   423 	{

   424 	_asm jmp __e32_atomic_cas_ord64

   425 	}

   426 

   427 

   428 /** 64 bit compare and swap, release semantics.

   429 

   430 	Atomically performs the following operation:

   431 		if (*a == *q)	{ *a = v; return TRUE; }

   432 		else			{ *q = *a; return FALSE; }

   433 

   434 	@param	a	Address of word to be written - must be a multiple of 8

   435 	@param	q	Address of location containing expected value

   436 	@param	v	The new value to be written if the old value is as expected

   437 	@return		TRUE if *a was updated, FALSE otherwise

   438 */

   439 EXPORT_C __NAKED__ TBool		__e32_atomic_cas_rel64(volatile TAny* /*a*/, TUint64* /*q*/, TUint64 /*v*/)

   440 	{

   441 	_asm jmp __e32_atomic_cas_ord64

   442 	}

   443 

   444 

   445 /** 64 bit compare and swap, full barrier semantics.

   446 

   447 	Atomically performs the following operation:

   448 		if (*a == *q)	{ *a = v; return TRUE; }

   449 		else			{ *q = *a; return FALSE; }

   450 

   451 	@param	a	Address of word to be written - must be a multiple of 8

   452 	@param	q	Address of location containing expected value

   453 	@param	v	The new value to be written if the old value is as expected

   454 	@return		TRUE if *a was updated, FALSE otherwise

   455 */

   456 EXPORT_C __NAKED__ TBool		__e32_atomic_cas_ord64(volatile TAny* /*a*/, TUint64* /*q*/, TUint64 /*v*/)

   457 	{

   458 	_asm push ebx

   459 	_asm push edi

   460 	_asm push esi

   461 	_asm mov edi, [esp+16]				// edi = a

   462 	_asm mov esi, [esp+20]				// esi = q

   463 	_asm mov ebx, [esp+24]				// ecx:ebx = v

   464 	_asm mov ecx, [esp+28]

   465 	_asm mov eax, [esi]					// edx:eax = *q

   466 	_asm mov edx, [esi+4]

   467 	_asm __LOCK__ cmpxchg8b [edi]		// if (*a==*q) *a=v, ZF=1 else edx:eax=*a, ZF=0

   468 	_asm jne short cas_fail

   469 	_asm mov eax, 1

   470 	_asm pop esi

   471 	_asm pop edi

   472 	_asm pop ebx

   473 	_asm ret

   474 	_asm cas_fail:

   475 	_asm mov [esi], eax					// *q = edx:eax

   476 	_asm mov [esi+4], edx

   477 	_asm xor eax, eax

   478 	_asm pop esi

   479 	_asm pop edi

   480 	_asm pop ebx

   481 	_asm ret

   482 	}

   483 

   484 

   485 /** 64 bit atomic add, relaxed ordering.

   486 

   487 	Atomically performs the following operation:

   488 		oldv = *a; *a = oldv + v; return oldv;

   489 

   490 	@param	a	Address of word to be updated - must be a multiple of 8

   491 	@param	v	The value to be added

   492 	@return		The original value of *a

   493 */

   494 EXPORT_C __NAKED__ TUint64	__e32_atomic_add_rlx64(volatile TAny* /*a*/, TUint64 /*v*/)

   495 	{

   496 	_asm jmp __e32_atomic_add_ord64

   497 	}

   498 

   499 

   500 /** 64 bit atomic add, acquire semantics.

   501 

   502 	Atomically performs the following operation:

   503 		oldv = *a; *a = oldv + v; return oldv;

   504 

   505 	@param	a	Address of word to be updated - must be a multiple of 8

   506 	@param	v	The value to be added

   507 	@return		The original value of *a

   508 */

   509 EXPORT_C __NAKED__ TUint64	__e32_atomic_add_acq64(volatile TAny* /*a*/, TUint64 /*v*/)

   510 	{

   511 	_asm jmp __e32_atomic_add_ord64

   512 	}

   513 

   514 

   515 /** 64 bit atomic add, release semantics.

   516 

   517 	Atomically performs the following operation:

   518 		oldv = *a; *a = oldv + v; return oldv;

   519 

   520 	@param	a	Address of word to be updated - must be a multiple of 8

   521 	@param	v	The value to be added

   522 	@return		The original value of *a

   523 */

   524 EXPORT_C __NAKED__ TUint64	__e32_atomic_add_rel64(volatile TAny* /*a*/, TUint64 /*v*/)

   525 	{

   526 	_asm jmp __e32_atomic_add_ord64

   527 	}

   528 

   529 

   530 /** 64 bit atomic add, full barrier semantics.

   531 

   532 	Atomically performs the following operation:

   533 		oldv = *a; *a = oldv + v; return oldv;

   534 

   535 	@param	a	Address of word to be updated - must be a multiple of 8

   536 	@param	v	The value to be added

   537 	@return		The original value of *a

   538 */

   539 EXPORT_C __NAKED__ TUint64	__e32_atomic_add_ord64(volatile TAny* /*a*/, TUint64 /*v*/)

   540 	{

   541 	_asm push ebx

   542 	_asm push edi

   543 	_asm mov edi, [esp+12]				// edi = a

   544 	_asm mov eax, [edi]					// edx:eax = oldv

   545 	_asm mov edx, [edi+4]

   546 	_asm retry:

   547 	_asm mov ebx, eax

   548 	_asm mov ecx, edx

   549 	_asm add ebx, [esp+16]				// ecx:ebx = oldv + v

   550 	_asm adc ecx, [esp+20]

   551 	_asm __LOCK__ cmpxchg8b [edi]		// if (*a==oldv) *a=oldv+v, ZF=1 else edx:eax=*a, ZF=0

   552 	_asm jne short retry

   553 	_asm pop edi

   554 	_asm pop ebx

   555 	_asm ret

   556 	}

   557 

   558 

   559 /** 64 bit atomic bitwise logical AND, relaxed ordering.

   560 

   561 	Atomically performs the following operation:

   562 		oldv = *a; *a = oldv & v; return oldv;

   563 

   564 	@param	a	Address of word to be updated - must be a multiple of 8

   565 	@param	v	The value to be ANDed with *a

   566 	@return		The original value of *a

   567 */

   568 EXPORT_C __NAKED__ TUint64	__e32_atomic_and_rlx64(volatile TAny* /*a*/, TUint64 /*v*/)

   569 	{

   570 	_asm jmp __e32_atomic_and_ord64

   571 	}

   572 

   573 

   574 /** 64 bit atomic bitwise logical AND, acquire semantics.

   575 

   576 	Atomically performs the following operation:

   577 		oldv = *a; *a = oldv & v; return oldv;

   578 

   579 	@param	a	Address of word to be updated - must be a multiple of 8

   580 	@param	v	The value to be ANDed with *a

   581 	@return		The original value of *a

   582 */

   583 EXPORT_C __NAKED__ TUint64	__e32_atomic_and_acq64(volatile TAny* /*a*/, TUint64 /*v*/)

   584 	{

   585 	_asm jmp __e32_atomic_and_ord64

   586 	}

   587 

   588 

   589 /** 64 bit atomic bitwise logical AND, release semantics.

   590 

   591 	Atomically performs the following operation:

   592 		oldv = *a; *a = oldv & v; return oldv;

   593 

   594 	@param	a	Address of word to be updated - must be a multiple of 8

   595 	@param	v	The value to be ANDed with *a

   596 	@return		The original value of *a

   597 */

   598 EXPORT_C __NAKED__ TUint64	__e32_atomic_and_rel64(volatile TAny* /*a*/, TUint64 /*v*/)

   599 	{

   600 	_asm jmp __e32_atomic_and_ord64

   601 	}

   602 

   603 

   604 /** 64 bit atomic bitwise logical AND, full barrier semantics.

   605 

   606 	Atomically performs the following operation:

   607 		oldv = *a; *a = oldv & v; return oldv;

   608 

   609 	@param	a	Address of word to be updated - must be a multiple of 8

   610 	@param	v	The value to be ANDed with *a

   611 	@return		The original value of *a

   612 */

   613 EXPORT_C __NAKED__ TUint64	__e32_atomic_and_ord64(volatile TAny* /*a*/, TUint64 /*v*/)

   614 	{

   615 	_asm push ebx

   616 	_asm push edi

   617 	_asm mov edi, [esp+12]				// edi = a

   618 	_asm mov eax, [edi]					// edx:eax = oldv

   619 	_asm mov edx, [edi+4]

   620 	_asm retry:

   621 	_asm mov ebx, eax

   622 	_asm mov ecx, edx

   623 	_asm and ebx, [esp+16]				// ecx:ebx = oldv & v

   624 	_asm and ecx, [esp+20]

   625 	_asm __LOCK__ cmpxchg8b [edi]		// if (*a==oldv) *a=oldv&v, ZF=1 else edx:eax=*a, ZF=0

   626 	_asm jne short retry

   627 	_asm pop edi

   628 	_asm pop ebx

   629 	_asm ret

   630 	}

   631 

   632 

   633 /** 64 bit atomic bitwise logical inclusive OR, relaxed ordering.

   634 

   635 	Atomically performs the following operation:

   636 		oldv = *a; *a = oldv | v; return oldv;

   637 

   638 	@param	a	Address of word to be updated - must be a multiple of 8

   639 	@param	v	The value to be ORed with *a

   640 	@return		The original value of *a

   641 */

   642 EXPORT_C __NAKED__ TUint64	__e32_atomic_ior_rlx64(volatile TAny* /*a*/, TUint64 /*v*/)

   643 	{

   644 	_asm jmp __e32_atomic_ior_ord64

   645 	}

   646 

   647 

   648 /** 64 bit atomic bitwise logical inclusive OR, acquire semantics.

   649 

   650 	Atomically performs the following operation:

   651 		oldv = *a; *a = oldv | v; return oldv;

   652 

   653 	@param	a	Address of word to be updated - must be a multiple of 8

   654 	@param	v	The value to be ORed with *a

   655 	@return		The original value of *a

   656 */

   657 EXPORT_C __NAKED__ TUint64	__e32_atomic_ior_acq64(volatile TAny* /*a*/, TUint64 /*v*/)

   658 	{

   659 	_asm jmp __e32_atomic_ior_ord64

   660 	}

   661 

   662 

   663 /** 64 bit atomic bitwise logical inclusive OR, release semantics.

   664 

   665 	Atomically performs the following operation:

   666 		oldv = *a; *a = oldv | v; return oldv;

   667 

   668 	@param	a	Address of word to be updated - must be a multiple of 8

   669 	@param	v	The value to be ORed with *a

   670 	@return		The original value of *a

   671 */

   672 EXPORT_C __NAKED__ TUint64	__e32_atomic_ior_rel64(volatile TAny* /*a*/, TUint64 /*v*/)

   673 	{

   674 	_asm jmp __e32_atomic_ior_ord64

   675 	}

   676 

   677 

   678 /** 64 bit atomic bitwise logical inclusive OR, full barrier semantics.

   679 

   680 	Atomically performs the following operation:

   681 		oldv = *a; *a = oldv | v; return oldv;

   682 

   683 	@param	a	Address of word to be updated - must be a multiple of 8

   684 	@param	v	The value to be ORed with *a

   685 	@return		The original value of *a

   686 */

   687 EXPORT_C __NAKED__ TUint64	__e32_atomic_ior_ord64(volatile TAny* /*a*/, TUint64 /*v*/)

   688 	{

   689 	_asm push ebx

   690 	_asm push edi

   691 	_asm mov edi, [esp+12]				// edi = a

   692 	_asm mov eax, [edi]					// edx:eax = oldv

   693 	_asm mov edx, [edi+4]

   694 	_asm retry:

   695 	_asm mov ebx, eax

   696 	_asm mov ecx, edx

   697 	_asm or ebx, [esp+16]				// ecx:ebx = oldv | v

   698 	_asm or ecx, [esp+20]

   699 	_asm __LOCK__ cmpxchg8b [edi]		// if (*a==oldv) *a=oldv|v, ZF=1 else edx:eax=*a, ZF=0

   700 	_asm jne short retry

   701 	_asm pop edi

   702 	_asm pop ebx

   703 	_asm ret

   704 	}

   705 

   706 

   707 /** 64 bit atomic bitwise logical exclusive OR, relaxed ordering.

   708 

   709 	Atomically performs the following operation:

   710 		oldv = *a; *a = oldv ^ v; return oldv;

   711 

   712 	@param	a	Address of word to be updated - must be a multiple of 8

   713 	@param	v	The value to be XORed with *a

   714 	@return		The original value of *a

   715 */

   716 EXPORT_C __NAKED__ TUint64	__e32_atomic_xor_rlx64(volatile TAny* /*a*/, TUint64 /*v*/)

   717 	{

   718 	_asm jmp __e32_atomic_xor_ord64

   719 	}

   720 

   721 

   722 /** 64 bit atomic bitwise logical exclusive OR, acquire semantics.

   723 

   724 	Atomically performs the following operation:

   725 		oldv = *a; *a = oldv ^ v; return oldv;

   726 

   727 	@param	a	Address of word to be updated - must be a multiple of 8

   728 	@param	v	The value to be XORed with *a

   729 	@return		The original value of *a

   730 */

   731 EXPORT_C __NAKED__ TUint64	__e32_atomic_xor_acq64(volatile TAny* /*a*/, TUint64 /*v*/)

   732 	{

   733 	_asm jmp __e32_atomic_xor_ord64

   734 	}

   735 

   736 

   737 /** 64 bit atomic bitwise logical exclusive OR, release semantics.

   738 

   739 	Atomically performs the following operation:

   740 		oldv = *a; *a = oldv ^ v; return oldv;

   741 

   742 	@param	a	Address of word to be updated - must be a multiple of 8

   743 	@param	v	The value to be XORed with *a

   744 	@return		The original value of *a

   745 */

   746 EXPORT_C __NAKED__ TUint64	__e32_atomic_xor_rel64(volatile TAny* /*a*/, TUint64 /*v*/)

   747 	{

   748 	_asm jmp __e32_atomic_xor_ord64

   749 	}

   750 

   751 

   752 /** 64 bit atomic bitwise logical exclusive OR, full barrier semantics.

   753 

   754 	Atomically performs the following operation:

   755 		oldv = *a; *a = oldv ^ v; return oldv;

   756 

   757 	@param	a	Address of word to be updated - must be a multiple of 8

   758 	@param	v	The value to be XORed with *a

   759 	@return		The original value of *a

   760 */

   761 EXPORT_C __NAKED__ TUint64	__e32_atomic_xor_ord64(volatile TAny* /*a*/, TUint64 /*v*/)

   762 	{

   763 	_asm push ebx

   764 	_asm push edi

   765 	_asm mov edi, [esp+12]				// edi = a

   766 	_asm mov eax, [edi]					// edx:eax = oldv

   767 	_asm mov edx, [edi+4]

   768 	_asm retry:

   769 	_asm mov ebx, eax

   770 	_asm mov ecx, edx

   771 	_asm xor ebx, [esp+16]				// ecx:ebx = oldv ^ v

   772 	_asm xor ecx, [esp+20]

   773 	_asm __LOCK__ cmpxchg8b [edi]		// if (*a==oldv) *a=oldv^v, ZF=1 else edx:eax=*a, ZF=0

   774 	_asm jne short retry

   775 	_asm pop edi

   776 	_asm pop ebx

   777 	_asm ret

   778 	}

   779 

   780 

   781 /** 64 bit atomic bitwise universal function, relaxed ordering.

   782 

   783 	Atomically performs the following operation:

   784 		oldv = *a; *a = (oldv & u) ^ v; return oldv;

   785 

   786 	@param	a	Address of word to be updated - must be a multiple of 8

   787 	@param	u	The value to be ANDed with *a

   788 	@param	v	The value to be XORed with (*a&u)

   789 	@return		The original value of *a

   790 */

   791 EXPORT_C __NAKED__ TUint64	__e32_atomic_axo_rlx64(volatile TAny* /*a*/, TUint64 /*u*/, TUint64 /*v*/)

   792 	{

   793 	_asm jmp __e32_atomic_axo_ord64

   794 	}

   795 

   796 

   797 /** 64 bit atomic bitwise universal function, acquire semantics.

   798 

   799 	Atomically performs the following operation:

   800 		oldv = *a; *a = (oldv & u) ^ v; return oldv;

   801 

   802 	@param	a	Address of word to be updated - must be a multiple of 8

   803 	@param	u	The value to be ANDed with *a

   804 	@param	v	The value to be XORed with (*a&u)

   805 	@return		The original value of *a

   806 */

   807 EXPORT_C __NAKED__ TUint64	__e32_atomic_axo_acq64(volatile TAny* /*a*/, TUint64 /*u*/, TUint64 /*v*/)

   808 	{

   809 	_asm jmp __e32_atomic_axo_ord64

   810 	}

   811 

   812 

   813 /** 64 bit atomic bitwise universal function, release semantics.

   814 

   815 	Atomically performs the following operation:

   816 		oldv = *a; *a = (oldv & u) ^ v; return oldv;

   817 

   818 	@param	a	Address of word to be updated - must be a multiple of 8

   819 	@param	u	The value to be ANDed with *a

   820 	@param	v	The value to be XORed with (*a&u)

   821 	@return		The original value of *a

   822 */

   823 EXPORT_C __NAKED__ TUint64	__e32_atomic_axo_rel64(volatile TAny* /*a*/, TUint64 /*u*/, TUint64 /*v*/)

   824 	{

   825 	_asm jmp __e32_atomic_axo_ord64

   826 	}

   827 

   828 

   829 /** 64 bit atomic bitwise universal function, release semantics.

   830 

   831 	Atomically performs the following operation:

   832 		oldv = *a; *a = (oldv & u) ^ v; return oldv;

   833 

   834 	@param	a	Address of word to be updated - must be a multiple of 8

   835 	@param	u	The value to be ANDed with *a

   836 	@param	v	The value to be XORed with (*a&u)

   837 	@return		The original value of *a

   838 */

   839 EXPORT_C __NAKED__ TUint64	__e32_atomic_axo_ord64(volatile TAny* /*a*/, TUint64 /*u*/, TUint64 /*v*/)

   840 	{

   841 	_asm push ebx

   842 	_asm push edi

   843 	_asm mov edi, [esp+12]				// edi = a

   844 	_asm mov eax, [edi]					// edx:eax = oldv

   845 	_asm mov edx, [edi+4]

   846 	_asm retry:

   847 	_asm mov ebx, eax

   848 	_asm mov ecx, edx

   849 	_asm and ebx, [esp+16]				// ecx:ebx = oldv & u

   850 	_asm and ecx, [esp+20]

   851 	_asm xor ebx, [esp+24]				// ecx:ebx = (oldv & u) ^ v

   852 	_asm xor ecx, [esp+28]

   853 	_asm __LOCK__ cmpxchg8b [edi]		// if (*a==oldv) *a=(oldv&u)^v, ZF=1 else edx:eax=*a, ZF=0

   854 	_asm jne short retry

   855 	_asm pop edi

   856 	_asm pop ebx

   857 	_asm ret

   858 	}

   859 

   860 

   861 /** 64 bit threshold and add, unsigned, relaxed ordering.

   862 

   863 	Atomically performs the following operation:

   864 		oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv;

   865 

   866 	@param	a	Address of data to be updated - must be naturally aligned

   867 	@param	t	The threshold to compare *a to (unsigned compare)

   868 	@param	u	The value to be added to *a if it is originally >= t

   869 	@param	u	The value to be added to *a if it is originally < t

   870 	@return		The original value of *a

   871 */

   872 EXPORT_C __NAKED__ TUint64	__e32_atomic_tau_rlx64(volatile TAny* /*a*/, TUint64 /*t*/, TUint64 /*u*/, TUint64 /*v*/)

   873 	{

   874 	_asm jmp __e32_atomic_tau_ord64

   875 	}

   876 

   877 

   878 /** 64 bit threshold and add, unsigned, acquire semantics.

   879 

   880 	Atomically performs the following operation:

   881 		oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv;

   882 

   883 	@param	a	Address of data to be updated - must be naturally aligned

   884 	@param	t	The threshold to compare *a to (unsigned compare)

   885 	@param	u	The value to be added to *a if it is originally >= t

   886 	@param	u	The value to be added to *a if it is originally < t

   887 	@return		The original value of *a

   888 */

   889 EXPORT_C __NAKED__ TUint64	__e32_atomic_tau_acq64(volatile TAny* /*a*/, TUint64 /*t*/, TUint64 /*u*/, TUint64 /*v*/)

   890 	{

   891 	_asm jmp __e32_atomic_tau_ord64

   892 	}

   893 

   894 

   895 /** 64 bit threshold and add, unsigned, release semantics.

   896 

   897 	Atomically performs the following operation:

   898 		oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv;

   899 

   900 	@param	a	Address of data to be updated - must be naturally aligned

   901 	@param	t	The threshold to compare *a to (unsigned compare)

   902 	@param	u	The value to be added to *a if it is originally >= t

   903 	@param	u	The value to be added to *a if it is originally < t

   904 	@return		The original value of *a

   905 */

   906 EXPORT_C __NAKED__ TUint64	__e32_atomic_tau_rel64(volatile TAny* /*a*/, TUint64 /*t*/, TUint64 /*u*/, TUint64 /*v*/)

   907 	{

   908 	_asm jmp __e32_atomic_tau_ord64

   909 	}

   910 

   911 

   912 /** 64 bit threshold and add, unsigned, full barrier semantics.

   913 

   914 	Atomically performs the following operation:

   915 		oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv;

   916 

   917 	@param	a	Address of data to be updated - must be naturally aligned

   918 	@param	t	The threshold to compare *a to (unsigned compare)

   919 	@param	u	The value to be added to *a if it is originally >= t

   920 	@param	u	The value to be added to *a if it is originally < t

   921 	@return		The original value of *a

   922 */

   923 EXPORT_C __NAKED__ TUint64	__e32_atomic_tau_ord64(volatile TAny* /*a*/, TUint64 /*t*/, TUint64 /*u*/, TUint64 /*v*/)

   924 	{

   925 	_asm push ebx

   926 	_asm push edi

   927 	_asm mov edi, [esp+12]				// edi = a

   928 	_asm mov eax, [edi]					// edx:eax = oldv

   929 	_asm mov edx, [edi+4]

   930 	_asm retry:

   931 	_asm mov ebx, edx

   932 	_asm cmp eax, [esp+16]				// eax - t.low, CF=borrow

   933 	_asm sbb ebx, [esp+20]				// CF = borrow from (oldv - t)

   934 	_asm jnc short use_u				// no borrow means oldv>=t so use u

   935 	_asm mov ebx, [esp+32]				// ecx:ebx = v

   936 	_asm mov ecx, [esp+36]

   937 	_asm jmp short use_v

   938 	_asm use_u:

   939 	_asm mov ebx, [esp+24]				// ecx:ebx = u

   940 	_asm mov ecx, [esp+28]

   941 	_asm use_v:

   942 	_asm add ebx, eax					// ecx:ebx = oldv + u or v

   943 	_asm adc ecx, edx

   944 	_asm __LOCK__ cmpxchg8b [edi]

   945 	_asm jne short retry

   946 	_asm pop edi

   947 	_asm pop ebx

   948 	_asm ret

   949 	}

   950 

   951 

   952 /** 64 bit threshold and add, signed, relaxed ordering.

   953 

   954 	Atomically performs the following operation:

   955 		oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv;

   956 

   957 	@param	a	Address of data to be updated - must be naturally aligned

   958 	@param	t	The threshold to compare *a to (signed compare)

   959 	@param	u	The value to be added to *a if it is originally >= t

   960 	@param	u	The value to be added to *a if it is originally < t

   961 	@return		The original value of *a

   962 */

   963 EXPORT_C __NAKED__ TInt64	__e32_atomic_tas_rlx64(volatile TAny* /*a*/, TInt64 /*t*/, TInt64 /*u*/, TInt64 /*v*/)

   964 	{

   965 	_asm jmp __e32_atomic_tas_ord64

   966 	}

   967 

   968 

   969 /** 64 bit threshold and add, signed, acquire semantics.

   970 

   971 	Atomically performs the following operation:

   972 		oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv;

   973 

   974 	@param	a	Address of data to be updated - must be naturally aligned

   975 	@param	t	The threshold to compare *a to (signed compare)

   976 	@param	u	The value to be added to *a if it is originally >= t

   977 	@param	u	The value to be added to *a if it is originally < t

   978 	@return		The original value of *a

   979 */

   980 EXPORT_C __NAKED__ TInt64	__e32_atomic_tas_acq64(volatile TAny* /*a*/, TInt64 /*t*/, TInt64 /*u*/, TInt64 /*v*/)

   981 	{

   982 	_asm jmp __e32_atomic_tas_ord64

   983 	}

   984 

   985 

   986 /** 64 bit threshold and add, signed, release semantics.

   987 

   988 	Atomically performs the following operation:

   989 		oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv;

   990 

   991 	@param	a	Address of data to be updated - must be naturally aligned

   992 	@param	t	The threshold to compare *a to (signed compare)

   993 	@param	u	The value to be added to *a if it is originally >= t

   994 	@param	u	The value to be added to *a if it is originally < t

   995 	@return		The original value of *a

   996 */

   997 EXPORT_C __NAKED__ TInt64	__e32_atomic_tas_rel64(volatile TAny* /*a*/, TInt64 /*t*/, TInt64 /*u*/, TInt64 /*v*/)

   998 	{

   999 	_asm jmp __e32_atomic_tas_ord64

  1000 	}

  1001 

  1002 

  1003 /** 64 bit threshold and add, signed, full barrier semantics.

  1004 

  1005 	Atomically performs the following operation:

  1006 		oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv;

  1007 

  1008 	@param	a	Address of data to be updated - must be naturally aligned

  1009 	@param	t	The threshold to compare *a to (signed compare)

  1010 	@param	u	The value to be added to *a if it is originally >= t

  1011 	@param	u	The value to be added to *a if it is originally < t

  1012 	@return		The original value of *a

  1013 */

  1014 EXPORT_C __NAKED__ TInt64	__e32_atomic_tas_ord64(volatile TAny* /*a*/, TInt64 /*t*/, TInt64 /*u*/, TInt64 /*v*/)

  1015 	{

  1016 	_asm push ebx

  1017 	_asm push edi

  1018 	_asm mov edi, [esp+12]				// edi = a

  1019 	_asm mov eax, [edi]					// edx:eax = oldv

  1020 	_asm mov edx, [edi+4]

  1021 	_asm retry:

  1022 	_asm mov ebx, edx

  1023 	_asm cmp eax, [esp+16]				// eax - t.low, CF=borrow

  1024 	_asm sbb ebx, [esp+20]				// SF=sign, OF=overflow from (oldv - t)

  1025 	_asm jge short use_u				// SF==OF (GE condition) means oldv>=t so use u

  1026 	_asm mov ebx, [esp+32]				// ecx:ebx = v

  1027 	_asm mov ecx, [esp+36]

  1028 	_asm jmp short use_v

  1029 	_asm use_u:

  1030 	_asm mov ebx, [esp+24]				// ecx:ebx = u

  1031 	_asm mov ecx, [esp+28]

  1032 	_asm use_v:

  1033 	_asm add ebx, eax					// ecx:ebx = oldv + u or v

  1034 	_asm adc ecx, edx

  1035 	_asm __LOCK__ cmpxchg8b [edi]

  1036 	_asm jne short retry

  1037 	_asm pop edi

  1038 	_asm pop ebx

  1039 	_asm ret

  1040 	}

  1041 

  1042 } // extern "C"