// Copyright (c) 2008-2009 Nokia Corporation and/or its subsidiary(-ies).
// All rights reserved.
// This component and the accompanying materials are made available
// under the terms of the License "Eclipse Public License v1.0"
// which accompanies this distribution, and is available
// at the URL "http://www.eclipse.org/legal/epl-v10.html".
//
// Initial Contributors:
// Nokia Corporation - initial contribution.
//
// Contributors:
//
// Description:
// e32\common\x86\atomics.cpp
// 
//

#include <e32atomics.h>
#include <cpudefs.h>

/*
Versions needed:
	WINS/WINSCW		Use X86 locked operations. Assume Pentium or above CPU (CMPXCHG8B available)
	X86				For Pentium and above use locked operations
					For 486 use locked operations for 8, 16, 32 bit. For 64 bit must disable interrupts.
					NOTE: 486 not supported at the moment
	ARMv4/ARMv5		Must disable interrupts.
	ARMv6			LDREX/STREX for 8, 16, 32 bit. For 64 bit must disable interrupts (maybe).
	ARMv6K/ARMv7	LDREXB/LDREXH/LDREX/LDREXD

Need both kernel side and user side versions
*/

#if	defined(__SMP__) || !defined(__EPOC32__)
#define	__BARRIERS_NEEDED__
#define	__LOCK__	lock
#else
#define	__LOCK__
#endif


extern "C" {

#undef	__TUintX__
#undef	__TIntX__
#undef	__fname__
#undef	__A_REG__
#undef	__C_REG__
#undef	__D_REG__
#define	__TUintX__		TUint32
#define	__TIntX__		TInt32
#define	__fname__(x)	x##32
#define	__A_REG__		eax
#define	__C_REG__		ecx
#define	__D_REG__		edx
#include "atomic_skeleton.h"

#undef	__TUintX__
#undef	__TIntX__
#undef	__fname__
#undef	__A_REG__
#undef	__C_REG__
#undef	__D_REG__
#define	__TUintX__		TUint16
#define	__TIntX__		TInt16
#define	__fname__(x)	x##16
#define	__A_REG__		ax
#define	__C_REG__		cx
#define	__D_REG__		dx
#include "atomic_skeleton.h"

#undef	__TUintX__
#undef	__TIntX__
#undef	__fname__
#undef	__A_REG__
#undef	__C_REG__
#undef	__D_REG__
#define	__TUintX__		TUint8
#define	__TIntX__		TInt8
#define	__fname__(x)	x##8
#define	__A_REG__		al
#define	__C_REG__		cl
#define	__D_REG__		dl
#include "atomic_skeleton.h"

#undef	__TUintX__
#undef	__TIntX__
#undef	__fname__
#undef	__A_REG__
#undef	__C_REG__
#undef	__D_REG__

/** Full memory barrier for explicit memory accesses

*/
EXPORT_C __NAKED__ void __e32_memory_barrier()
	{
#ifdef __BARRIERS_NEEDED__
	_asm lock add dword ptr [esp], 0
#endif
	_asm ret
	}


/** Barrier guaranteeing completion as well as ordering

*/
EXPORT_C __NAKED__ void __e32_io_completion_barrier()
	{
	_asm push ebx
	_asm cpuid
	_asm pop ebx
	_asm ret
	}


/** Find the most significant 1 in a 32 bit word

	@param	v	The word to be scanned
	@return		The bit number of the most significant 1 if v != 0
				-1 if v == 0
*/
EXPORT_C __NAKED__ TInt __e32_find_ms1_32(TUint32 /*v*/)
	{
	_asm bsr eax, [esp+4]
	_asm jnz short done
	_asm mov eax, 0ffffffffh
done:
	_asm ret
	}


/** Find the least significant 1 in a 32 bit word

	@param	v	The word to be scanned
	@return		The bit number of the least significant 1 if v != 0
				-1 if v == 0
*/
EXPORT_C __NAKED__ TInt __e32_find_ls1_32(TUint32 /*v*/)
	{
	_asm bsf eax, [esp+4]
	_asm jnz short done
	_asm mov eax, 0ffffffffh
done:
	_asm ret
	}


/** Count the number of 1's in a 32 bit word

	@param	v	The word to be scanned
	@return		The number of 1's
*/
EXPORT_C __NAKED__ TInt __e32_bit_count_32(TUint32 /*v*/)
	{
	_asm mov eax, [esp+4]
	_asm mov edx, eax
	_asm and eax, 0aaaaaaaah
	_asm and edx, 055555555h
	_asm shr eax, 1
	_asm add eax, edx
	_asm mov edx, eax
	_asm and eax, 0cccccccch
	_asm and edx, 033333333h
	_asm shr eax, 2
	_asm add eax, edx
	_asm mov edx, eax
	_asm shr eax, 4
	_asm add eax, edx
	_asm and eax, 00f0f0f0fh
	_asm add al, ah
	_asm mov dl, al
	_asm shr eax, 16
	_asm add al, ah
	_asm xor ah, ah
	_asm add al, dl
	_asm ret
	}


/** Find the most significant 1 in a 64 bit word

	@param	v	The word to be scanned
	@return		The bit number of the most significant 1 if v != 0
				-1 if v == 0
*/
EXPORT_C __NAKED__ TInt __e32_find_ms1_64(TUint64 /*v*/)
	{
	_asm bsr eax, [esp+8]
	_asm jnz short mswnz
	_asm bsr eax, [esp+4]
	_asm jnz short lswnz
	_asm mov eax, 0ffffffffh
mswnz:
	_asm or eax, 32
lswnz:
	_asm ret
	}


/** Find the least significant 1 in a 64 bit word

	@param	v	The word to be scanned
	@return		The bit number of the least significant 1 if v != 0
				-1 if v == 0
*/
EXPORT_C __NAKED__ TInt __e32_find_ls1_64(TUint64 /*v*/)
	{
	_asm bsf eax, [esp+4]
	_asm jnz short lswnz
	_asm bsf eax, [esp+8]
	_asm jnz short mswnz
	_asm mov eax, 0ffffffffh
mswnz:
	_asm or eax, 32
lswnz:
	_asm ret
	}


/** Count the number of 1's in a 64 bit word

	@param	v	The word to be scanned
	@return		The number of 1's
*/
EXPORT_C __NAKED__ TInt __e32_bit_count_64(TUint64 /*v*/)
	{
	_asm mov eax, [esp+4]
	_asm mov edx, [esp+8]

	_asm mov ecx, eax
	_asm and eax, 0aaaaaaaah
	_asm and ecx, 055555555h
	_asm shr eax, 1
	_asm add eax, ecx			/* 16 groups of 2 bits, count=0,1,2 */
	_asm mov ecx, eax
	_asm and eax, 0cccccccch
	_asm and ecx, 033333333h
	_asm shr eax, 2
	_asm add ecx, eax			/* 8 groups of 4 bits, count=0...4 */

	_asm mov eax, edx
	_asm and eax, 0aaaaaaaah
	_asm and edx, 055555555h
	_asm shr eax, 1
	_asm add eax, edx			/* 16 groups of 2 bits, count=0,1,2 */
	_asm mov edx, eax
	_asm and eax, 0cccccccch
	_asm and edx, 033333333h
	_asm shr eax, 2
	_asm add eax, edx			/* 8 groups of 4 bits, count=0...4 */

	_asm add eax, ecx			/* 8 groups of 4 bits, count=0...8 */
	_asm mov edx, eax
	_asm and eax, 0f0f0f0f0h
	_asm and edx, 00f0f0f0fh
	_asm shr eax, 4
	_asm add eax, edx			/* 4 groups of 8 bits, count=0...16 */
	_asm add al, ah
	_asm mov dl, al
	_asm shr eax, 16
	_asm add al, ah
	_asm xor ah, ah
	_asm add al, dl
	_asm ret
	}




/** Read a 64 bit word with acquire semantics

	@param	a	Address of word to be read - must be a multiple of 8
	@return		The value read
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_load_acq64(const volatile TAny* /*a*/)
	{
	_asm push ebx
	_asm push edi
	_asm mov edi, [esp+12]
	_asm mov eax, 0badbeefh
	_asm mov edx, eax
	_asm mov ebx, eax
	_asm mov ecx, eax
	_asm __LOCK__ cmpxchg8b [edi]
	_asm pop edi
	_asm pop ebx
	_asm ret
	}


/** Write a 64 bit word with release semantics

	@param	a	Address of word to be written - must be a multiple of 8
	@param	v	The value to be written
	@return		The value written
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_store_rel64(volatile TAny* /*a*/, TUint64 /*v*/)
	{
	_asm push ebx
	_asm push edi
	_asm mov edi, [esp+12]
	_asm mov ebx, [esp+16]
	_asm mov ecx, [esp+20]
	_asm mov eax, [edi]
	_asm mov edx, [edi+4]
	_asm retry:
	_asm __LOCK__ cmpxchg8b [edi]
	_asm jne short retry
	_asm mov eax, ebx
	_asm mov edx, ecx
	_asm pop edi
	_asm pop ebx
	_asm ret
	}


/** Write a 64 bit word with full barrier semantics

	@param	a	Address of word to be written - must be a multiple of 8
	@param	v	The value to be written
	@return		The value written
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_store_ord64(volatile TAny* /*a*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_store_rel64
	}


/** Write a 64 bit word to memory and return the original value of the memory.
	Relaxed ordering.

	@param	a	Address of word to be written - must be a multiple of 8
	@param	v	The value to be written
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_swp_rlx64(volatile TAny* /*a*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_swp_ord64
	}


/** Write a 64 bit word to memory and return the original value of the memory.
	Acquire semantics.

	@param	a	Address of word to be written - must be a multiple of 8
	@param	v	The value to be written
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_swp_acq64(volatile TAny* /*a*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_swp_ord64
	}


/** Write a 64 bit word to memory and return the original value of the memory.
	Release semantics.

	@param	a	Address of word to be written - must be a multiple of 8
	@param	v	The value to be written
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_swp_rel64(volatile TAny* /*a*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_swp_ord64
	}


/** Write a 64 bit word to memory and return the original value of the memory.
	Full barrier semantics.

	@param	a	Address of word to be written - must be a multiple of 8
	@param	v	The value to be written
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_swp_ord64(volatile TAny* /*a*/, TUint64 /*v*/)
	{
	_asm push ebx
	_asm push edi
	_asm mov edi, [esp+12]
	_asm mov ebx, [esp+16]
	_asm mov ecx, [esp+20]
	_asm mov eax, [edi]
	_asm mov edx, [edi+4]
	_asm retry:
	_asm __LOCK__ cmpxchg8b [edi]
	_asm jne short retry
	_asm pop edi
	_asm pop ebx
	_asm ret
	}


/** 64 bit compare and swap, relaxed ordering.

	Atomically performs the following operation:
		if (*a == *q)	{ *a = v; return TRUE; }
		else			{ *q = *a; return FALSE; }

	@param	a	Address of word to be written - must be a multiple of 8
	@param	q	Address of location containing expected value
	@param	v	The new value to be written if the old value is as expected
	@return		TRUE if *a was updated, FALSE otherwise
*/
EXPORT_C __NAKED__ TBool		__e32_atomic_cas_rlx64(volatile TAny* /*a*/, TUint64* /*q*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_cas_ord64
	}


/** 64 bit compare and swap, acquire semantics.

	Atomically performs the following operation:
		if (*a == *q)	{ *a = v; return TRUE; }
		else			{ *q = *a; return FALSE; }

	@param	a	Address of word to be written - must be a multiple of 8
	@param	q	Address of location containing expected value
	@param	v	The new value to be written if the old value is as expected
	@return		TRUE if *a was updated, FALSE otherwise
*/
EXPORT_C __NAKED__ TBool		__e32_atomic_cas_acq64(volatile TAny* /*a*/, TUint64* /*q*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_cas_ord64
	}


/** 64 bit compare and swap, release semantics.

	Atomically performs the following operation:
		if (*a == *q)	{ *a = v; return TRUE; }
		else			{ *q = *a; return FALSE; }

	@param	a	Address of word to be written - must be a multiple of 8
	@param	q	Address of location containing expected value
	@param	v	The new value to be written if the old value is as expected
	@return		TRUE if *a was updated, FALSE otherwise
*/
EXPORT_C __NAKED__ TBool		__e32_atomic_cas_rel64(volatile TAny* /*a*/, TUint64* /*q*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_cas_ord64
	}


/** 64 bit compare and swap, full barrier semantics.

	Atomically performs the following operation:
		if (*a == *q)	{ *a = v; return TRUE; }
		else			{ *q = *a; return FALSE; }

	@param	a	Address of word to be written - must be a multiple of 8
	@param	q	Address of location containing expected value
	@param	v	The new value to be written if the old value is as expected
	@return		TRUE if *a was updated, FALSE otherwise
*/
EXPORT_C __NAKED__ TBool		__e32_atomic_cas_ord64(volatile TAny* /*a*/, TUint64* /*q*/, TUint64 /*v*/)
	{
	_asm push ebx
	_asm push edi
	_asm push esi
	_asm mov edi, [esp+16]				// edi = a
	_asm mov esi, [esp+20]				// esi = q
	_asm mov ebx, [esp+24]				// ecx:ebx = v
	_asm mov ecx, [esp+28]
	_asm mov eax, [esi]					// edx:eax = *q
	_asm mov edx, [esi+4]
	_asm __LOCK__ cmpxchg8b [edi]		// if (*a==*q) *a=v, ZF=1 else edx:eax=*a, ZF=0
	_asm jne short cas_fail
	_asm mov eax, 1
	_asm pop esi
	_asm pop edi
	_asm pop ebx
	_asm ret
	_asm cas_fail:
	_asm mov [esi], eax					// *q = edx:eax
	_asm mov [esi+4], edx
	_asm xor eax, eax
	_asm pop esi
	_asm pop edi
	_asm pop ebx
	_asm ret
	}


/** 64 bit atomic add, relaxed ordering.

	Atomically performs the following operation:
		oldv = *a; *a = oldv + v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8
	@param	v	The value to be added
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_add_rlx64(volatile TAny* /*a*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_add_ord64
	}


/** 64 bit atomic add, acquire semantics.

	Atomically performs the following operation:
		oldv = *a; *a = oldv + v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8
	@param	v	The value to be added
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_add_acq64(volatile TAny* /*a*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_add_ord64
	}


/** 64 bit atomic add, release semantics.

	Atomically performs the following operation:
		oldv = *a; *a = oldv + v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8
	@param	v	The value to be added
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_add_rel64(volatile TAny* /*a*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_add_ord64
	}


/** 64 bit atomic add, full barrier semantics.

	Atomically performs the following operation:
		oldv = *a; *a = oldv + v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8
	@param	v	The value to be added
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_add_ord64(volatile TAny* /*a*/, TUint64 /*v*/)
	{
	_asm push ebx
	_asm push edi
	_asm mov edi, [esp+12]				// edi = a
	_asm mov eax, [edi]					// edx:eax = oldv
	_asm mov edx, [edi+4]
	_asm retry:
	_asm mov ebx, eax
	_asm mov ecx, edx
	_asm add ebx, [esp+16]				// ecx:ebx = oldv + v
	_asm adc ecx, [esp+20]
	_asm __LOCK__ cmpxchg8b [edi]		// if (*a==oldv) *a=oldv+v, ZF=1 else edx:eax=*a, ZF=0
	_asm jne short retry
	_asm pop edi
	_asm pop ebx
	_asm ret
	}


/** 64 bit atomic bitwise logical AND, relaxed ordering.

	Atomically performs the following operation:
		oldv = *a; *a = oldv & v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8
	@param	v	The value to be ANDed with *a
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_and_rlx64(volatile TAny* /*a*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_and_ord64
	}


/** 64 bit atomic bitwise logical AND, acquire semantics.

	Atomically performs the following operation:
		oldv = *a; *a = oldv & v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8
	@param	v	The value to be ANDed with *a
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_and_acq64(volatile TAny* /*a*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_and_ord64
	}


/** 64 bit atomic bitwise logical AND, release semantics.

	Atomically performs the following operation:
		oldv = *a; *a = oldv & v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8
	@param	v	The value to be ANDed with *a
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_and_rel64(volatile TAny* /*a*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_and_ord64
	}


/** 64 bit atomic bitwise logical AND, full barrier semantics.

	Atomically performs the following operation:
		oldv = *a; *a = oldv & v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8
	@param	v	The value to be ANDed with *a
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_and_ord64(volatile TAny* /*a*/, TUint64 /*v*/)
	{
	_asm push ebx
	_asm push edi
	_asm mov edi, [esp+12]				// edi = a
	_asm mov eax, [edi]					// edx:eax = oldv
	_asm mov edx, [edi+4]
	_asm retry:
	_asm mov ebx, eax
	_asm mov ecx, edx
	_asm and ebx, [esp+16]				// ecx:ebx = oldv & v
	_asm and ecx, [esp+20]
	_asm __LOCK__ cmpxchg8b [edi]		// if (*a==oldv) *a=oldv&v, ZF=1 else edx:eax=*a, ZF=0
	_asm jne short retry
	_asm pop edi
	_asm pop ebx
	_asm ret
	}


/** 64 bit atomic bitwise logical inclusive OR, relaxed ordering.

	Atomically performs the following operation:
		oldv = *a; *a = oldv | v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8
	@param	v	The value to be ORed with *a
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_ior_rlx64(volatile TAny* /*a*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_ior_ord64
	}


/** 64 bit atomic bitwise logical inclusive OR, acquire semantics.

	Atomically performs the following operation:
		oldv = *a; *a = oldv | v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8
	@param	v	The value to be ORed with *a
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_ior_acq64(volatile TAny* /*a*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_ior_ord64
	}


/** 64 bit atomic bitwise logical inclusive OR, release semantics.

	Atomically performs the following operation:
		oldv = *a; *a = oldv | v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8
	@param	v	The value to be ORed with *a
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_ior_rel64(volatile TAny* /*a*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_ior_ord64
	}


/** 64 bit atomic bitwise logical inclusive OR, full barrier semantics.

	Atomically performs the following operation:
		oldv = *a; *a = oldv | v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8
	@param	v	The value to be ORed with *a
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_ior_ord64(volatile TAny* /*a*/, TUint64 /*v*/)
	{
	_asm push ebx
	_asm push edi
	_asm mov edi, [esp+12]				// edi = a
	_asm mov eax, [edi]					// edx:eax = oldv
	_asm mov edx, [edi+4]
	_asm retry:
	_asm mov ebx, eax
	_asm mov ecx, edx
	_asm or ebx, [esp+16]				// ecx:ebx = oldv | v
	_asm or ecx, [esp+20]
	_asm __LOCK__ cmpxchg8b [edi]		// if (*a==oldv) *a=oldv|v, ZF=1 else edx:eax=*a, ZF=0
	_asm jne short retry
	_asm pop edi
	_asm pop ebx
	_asm ret
	}


/** 64 bit atomic bitwise logical exclusive OR, relaxed ordering.

	Atomically performs the following operation:
		oldv = *a; *a = oldv ^ v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8
	@param	v	The value to be XORed with *a
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_xor_rlx64(volatile TAny* /*a*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_xor_ord64
	}


/** 64 bit atomic bitwise logical exclusive OR, acquire semantics.

	Atomically performs the following operation:
		oldv = *a; *a = oldv ^ v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8
	@param	v	The value to be XORed with *a
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_xor_acq64(volatile TAny* /*a*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_xor_ord64
	}


/** 64 bit atomic bitwise logical exclusive OR, release semantics.

	Atomically performs the following operation:
		oldv = *a; *a = oldv ^ v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8
	@param	v	The value to be XORed with *a
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_xor_rel64(volatile TAny* /*a*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_xor_ord64
	}


/** 64 bit atomic bitwise logical exclusive OR, full barrier semantics.

	Atomically performs the following operation:
		oldv = *a; *a = oldv ^ v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8
	@param	v	The value to be XORed with *a
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_xor_ord64(volatile TAny* /*a*/, TUint64 /*v*/)
	{
	_asm push ebx
	_asm push edi
	_asm mov edi, [esp+12]				// edi = a
	_asm mov eax, [edi]					// edx:eax = oldv
	_asm mov edx, [edi+4]
	_asm retry:
	_asm mov ebx, eax
	_asm mov ecx, edx
	_asm xor ebx, [esp+16]				// ecx:ebx = oldv ^ v
	_asm xor ecx, [esp+20]
	_asm __LOCK__ cmpxchg8b [edi]		// if (*a==oldv) *a=oldv^v, ZF=1 else edx:eax=*a, ZF=0
	_asm jne short retry
	_asm pop edi
	_asm pop ebx
	_asm ret
	}


/** 64 bit atomic bitwise universal function, relaxed ordering.

	Atomically performs the following operation:
		oldv = *a; *a = (oldv & u) ^ v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8
	@param	u	The value to be ANDed with *a
	@param	v	The value to be XORed with (*a&u)
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_axo_rlx64(volatile TAny* /*a*/, TUint64 /*u*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_axo_ord64
	}


/** 64 bit atomic bitwise universal function, acquire semantics.

	Atomically performs the following operation:
		oldv = *a; *a = (oldv & u) ^ v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8
	@param	u	The value to be ANDed with *a
	@param	v	The value to be XORed with (*a&u)
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_axo_acq64(volatile TAny* /*a*/, TUint64 /*u*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_axo_ord64
	}


/** 64 bit atomic bitwise universal function, release semantics.

	Atomically performs the following operation:
		oldv = *a; *a = (oldv & u) ^ v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8
	@param	u	The value to be ANDed with *a
	@param	v	The value to be XORed with (*a&u)
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_axo_rel64(volatile TAny* /*a*/, TUint64 /*u*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_axo_ord64
	}


/** 64 bit atomic bitwise universal function, release semantics.

	Atomically performs the following operation:
		oldv = *a; *a = (oldv & u) ^ v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8
	@param	u	The value to be ANDed with *a
	@param	v	The value to be XORed with (*a&u)
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_axo_ord64(volatile TAny* /*a*/, TUint64 /*u*/, TUint64 /*v*/)
	{
	_asm push ebx
	_asm push edi
	_asm mov edi, [esp+12]				// edi = a
	_asm mov eax, [edi]					// edx:eax = oldv
	_asm mov edx, [edi+4]
	_asm retry:
	_asm mov ebx, eax
	_asm mov ecx, edx
	_asm and ebx, [esp+16]				// ecx:ebx = oldv & u
	_asm and ecx, [esp+20]
	_asm xor ebx, [esp+24]				// ecx:ebx = (oldv & u) ^ v
	_asm xor ecx, [esp+28]
	_asm __LOCK__ cmpxchg8b [edi]		// if (*a==oldv) *a=(oldv&u)^v, ZF=1 else edx:eax=*a, ZF=0
	_asm jne short retry
	_asm pop edi
	_asm pop ebx
	_asm ret
	}


/** 64 bit threshold and add, unsigned, relaxed ordering.

	Atomically performs the following operation:
		oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv;

	@param	a	Address of data to be updated - must be naturally aligned
	@param	t	The threshold to compare *a to (unsigned compare)
	@param	u	The value to be added to *a if it is originally >= t
	@param	u	The value to be added to *a if it is originally < t
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_tau_rlx64(volatile TAny* /*a*/, TUint64 /*t*/, TUint64 /*u*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_tau_ord64
	}


/** 64 bit threshold and add, unsigned, acquire semantics.

	Atomically performs the following operation:
		oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv;

	@param	a	Address of data to be updated - must be naturally aligned
	@param	t	The threshold to compare *a to (unsigned compare)
	@param	u	The value to be added to *a if it is originally >= t
	@param	u	The value to be added to *a if it is originally < t
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_tau_acq64(volatile TAny* /*a*/, TUint64 /*t*/, TUint64 /*u*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_tau_ord64
	}


/** 64 bit threshold and add, unsigned, release semantics.

	Atomically performs the following operation:
		oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv;

	@param	a	Address of data to be updated - must be naturally aligned
	@param	t	The threshold to compare *a to (unsigned compare)
	@param	u	The value to be added to *a if it is originally >= t
	@param	u	The value to be added to *a if it is originally < t
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_tau_rel64(volatile TAny* /*a*/, TUint64 /*t*/, TUint64 /*u*/, TUint64 /*v*/)
	{
	_asm jmp __e32_atomic_tau_ord64
	}


/** 64 bit threshold and add, unsigned, full barrier semantics.

	Atomically performs the following operation:
		oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv;

	@param	a	Address of data to be updated - must be naturally aligned
	@param	t	The threshold to compare *a to (unsigned compare)
	@param	u	The value to be added to *a if it is originally >= t
	@param	u	The value to be added to *a if it is originally < t
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TUint64	__e32_atomic_tau_ord64(volatile TAny* /*a*/, TUint64 /*t*/, TUint64 /*u*/, TUint64 /*v*/)
	{
	_asm push ebx
	_asm push edi
	_asm mov edi, [esp+12]				// edi = a
	_asm mov eax, [edi]					// edx:eax = oldv
	_asm mov edx, [edi+4]
	_asm retry:
	_asm mov ebx, edx
	_asm cmp eax, [esp+16]				// eax - t.low, CF=borrow
	_asm sbb ebx, [esp+20]				// CF = borrow from (oldv - t)
	_asm jnc short use_u				// no borrow means oldv>=t so use u
	_asm mov ebx, [esp+32]				// ecx:ebx = v
	_asm mov ecx, [esp+36]
	_asm jmp short use_v
	_asm use_u:
	_asm mov ebx, [esp+24]				// ecx:ebx = u
	_asm mov ecx, [esp+28]
	_asm use_v:
	_asm add ebx, eax					// ecx:ebx = oldv + u or v
	_asm adc ecx, edx
	_asm __LOCK__ cmpxchg8b [edi]
	_asm jne short retry
	_asm pop edi
	_asm pop ebx
	_asm ret
	}


/** 64 bit threshold and add, signed, relaxed ordering.

	Atomically performs the following operation:
		oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv;

	@param	a	Address of data to be updated - must be naturally aligned
	@param	t	The threshold to compare *a to (signed compare)
	@param	u	The value to be added to *a if it is originally >= t
	@param	u	The value to be added to *a if it is originally < t
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TInt64	__e32_atomic_tas_rlx64(volatile TAny* /*a*/, TInt64 /*t*/, TInt64 /*u*/, TInt64 /*v*/)
	{
	_asm jmp __e32_atomic_tas_ord64
	}


/** 64 bit threshold and add, signed, acquire semantics.

	Atomically performs the following operation:
		oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv;

	@param	a	Address of data to be updated - must be naturally aligned
	@param	t	The threshold to compare *a to (signed compare)
	@param	u	The value to be added to *a if it is originally >= t
	@param	u	The value to be added to *a if it is originally < t
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TInt64	__e32_atomic_tas_acq64(volatile TAny* /*a*/, TInt64 /*t*/, TInt64 /*u*/, TInt64 /*v*/)
	{
	_asm jmp __e32_atomic_tas_ord64
	}


/** 64 bit threshold and add, signed, release semantics.

	Atomically performs the following operation:
		oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv;

	@param	a	Address of data to be updated - must be naturally aligned
	@param	t	The threshold to compare *a to (signed compare)
	@param	u	The value to be added to *a if it is originally >= t
	@param	u	The value to be added to *a if it is originally < t
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TInt64	__e32_atomic_tas_rel64(volatile TAny* /*a*/, TInt64 /*t*/, TInt64 /*u*/, TInt64 /*v*/)
	{
	_asm jmp __e32_atomic_tas_ord64
	}


/** 64 bit threshold and add, signed, full barrier semantics.

	Atomically performs the following operation:
		oldv = *a; if (oldv>=t) *a=oldv+u else *a=oldv+v; return oldv;

	@param	a	Address of data to be updated - must be naturally aligned
	@param	t	The threshold to compare *a to (signed compare)
	@param	u	The value to be added to *a if it is originally >= t
	@param	u	The value to be added to *a if it is originally < t
	@return		The original value of *a
*/
EXPORT_C __NAKED__ TInt64	__e32_atomic_tas_ord64(volatile TAny* /*a*/, TInt64 /*t*/, TInt64 /*u*/, TInt64 /*v*/)
	{
	_asm push ebx
	_asm push edi
	_asm mov edi, [esp+12]				// edi = a
	_asm mov eax, [edi]					// edx:eax = oldv
	_asm mov edx, [edi+4]
	_asm retry:
	_asm mov ebx, edx
	_asm cmp eax, [esp+16]				// eax - t.low, CF=borrow
	_asm sbb ebx, [esp+20]				// SF=sign, OF=overflow from (oldv - t)
	_asm jge short use_u				// SF==OF (GE condition) means oldv>=t so use u
	_asm mov ebx, [esp+32]				// ecx:ebx = v
	_asm mov ecx, [esp+36]
	_asm jmp short use_v
	_asm use_u:
	_asm mov ebx, [esp+24]				// ecx:ebx = u
	_asm mov ecx, [esp+28]
	_asm use_v:
	_asm add ebx, eax					// ecx:ebx = oldv + u or v
	_asm adc ecx, edx
	_asm __LOCK__ cmpxchg8b [edi]
	_asm jne short retry
	_asm pop edi
	_asm pop ebx
	_asm ret
	}

} // extern "C"