// Copyright (c) 2008-2009 Nokia Corporation and/or its subsidiary(-ies).

// All rights reserved.

// This component and the accompanying materials are made available

// under the terms of the License "Eclipse Public License v1.0"

// which accompanies this distribution, and is available

// at the URL "http://www.eclipse.org/legal/epl-v10.html".

//

// Initial Contributors:

// Nokia Corporation - initial contribution.

//

// Contributors:

//

// Description:

// e32\common\x86\atomics.cia

// 

//

#include <e32atomics.h>

#include <cpudefs.h>

/*

Versions needed:

	WINS/WINSCW		Use X86 locked operations. Assume Pentium or above CPU (CMPXCHG8B available)

	X86				For Pentium and above use locked operations

					For 486 use locked operations for 8, 16, 32 bit. For 64 bit must disable interrupts.

					NOTE: 486 not supported at the moment

	ARMv4/ARMv5		Must disable interrupts.

	ARMv6			LDREX/STREX for 8, 16, 32 bit. For 64 bit must disable interrupts (maybe).

	ARMv6K/ARMv7	LDREXB/LDREXH/LDREX/LDREXD

Need both kernel side and user side versions

*/

#if	defined(__SMP__) || !defined(__EPOC32__)

#define	__BARRIERS_NEEDED__

#define	__LOCK__	"lock "

#else

#define	__LOCK__

#endif

extern "C" {

#undef	__TUintX__

#undef	__TIntX__

#undef	__fname__

#undef	__redir__

#undef	__A_REG__

#undef	__C_REG__

#undef	__D_REG__

#define	__TUintX__		TUint32

#define	__TIntX__		TInt32

#define	__fname__(x)	x##32

#define	__redir__(x)	asm("jmp _"#x "32")

#define	__A_REG__		"eax"

#define	__C_REG__		"ecx"

#define	__D_REG__		"edx"

#include "atomic_skeleton.h"

#undef	__TUintX__

#undef	__TIntX__

#undef	__fname__

#undef	__redir__

#undef	__A_REG__

#undef	__C_REG__

#undef	__D_REG__

#define	__TUintX__		TUint16

#define	__TIntX__		TInt16

#define	__fname__(x)	x##16

#define	__redir__(x)	asm("jmp _"#x "16")

#define	__A_REG__		"ax"

#define	__C_REG__		"cx"

#define	__D_REG__		"dx"

#include "atomic_skeleton.h"

#undef	__TUintX__

#undef	__TIntX__

#undef	__fname__

#undef	__redir__

#undef	__A_REG__

#undef	__C_REG__

#undef	__D_REG__

#define	__TUintX__		TUint8

#define	__TIntX__		TInt8

#define	__fname__(x)	x##8

#define	__redir__(x)	asm("jmp _"#x "8")

#define	__A_REG__		"al"

#define	__C_REG__		"cl"

#define	__D_REG__		"dl"

#include "atomic_skeleton.h"

#undef	__TUintX__

#undef	__TIntX__

#undef	__fname__

#undef	__redir__

#undef	__A_REG__

#undef	__C_REG__

#undef	__D_REG__

/** Full memory barrier for explicit memory accesses

*/

EXPORT_C __NAKED__ void __e32_memory_barrier()

	{

#ifdef __BARRIERS_NEEDED__

	asm("lock add dword ptr [esp], 0 ");

#endif

	asm("ret ");

	}

/** Barrier guaranteeing completion as well as ordering

*/

EXPORT_C __NAKED__ void __e32_io_completion_barrier()

	{

	asm("push ebx ");

	asm("cpuid ");

	asm("pop ebx ");

	asm("ret ");

	}

/** Find the most significant 1 in a 32 bit word

	@param	v	The word to be scanned

	@return		The bit number of the most significant 1 if v != 0

				-1 if v == 0

*/

EXPORT_C __NAKED__ TInt __e32_find_ms1_32(TUint32 /*v*/)

	{

	asm("bsr eax, [esp+4] ");

	asm("jnz short 1f ");

	asm("mov eax, 0xffffffff ");

	asm("1: ");

	asm("ret ");

	}

/** Find the least significant 1 in a 32 bit word

	@param	v	The word to be scanned

	@return		The bit number of the least significant 1 if v != 0

				-1 if v == 0

*/

EXPORT_C __NAKED__ TInt __e32_find_ls1_32(TUint32 /*v*/)

	{

	asm("bsf eax, [esp+4] ");

	asm("jnz short 1f ");

	asm("mov eax, 0xffffffff ");

	asm("1: ");

	asm("ret ");

	}

/** Count the number of 1's in a 32 bit word

	@param	v	The word to be scanned

	@return		The number of 1's

*/

EXPORT_C __NAKED__ TInt __e32_bit_count_32(TUint32 /*v*/)

	{

	asm("mov eax, [esp+4] ");

	asm("mov edx, eax ");

	asm("and eax, 0xaaaaaaaa ");

	asm("and edx, 0x55555555 ");	/* edx = even bits of arg */

	asm("shr eax, 1 ");				/* eax = odd bits of arg shifted into even bits */

	asm("add eax, edx ");			/* eax = 16 groups of 2 bit counts */

	asm("mov edx, eax ");

	asm("and eax, 0xcccccccc ");

	asm("and edx, 0x33333333 ");	/* even groups of 2 */

	asm("shr eax, 2 ");				/* odd groups of 2 shifted to even positions */

	asm("add eax, edx ");			/* 8 groups of 4 bit counts */

	asm("mov edx, eax ");

	asm("shr eax, 4 ");

	asm("add eax, edx ");			/* even nibbles = sum of 8 bits, odd nibbles garbage */

	asm("and eax, 0x0f0f0f0f ");	/* eliminate garbage nibbles */

	asm("add al, ah ");				/* AL = bit count of lower 16 bits */

	asm("mov dl, al ");

	asm("shr eax, 16 ");

	asm("add al, ah ");				/* AL = bit count of upper 16 bits */

	asm("xor ah, ah ");				/* top 24 bits of EAX now zero */

	asm("add al, dl ");				/* AL = bit count of entire 32 bits */

	asm("ret ");

	}

/** Find the most significant 1 in a 64 bit word

	@param	v	The word to be scanned

	@return		The bit number of the most significant 1 if v != 0

				-1 if v == 0

*/

EXPORT_C __NAKED__ TInt __e32_find_ms1_64(TUint64 /*v*/)

	{

	asm("bsr eax, [esp+8] ");

	asm("jnz short 2f ");

	asm("bsr eax, [esp+4] ");

	asm("jnz short 1f ");

	asm("mov eax, 0xffffffff ");

	asm("2: ");

	asm("or eax, 32 ");

	asm("1: ");

	asm("ret ");

	}

/** Find the least significant 1 in a 64 bit word

	@param	v	The word to be scanned

	@return		The bit number of the least significant 1 if v != 0

				-1 if v == 0

*/

EXPORT_C __NAKED__ TInt __e32_find_ls1_64(TUint64 /*v*/)

	{

	asm("bsf eax, [esp+4] ");

	asm("jnz short 1f ");

	asm("bsf eax, [esp+8] ");

	asm("jnz short 2f ");

	asm("mov eax, 0xffffffff ");

	asm("2: ");

	asm("or eax, 32 ");

	asm("1: ");

	asm("ret ");

	}

/** Count the number of 1's in a 64 bit word

	@param	v	The word to be scanned

	@return		The number of 1's

*/

EXPORT_C __NAKED__ TInt __e32_bit_count_64(TUint64 /*v*/)

	{

	asm("mov eax, [esp+4] ");

	asm("mov edx, [esp+8] ");

	asm("mov ecx, eax ");

	asm("and eax, 0xaaaaaaaa ");

	asm("and ecx, 0x55555555 ");

	asm("shr eax, 1 ");

	asm("add eax, ecx ");

	asm("mov ecx, eax ");

	asm("and eax, 0xcccccccc ");

	asm("and ecx, 0x33333333 ");

	asm("shr eax, 2 ");

	asm("add ecx, eax ");

	asm("mov eax, edx ");

	asm("and eax, 0xaaaaaaaa ");

	asm("and edx, 0x55555555 ");

	asm("shr eax, 1 ");

	asm("add eax, edx ");

	asm("mov edx, eax ");

	asm("and eax, 0xcccccccc ");

	asm("and edx, 0x33333333 ");

	asm("shr eax, 2 ");

	asm("add eax, edx ");

	asm("add eax, ecx ");

	asm("mov edx, eax ");

	asm("and eax, 0xf0f0f0f0 ");

	asm("and edx, 0x0f0f0f0f ");

	asm("shr eax, 4 ");

	asm("add eax, edx ");

	asm("add al, ah ");

	asm("mov dl, al ");

	asm("shr eax, 16 ");

	asm("add al, ah ");

	asm("xor ah, ah ");

	asm("add al, dl ");

	asm("ret ");

	}

/** Read a 64 bit word with acquire semantics

	@param	a	Address of word to be read - must be a multiple of 8

	@return		The value read

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_load_acq64(const volatile TAny* /*a*/)

	{

	asm("push ebx ");

	asm("push edi ");

	asm("mov edi, [esp+12] ");

	asm("mov eax, 0x0badbeef ");

	asm("mov edx, eax ");

	asm("mov ebx, eax ");

	asm("mov ecx, eax ");

	asm(__LOCK__ "cmpxchg8b [edi] ");

	asm("pop edi ");

	asm("pop ebx ");

	asm("ret ");

	}

/** Write a 64 bit word with release semantics

	@param	a	Address of word to be written - must be a multiple of 8

	@param	v	The value to be written

	@return		The value written

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_store_rel64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	asm("push ebx ");

	asm("push edi ");

	asm("mov edi, [esp+12] ");

	asm("mov ebx, [esp+16] ");

	asm("mov ecx, [esp+20] ");

	asm("mov eax, [edi] ");

	asm("mov edx, [edi+4] ");

	asm("1: ");

	asm(__LOCK__ "cmpxchg8b [edi] " );

	asm("jne short 1b ");

	asm("mov eax, ebx ");

	asm("mov edx, ecx ");

	asm("pop edi ");

	asm("pop ebx ");

	asm("ret ");

	}

/** Write a 64 bit word with full barrier semantics

	@param	a	Address of word to be written - must be a multiple of 8

	@param	v	The value to be written

	@return		The value written

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_store_ord64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	asm("jmp ___e32_atomic_store_rel64 ");

	}

/** Write a 64 bit word to memory and return the original value of the memory.

	Relaxed ordering.

	@param	a	Address of word to be written - must be a multiple of 8

	@param	v	The value to be written

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_swp_rlx64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	asm("jmp ___e32_atomic_swp_ord64 ");

	}

/** Write a 64 bit word to memory and return the original value of the memory.

	Acquire semantics.

	@param	a	Address of word to be written - must be a multiple of 8

	@param	v	The value to be written

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_swp_acq64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	asm("jmp ___e32_atomic_swp_ord64 ");

	}

/** Write a 64 bit word to memory and return the original value of the memory.

	Release semantics.

	@param	a	Address of word to be written - must be a multiple of 8

	@param	v	The value to be written

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_swp_rel64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	asm("jmp ___e32_atomic_swp_ord64 ");

	}

/** Write a 64 bit word to memory and return the original value of the memory.

	Full barrier semantics.

	@param	a	Address of word to be written - must be a multiple of 8

	@param	v	The value to be written

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_swp_ord64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	asm("push ebx ");

	asm("push edi ");

	asm("mov edi, [esp+12] ");

	asm("mov ebx, [esp+16] ");

	asm("mov ecx, [esp+20] ");

	asm("mov eax, [edi] ");

	asm("mov edx, [edi+4] ");

	asm("1: ");

	asm(__LOCK__ "cmpxchg8b [edi] ");

	asm("jne short 1b ");

	asm("pop edi ");

	asm("pop ebx ");

	asm("ret ");

	}

/** 64 bit compare and swap, relaxed ordering.

	Atomically performs the following operation:

		if (*a == *q)	{ *a = v; return TRUE; }

		else			{ *q = *a; return FALSE; }

	@param	a	Address of word to be written - must be a multiple of 8

	@param	q	Address of location containing expected value

	@param	v	The new value to be written if the old value is as expected

	@return		TRUE if *a was updated, FALSE otherwise

*/

EXPORT_C __NAKED__ TBool		__e32_atomic_cas_rlx64(volatile TAny* /*a*/, TUint64* /*q*/, TUint64 /*v*/)

	{

	asm("jmp ___e32_atomic_cas_ord64 ");

	}

/** 64 bit compare and swap, acquire semantics.

	Atomically performs the following operation:

		if (*a == *q)	{ *a = v; return TRUE; }

		else			{ *q = *a; return FALSE; }

	@param	a	Address of word to be written - must be a multiple of 8

	@param	q	Address of location containing expected value

	@param	v	The new value to be written if the old value is as expected

	@return		TRUE if *a was updated, FALSE otherwise

*/

EXPORT_C __NAKED__ TBool		__e32_atomic_cas_acq64(volatile TAny* /*a*/, TUint64* /*q*/, TUint64 /*v*/)

	{

	asm("jmp ___e32_atomic_cas_ord64 ");

	}

/** 64 bit compare and swap, release semantics.

	Atomically performs the following operation:

		if (*a == *q)	{ *a = v; return TRUE; }

		else			{ *q = *a; return FALSE; }

	@param	a	Address of word to be written - must be a multiple of 8

	@param	q	Address of location containing expected value

	@param	v	The new value to be written if the old value is as expected

	@return		TRUE if *a was updated, FALSE otherwise

*/

EXPORT_C __NAKED__ TBool		__e32_atomic_cas_rel64(volatile TAny* /*a*/, TUint64* /*q*/, TUint64 /*v*/)

	{

	asm("jmp ___e32_atomic_cas_ord64 ");

	}

/** 64 bit compare and swap, full barrier semantics.

	Atomically performs the following operation:

		if (*a == *q)	{ *a = v; return TRUE; }

		else			{ *q = *a; return FALSE; }

	@param	a	Address of word to be written - must be a multiple of 8

	@param	q	Address of location containing expected value

	@param	v	The new value to be written if the old value is as expected

	@return		TRUE if *a was updated, FALSE otherwise

*/

EXPORT_C __NAKED__ TBool		__e32_atomic_cas_ord64(volatile TAny* /*a*/, TUint64* /*q*/, TUint64 /*v*/)

	{

	asm("push ebx ");

	asm("push edi ");

	asm("push esi ");

	asm("mov edi, [esp+16] ");			// edi = a

	asm("mov esi, [esp+20] ");			// esi = q

	asm("mov ebx, [esp+24] ");			// ecx:ebx = v

	asm("mov ecx, [esp+28] ");

	asm("mov eax, [esi] ");				// edx:eax = *q

	asm("mov edx, [esi+4] ");

	asm(__LOCK__ "cmpxchg8b [edi] ");	// if (*a==*q) *a=v, ZF=1 else edx:eax=*a, ZF=0

	asm("jne short 2f ");

	asm("mov eax, 1 ");

	asm("pop esi ");

	asm("pop edi ");

	asm("pop ebx ");

	asm("ret ");

	asm("2: ");

	asm("mov [esi], eax ");				// *q = edx:eax

	asm("mov [esi+4], edx ");

	asm("xor eax, eax ");

	asm("pop esi ");

	asm("pop edi ");

	asm("pop ebx ");

	asm("ret ");

	}

/** 64 bit atomic add, relaxed ordering.

	Atomically performs the following operation:

		oldv = *a; *a = oldv + v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8

	@param	v	The value to be added

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_add_rlx64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	asm("jmp ___e32_atomic_add_ord64 ");

	}

/** 64 bit atomic add, acquire semantics.

	Atomically performs the following operation:

		oldv = *a; *a = oldv + v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8

	@param	v	The value to be added

	@return		The original value of *a

*/

EXPORT_C __NAKED__ TUint64	__e32_atomic_add_acq64(volatile TAny* /*a*/, TUint64 /*v*/)

	{

	asm("jmp ___e32_atomic_add_ord64 ");

	}

/** 64 bit atomic add, release semantics.

	Atomically performs the following operation:

		oldv = *a; *a = oldv + v; return oldv;

	@param	a	Address of word to be updated - must be a multiple of 8

	@param	v	The value to be added