// Copyright (c) 2007-2009 Nokia Corporation and/or its subsidiary(-ies).
// All rights reserved.
// This component and the accompanying materials are made available
// under the terms of the License "Eclipse Public License v1.0"
// which accompanies this distribution, and is available
// at the URL "http://www.eclipse.org/legal/epl-v10.html".
//
// Initial Contributors:
// Nokia Corporation - initial contribution.
//
// Contributors:
//
// Description:
// e32\nkernsmp\x86\ncutilf.cia
// 
//

#include <x86.h>
#include <apic.h>

#if defined(__VC32__)
#define	__ASM_CALL(func)	_asm call func
#elif defined(__GCC32__)
#define	__ASM_CALL(func)	asm("call _" #func);
#else
#error Unknown x86 compiler
#endif

#if defined(__INCLUDE_SPIN_LOCK_CHECKS__)
#define	SPIN_LOCK_ENTRY_CHECK()			__ASM_CALL(spin_lock_entry_check)
#define	SPIN_LOCK_MARK_ACQ()			__ASM_CALL(spin_lock_mark_acq)
#define	SPIN_UNLOCK_ENTRY_CHECK()		__ASM_CALL(spin_unlock_entry_check)

#define	RWSPIN_RLOCK_ENTRY_CHECK()		__ASM_CALL(rwspin_rlock_entry_check)
#define	RWSPIN_RLOCK_MARK_ACQ()			__ASM_CALL(rwspin_rlock_mark_acq)
#define	RWSPIN_RUNLOCK_ENTRY_CHECK()	__ASM_CALL(rwspin_runlock_entry_check)

#define	RWSPIN_WLOCK_ENTRY_CHECK()		__ASM_CALL(rwspin_wlock_entry_check)
#define	RWSPIN_WLOCK_MARK_ACQ()			__ASM_CALL(rwspin_wlock_mark_acq)
#define	RWSPIN_WUNLOCK_ENTRY_CHECK()	__ASM_CALL(rwspin_wunlock_entry_check)

#else
#define	SPIN_LOCK_ENTRY_CHECK()
#define	SPIN_LOCK_MARK_ACQ()
#define	SPIN_UNLOCK_ENTRY_CHECK()

#define	RWSPIN_RLOCK_ENTRY_CHECK()
#define	RWSPIN_RLOCK_MARK_ACQ()
#define	RWSPIN_RUNLOCK_ENTRY_CHECK()

#define	RWSPIN_WLOCK_ENTRY_CHECK()
#define	RWSPIN_WLOCK_MARK_ACQ()
#define	RWSPIN_WUNLOCK_ENTRY_CHECK()

#endif


/******************************************************************************
 * Timestamp
 ******************************************************************************/

/** Returns a timestamp value which is consistent across CPUs.

*/
EXPORT_C __NAKED__ TUint64 NKern::Timestamp()
	{
	asm("pushfd ");
	asm("cli ");		// stop thread migration between reading APIC ID and thread pointer
	asm("mov ecx, ds:[%0]" : : "i" (X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID));
	asm("shr ecx, 24 ");
	asm("mov ecx, [ecx*4+%0]" : : "i" (&SubSchedulerLookupTable));
	asm("cmp ecx, 0 ");
	asm("jz short use_tsc_only ");
	asm("test cl, 3 ");
	asm("jnz short use_tsc_only ");
	asm("rdtsc ");
	asm("lea ecx, [ecx+%0]" : : "i" _FOFF(TSubScheduler, iSSX.iTimestampOffset));
	asm("add eax, [ecx] ");
	asm("adc edx, [ecx+4] ");
	asm("popfd ");
	asm("ret ");

	asm("use_tsc_only: ");
	asm("rdtsc ");
	asm("popfd ");
	asm("ret ");
	}

/** Get the current value of the CPU timestamp counter

*/
EXPORT_C __NAKED__ TUint64 X86::Timestamp()
	{
	asm("rdtsc ");
	asm("ret ");
	}



/******************************************************************************
 * Spin locks
 *
 * [this+0]		in count (byte)
 * [this+1]		out count (byte)
 * [this+6]		order (byte)
 * [this+7]		holding CPU (byte)
 ******************************************************************************/

#if defined(__INCLUDE_SPIN_LOCK_CHECKS__)
extern "C" __NAKED__ void spin_lock_entry_check()
	{
	/* ecx points to lock */
	asm("push eax ");
	asm("push ecx ");
	asm("push edx ");
	asm("pushfd ");
	asm("cli ");
	asm("mov edx, ds:[%0]" : : "i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID));
	asm("shr edx, 24");
	asm("mov edx, [edx*4+%0]" : : "i" (&SubSchedulerLookupTable));
	asm("cmp edx, 0 ");						/* Skip checks if subschedulers not yet initialised */
	asm("je short slec_ok ");
	asm("test edx, 3 ");					/* Skip checks if subscheduler for this CPU not yet initialised */
	asm("jnz short slec_ok ");
	asm("movzx ecx, word ptr [ecx+6] ");	/* CL = order, CH = holding CPU */
	asm("cmp cl, 0x20 ");
	asm("jae short slec_preemption ");		/* This lock requires preemption to be disabled */

	/* check interrupts disabled */
	asm("test dword ptr [esp], 0x200 ");	/* Interrupts enabled? */
	asm("jz short slec_1 ");				/* No - OK */
	asm("int 0xff ");						/* Yes - die */

	asm("slec_preemption: ");
	asm("cmp cl, 0xff ");
	asm("je short slec_1 ");			/* EOrderNone - don't check interrupts or preemption */
	asm("cmp dword ptr [edx+%0], 0" : : "i"_FOFF(TSubScheduler, iSSX.iIrqNestCount));
	asm("jge short slec_preemption_die ");	/* If called from ISR, die */
	asm("cmp dword ptr [edx+%0], 0" : : "i" _FOFF(TSubScheduler, iKernLockCount));
	asm("jnz short slec_1 ");			/* Preemption disabled - OK */
	asm("slec_preemption_die: ");
	asm("int 0xff ");					/* Preemption enabled - die */

	asm("slec_1: ");
	asm("lea eax, [edx+%0]" : : "i" _FOFF(TSubScheduler, iCpuNum));
	asm("cmp ch, [eax] ");
	asm("jnz short slec_2 ");			/* Not already held by this CPU - OK */
	asm("int 0xff ");					/* Already held by this CPU - die */

	asm("slec_2: ");
	asm("lea edx, [edx+%0]" : : "i" _FOFF(TSubScheduler, iSpinLockOrderCheck));
	asm("bsf eax, [edx] ");				/* find LSB of low dword */
	asm("jnz short slec_3 ");			/* skip if low dword nonzero */
	asm("bsf eax, [edx+4] ");			/* else find LSB of high dword */
	asm("lea eax, [eax+32] ");			/* add 32 to eax without changing flags */
	asm("jnz short slec_3 ");			/* skip if high dword nonzero */
	asm("mov eax, 0x7f ");				/* else set EAX = 0x7F */

	asm("slec_3: ");
	asm("cmp cl, al ");					/* check order of this lock against lowest currently held order */
	asm("jl short slec_ok ");			/* if this lock has lower order, OK - signed comparison so EOrderNone always works */
	asm("int 0xff ");					/* ordering violation - die */

	asm("slec_ok: ");
	asm("popfd ");
	asm("pop edx ");
	asm("pop ecx ");
	asm("pop eax ");
	asm("ret ");
	}

extern "C" __NAKED__ void spin_lock_mark_acq()
	{
	/* ecx points to lock */
	asm("push eax ");
	asm("push ecx ");
	asm("push edx ");
	asm("pushfd ");
	asm("cli ");
	asm("mov edx, ds:[%0]" : : "i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID));
	asm("shr edx, 24");
	asm("mov edx, [edx*4+%0]" : : "i"(&SubSchedulerLookupTable));
	asm("cmp edx, 0 ");						/* Skip checks if subschedulers not yet initialised */
	asm("je short slma_ok ");
	asm("test edx, 3 ");					/* Skip checks if subscheduler for this CPU not yet initialised */
	asm("jnz short slma_ok ");
	asm("mov eax, [edx+%0]" : : "i" _FOFF(TSubScheduler, iCpuNum));
	asm("mov [ecx+7], al ");				/* set byte 7 to holding CPU number */
	asm("movzx ecx, byte ptr [ecx+6] ");	/* CL = order */
	asm("cmp ecx, 0x40 ");
	asm("jae short slma_ok ");				/* if EOrderNone, done */
	asm("bts [edx+%0], ecx" : : "i" _FOFF(TSubScheduler, iSpinLockOrderCheck));

	asm("slma_ok: ");
	asm("popfd ");
	asm("pop edx ");
	asm("pop ecx ");
	asm("pop eax ");
	asm("ret ");
	}

extern "C" __NAKED__ void spin_unlock_entry_check()
	{
	/* ecx points to lock */
	asm("push eax ");
	asm("push ecx ");
	asm("push edx ");
	asm("pushfd ");
	asm("cli ");
	asm("mov edx, ds:[%0]" : : "i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID));
	asm("shr edx, 24");
	asm("mov edx, [edx*4+%0]" : : "i"(&SubSchedulerLookupTable));
	asm("cmp edx, 0 ");						/* Skip checks if subschedulers not yet initialised */
	asm("je short suec_ok ");
	asm("test edx, 3 ");					/* Skip checks if subscheduler for this CPU not yet initialised */
	asm("jnz short suec_ok ");
	asm("mov eax, [edx+%0]" : : "i" _FOFF(TSubScheduler, iCpuNum));		/* eax = current CPU number */
	asm("shl eax, 8 ");						/* AL = 0, AH = current CPU number */
	asm("xor ax, [ecx+6] ");				/* AL = order, AH = holding CPU ^ current CPU number */
	asm("cmp al, 0x20 ");
	asm("jae short suec_preemption ");		/* This lock requires preemption to be disabled */

	/* check interrupts disabled */
	asm("test dword ptr [esp], 0x200 ");	/* Interrupts enabled? */
	asm("jz short suec_1 ");				/* No - OK */
	asm("int 0xff ");						/* Yes - die */

	asm("suec_preemption: ");
	asm("cmp al, 0xff ");
	asm("je short suec_1 ");			/* EOrderNone - don't check interrupts or preemption */
	asm("cmp dword ptr [edx+%0], 0" : : "i" _FOFF(TSubScheduler, iKernLockCount));
	asm("jnz short suec_1 ");			/* Preemption disabled - OK */
	asm("int 0xff ");					/* Preemption enabled - die */

	asm("suec_1: ");
	asm("cmp ah, 0 ");					/* Check if holding CPU ^ current CPU number == 0 */
	asm("jz short suec_2 ");			/* Already held by this CPU - OK */
	asm("int 0xff ");					/* We don't hold lock - die */

	asm("suec_2: ");
	asm("mov byte ptr [ecx+7], 0xff ");	/* reset holding CPU */
	asm("cmp eax, 0x40 ");				/* EAX = lock order */
	asm("jae short suec_ok ");			/* if EOrderNone, done */
	asm("btr [edx+%0], eax" : : "i" _FOFF(TSubScheduler, iSpinLockOrderCheck));
	asm("jc short suec_ok ");			/* bit should have been set originally */
	asm("int 0xff ");					/* if not, die - something must have got corrupted */

	asm("suec_ok: ");
	asm("popfd ");
	asm("pop edx ");
	asm("pop ecx ");
	asm("pop eax ");
	asm("ret ");
	}
#endif


/******************************************************************************
 * Plain old spin lock
 *
 * Fundamental algorithm:
 *	lock()		{ old_in = in++; while(out!=old_in) __chill(); }
 *	unlock()	{ ++out; }
 *
 * [this+0]		in count (byte)
 * [this+1]		out count (byte)
 * [this+6]		order value
 * [this+7]		holding CPU number, 0xFF if none
 *
 ******************************************************************************/
__NAKED__ EXPORT_C void TSpinLock::LockIrq()
	{
	THISCALL_PROLOG0()
	asm("cli ");
	SPIN_LOCK_ENTRY_CHECK()
	asm("mov al, 1 ");
	asm("lock xadd [ecx], al ");			/* al = in++ */
	asm("sl_lockirq_loop: ");
	asm("cmp al, [ecx+1] ");				/* compare al to out */
	asm("jnz short sl_lockirq_loop2 ");
	SPIN_LOCK_MARK_ACQ()
	asm("lock add dword ptr [esp], 0 ");	/* make sure subsequent accesses don't happen until lock acquired */
	THISCALL_EPILOG0()

	asm("sl_lockirq_loop2: ");
	X86_PAUSE
	asm("jmp short sl_lockirq_loop ");
	}

__NAKED__ EXPORT_C void TSpinLock::UnlockIrq()
	{
	THISCALL_PROLOG0()
	SPIN_UNLOCK_ENTRY_CHECK()
	asm("lock inc byte ptr [ecx+1] ");		/* ++out */
	asm("sti ");
	THISCALL_EPILOG0()
	}

extern "C" TBool __fastcall spin_lock_flash_irq(TSpinLock* a)
	{
	a->UnlockIrq();
	a->LockIrq();
	return TRUE;
	}

__NAKED__ EXPORT_C TBool TSpinLock::FlashIrq()
	{
	THISCALL_PROLOG0()
	asm("mov ax, [ecx] ");
	asm("inc ah ");
	asm("xor al, ah ");
	asm("and eax, 0xff ");
	asm("jne %a0" : : "i" (&spin_lock_flash_irq));
	THISCALL_EPILOG0()
	}

__NAKED__ EXPORT_C void TSpinLock::LockOnly()
	{
	THISCALL_PROLOG0()
	SPIN_LOCK_ENTRY_CHECK()
	asm("mov al, 1 ");
	asm("lock xadd [ecx], al ");			/* al = in++ */
	asm("sl_lockonly_loop: ");
	asm("cmp al, [ecx+1] ");				/* compare al to out */
	asm("jnz short sl_lockonly_loop2 ");
	SPIN_LOCK_MARK_ACQ()
	asm("lock add dword ptr [esp], 0 ");	/* make sure subsequent accesses don't happen until lock acquired */
	THISCALL_EPILOG0()

	asm("sl_lockonly_loop2: ");
	X86_PAUSE
	asm("jmp short sl_lockonly_loop ");
	}

__NAKED__ EXPORT_C void TSpinLock::UnlockOnly()
	{
	THISCALL_PROLOG0()
	SPIN_UNLOCK_ENTRY_CHECK()
	asm("lock inc byte ptr [ecx+1] ");		/* ++out */
	THISCALL_EPILOG0()
	}

extern "C" TBool __fastcall spin_lock_flash_only(TSpinLock* a)
	{
	a->UnlockOnly();
	a->LockOnly();
	return TRUE;
	}

__NAKED__ EXPORT_C TBool TSpinLock::FlashOnly()
	{
	THISCALL_PROLOG0()
	asm("mov ax, [ecx] ");
	asm("inc ah ");
	asm("xor al, ah ");
	asm("and eax, 0xff ");
	asm("jne %a0" : : "i" (&spin_lock_flash_only));
	THISCALL_EPILOG0()
	}

__NAKED__ EXPORT_C TInt TSpinLock::LockIrqSave()
	{
	THISCALL_PROLOG0()
	asm("pushfd ");
	asm("cli ");
	SPIN_LOCK_ENTRY_CHECK()
	asm("mov al, 1 ");
	asm("lock xadd [ecx], al ");			/* al = in++ */
	asm("sl_lockirqs_loop: ");
	asm("cmp al, [ecx+1] ");				/* compare al to out */
	asm("jnz short sl_lockirqs_loop2 ");
	SPIN_LOCK_MARK_ACQ()
	asm("lock add dword ptr [esp], 0 ");	/* make sure subsequent accesses don't happen until lock acquired */
	asm("pop eax ");						/* retrieve saved EFLAGS */
	asm("and eax, 0x200 ");					/* return just interrupt mask bit */
	THISCALL_EPILOG0()

	asm("sl_lockirqs_loop2: ");
	X86_PAUSE
	asm("jmp short sl_lockirqs_loop ");
	}

__NAKED__ EXPORT_C void TSpinLock::UnlockIrqRestore(TInt)
	{
	THISCALL_PROLOG1()
	SPIN_UNLOCK_ENTRY_CHECK()
	asm("lock inc byte ptr [ecx+1] ");		/* ++out */
	asm("test dword ptr [esp+4], 0x200 ");
	asm("jz short sl_unlockirqr_1 ");
	asm("sti ");
	asm("sl_unlockirqr_1: ");
	THISCALL_EPILOG1()
	}

__NAKED__ EXPORT_C TBool TSpinLock::FlashIrqRestore(TInt)
	{
	/* don't mess with stacked args, yet */
	THISCALL_PROLOG0()
	asm("mov ax, [ecx] ");
	asm("inc ah ");
	asm("xor al, ah ");
	asm("and eax, 0xff ");
	asm("jne short sl_flashirqr_1 ");

	/* now we can remove stacked arg since we don't need it */
	THISCALL_EPILOG1()

	asm("sl_flashirqr_1: ");
	THISCALL_PROLOG1()
	asm("test dword ptr [esp+4], 0x200 ");
	asm("jnz short sl_flashirqr_2 ");
	asm("call %a0" : : "i" (&spin_lock_flash_only));
	asm("jmp short sl_flashirqr_3 ");
	asm("sl_flashirqr_2: ");
	asm("call %a0" : : "i" (&spin_lock_flash_irq));
	asm("sl_flashirqr_3: ");
	THISCALL_EPILOG1()
	}

extern "C" TBool __fastcall spin_lock_flash_preempt(TSpinLock* a)
	{
	a->UnlockOnly();
	NKern::PreemptionPoint();
	a->LockOnly();
	return TRUE;
	}

__NAKED__ EXPORT_C TBool TSpinLock::FlashPreempt()
	{
	THISCALL_PROLOG0()
	asm("mov ax, [ecx] ");
	asm("inc ah ");
	asm("xor al, ah ");
	asm("and eax, 0xff ");
	asm("jne %a0" : : "i" (&spin_lock_flash_preempt));
	THISCALL_EPILOG0()
	}


/******************************************************************************
 * Read/Write Spin lock
 *
 * Structure ( (in.r,in.w) , (out.r,out.w) )
 * Fundamental algorithm:
 *	lockr()		{ old_in = (in.r++,in.w); while(out.w!=old_in.w) __chill(); }
 *	unlockr()	{ ++out.r; }
 *	lockw()		{ old_in = (in.r,in.w++); while(out!=old_in) __chill(); }
 *	unlockw()	{ ++out.w; }
 *
 * [this+0]		in.w
 * [this+1]		in.r
 * [this+2]		out.w
 * [this+3]		out.r
 * [this+4]		Bit mask of CPUs which hold read locks
 * [this+6]		order value
 * [this+7]		CPU number which holds write lock, 0xFF if none
 *
 ******************************************************************************/

#if defined(__INCLUDE_SPIN_LOCK_CHECKS__)
extern "C" __NAKED__ void rwspin_rlock_entry_check()
	{
	/* ecx points to lock */
	asm("push eax ");
	asm("push ecx ");
	asm("push edx ");
	asm("pushfd ");
	asm("cli ");
	asm("mov edx, ds:[%0]" : : "i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID));
	asm("shr edx, 24");
	asm("mov edx, [edx*4+%0]" : : "i" (&SubSchedulerLookupTable));
	asm("cmp edx, 0 ");						/* Skip checks if subschedulers not yet initialised */
	asm("je short rwrlec_ok ");
	asm("test edx, 3 ");					/* Skip checks if subscheduler for this CPU not yet initialised */
	asm("jnz short rwrlec_ok ");
	asm("movzx ecx, word ptr [ecx+6] ");	/* CL = order, CH = holding CPU for write lock */
	asm("cmp cl, 0x20 ");
	asm("jae short rwrlec_preemption ");		/* This lock requires preemption to be disabled */

	/* check interrupts disabled */
	asm("test dword ptr [esp], 0x200 ");	/* Interrupts enabled? */
	asm("jz short rwrlec_1 ");				/* No - OK */
	asm("int 0xff ");						/* Yes - die */

	asm("rwrlec_preemption: ");
	asm("cmp cl, 0xff ");
	asm("je short rwrlec_1 ");			/* EOrderNone - don't check interrupts or preemption */
	asm("cmp dword ptr [edx+%0], 0" : : "i"_FOFF(TSubScheduler, iSSX.iIrqNestCount));
	asm("jge short rwrlec_preemption_die ");	/* If called from ISR, die */
	asm("cmp dword ptr [edx+%0], 0" : : "i" _FOFF(TSubScheduler, iKernLockCount));
	asm("jnz short rwrlec_1 ");			/* Preemption disabled - OK */
	asm("rwrlec_preemption_die: ");
	asm("int 0xff ");					/* Preemption enabled - die */

	asm("rwrlec_1: ");
	asm("lea eax, [edx+%0]" : : "i" _FOFF(TSubScheduler, iCpuNum));
	asm("cmp ch, [eax] ");
	asm("jnz short rwrlec_2 ");			/* Not already held by this CPU for write - OK */
	asm("int 0xff ");					/* Already held by this CPU for write - die */

	asm("rwrlec_2: ");
	asm("mov eax, [edx+%0]" : : "i" _FOFF(TSubScheduler, iCpuMask));
	asm("test al, [ecx+4] ");			/* Test if already held by this CPU for read */
	asm("jz short rwrlec_3 ");
	asm("int 0xff ");					/* if so, die */

	asm("rwrlec_3: ");
	asm("lea edx, [edx+%0]" : : "i" _FOFF(TSubScheduler, iSpinLockOrderCheck));
	asm("bsf eax, [edx] ");				/* find LSB of low dword */
	asm("jnz short rwrlec_3 ");			/* skip if low dword nonzero */
	asm("bsf eax, [edx+4] ");			/* else find LSB of high dword */
	asm("lea eax, [eax+32] ");			/* add 32 to eax without changing flags */
	asm("jnz short rwrlec_4 ");			/* skip if high dword nonzero */
	asm("mov eax, 0x7f ");				/* else set EAX = 0x7F */

	asm("rwrlec_4: ");
	asm("cmp cl, al ");					/* check order of this lock against lowest currently held order */
	asm("jl short rwrlec_ok ");			/* if this lock has lower order, OK - signed comparison so EOrderNone always works */
	asm("int 0xff ");					/* ordering violation - die */

	asm("rwrlec_ok: ");
	asm("popfd ");
	asm("pop edx ");
	asm("pop ecx ");
	asm("pop eax ");
	asm("ret ");
	}

extern "C" __NAKED__ void rwspin_rlock_mark_acq()
	{
	/* ecx points to lock */
	asm("push eax ");
	asm("push ecx ");
	asm("push edx ");
	asm("pushfd ");
	asm("cli ");
	asm("mov edx, ds:[%0]" : : "i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID));
	asm("shr edx, 24");
	asm("mov edx, [edx*4+%0]" : : "i"(&SubSchedulerLookupTable));
	asm("cmp edx, 0 ");						/* Skip checks if subschedulers not yet initialised */
	asm("je short rwrlma_ok ");
	asm("test edx, 3 ");					/* Skip checks if subscheduler for this CPU not yet initialised */
	asm("jnz short rwrlma_ok ");
	asm("mov eax, [edx+%0]" : : "i" _FOFF(TSubScheduler, iCpuMask));
	asm("lock or [ecx+4], al ");			/* set bit in byte 4 corresponding to this CPU */
	asm("movzx ecx, byte ptr [ecx+6] ");	/* CL = order */
	asm("cmp ecx, 0x40 ");
	asm("jae short rwrlma_ok ");			/* if EOrderNone, done */
	asm("bts [edx+%0], ecx" : : "i" _FOFF(TSubScheduler, iSpinLockOrderCheck));

	asm("rwrlma_ok: ");
	asm("popfd ");
	asm("pop edx ");
	asm("pop ecx ");
	asm("pop eax ");
	asm("ret ");
	}

extern "C" __NAKED__ void rwspin_runlock_entry_check()
	{
	/* ecx points to lock */
	asm("push eax ");
	asm("push ebx ");
	asm("push ecx ");
	asm("push edx ");
	asm("pushfd ");
	asm("cli ");
	asm("mov edx, ds:[%0]" : : "i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID));
	asm("shr edx, 24");
	asm("mov edx, [edx*4+%0]" : : "i"(&SubSchedulerLookupTable));
	asm("cmp edx, 0 ");						/* Skip checks if subschedulers not yet initialised */
	asm("je short rwruec_ok ");
	asm("test edx, 3 ");					/* Skip checks if subscheduler for this CPU not yet initialised */
	asm("jnz short rwruec_ok ");
	asm("mov eax, [ecx+4] ");				/* AL = R-mask, EAX byte 2 = order */
	asm("and eax, 0x00ffffff ");			/* mask out W CPU */
	asm("cmp eax, 0x00200000 ");
	asm("jae short rwruec_preemption ");	/* This lock requires preemption to be disabled */

	/* check interrupts disabled */
	asm("test dword ptr [esp], 0x200 ");	/* Interrupts enabled? */
	asm("jz short rwruec_1 ");				/* No - OK */
	asm("int 0xff ");						/* Yes - die */

	asm("rwruec_preemption: ");
	asm("cmp eax, 0x00ff0000 ");
	asm("jae short rwruec_1 ");			/* EOrderNone - don't check interrupts or preemption */
	asm("cmp dword ptr [edx+%0], 0" : : "i" _FOFF(TSubScheduler, iKernLockCount));
	asm("jnz short rwruec_1 ");			/* Preemption disabled - OK */
	asm("int 0xff ");					/* Preemption enabled - die */

	asm("rwruec_1: ");
	asm("mov ebx, [edx+%0]" : : "i" _FOFF(TSubScheduler, iCpuMask));
	asm("test al, bl ");				/* Check if current CPU holds read lock */
	asm("jnz short rwruec_2 ");			/* Already held by this CPU - OK */
	asm("int 0xff ");					/* We don't hold lock - die */

	asm("rwruec_2: ");
	asm("not bl ");
	asm("lock and [ecx+4], bl ");		/* clear bit in R-holding CPU mask */
	asm("shr eax, 16 ");				/* EAX = lock order */
	asm("cmp eax, 0x40 ");
	asm("jae short rwruec_ok ");		/* if EOrderNone, done */
	asm("btr [edx+%0], eax" : : "i" _FOFF(TSubScheduler, iSpinLockOrderCheck));
	asm("jc short rwruec_ok ");			/* bit should have been set originally */
	asm("int 0xff ");					/* if not, die - something must have got corrupted */

	asm("rwruec_ok: ");
	asm("popfd ");
	asm("pop edx ");
	asm("pop ecx ");
	asm("pop ebx ");
	asm("pop eax ");
	asm("ret ");
	}


extern "C" __NAKED__ void rwspin_wlock_entry_check()
	{
	/* ecx points to lock */
	asm("push eax ");
	asm("push ecx ");
	asm("push edx ");
	asm("pushfd ");
	asm("cli ");
	asm("mov edx, ds:[%0]" : : "i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID));
	asm("shr edx, 24");
	asm("mov edx, [edx*4+%0]" : : "i" (&SubSchedulerLookupTable));
	asm("cmp edx, 0 ");						/* Skip checks if subschedulers not yet initialised */
	asm("je short rwwlec_ok ");
	asm("test edx, 3 ");					/* Skip checks if subscheduler for this CPU not yet initialised */
	asm("jnz short rwwlec_ok ");
	asm("movzx ecx, word ptr [ecx+6] ");	/* CL = order, CH = write lock holding CPU */
	asm("cmp cl, 0x20 ");
	asm("jae short rwwlec_preemption ");	/* This lock requires preemption to be disabled */

	/* check interrupts disabled */
	asm("test dword ptr [esp], 0x200 ");	/* Interrupts enabled? */
	asm("jz short rwwlec_1 ");				/* No - OK */
	asm("int 0xff ");						/* Yes - die */

	asm("rwwlec_preemption: ");
	asm("cmp cl, 0xff ");
	asm("je short rwwlec_1 ");			/* EOrderNone - don't check interrupts or preemption */
	asm("cmp dword ptr [edx+%0], 0" : : "i"_FOFF(TSubScheduler, iSSX.iIrqNestCount));
	asm("jge short rwwlec_preemption_die ");	/* If called from ISR, die */
	asm("cmp dword ptr [edx+%0], 0" : : "i" _FOFF(TSubScheduler, iKernLockCount));
	asm("jnz short rwwlec_1 ");			/* Preemption disabled - OK */
	asm("rwwlec_preemption_die: ");
	asm("int 0xff ");					/* Preemption enabled - die */

	asm("rwwlec_1: ");
	asm("lea eax, [edx+%0]" : : "i" _FOFF(TSubScheduler, iCpuNum));
	asm("cmp ch, [eax] ");
	asm("jnz short rwwlec_2 ");			/* Not already held by this CPU for write - OK */
	asm("int 0xff ");					/* Already held by this CPU for write - die */

	asm("rwwlec_2: ");
	asm("mov eax, [edx+%0]" : : "i" _FOFF(TSubScheduler, iCpuMask));
	asm("test al, [ecx+4] ");			/* Test if already held by this CPU for read */
	asm("jz short rwwlec_3 ");
	asm("int 0xff ");					/* if so, die */

	asm("rwwlec_3: ");
	asm("lea edx, [edx+%0]" : : "i" _FOFF(TSubScheduler, iSpinLockOrderCheck));
	asm("bsf eax, [edx] ");				/* find LSB of low dword */
	asm("jnz short rwwlec_4 ");			/* skip if low dword nonzero */
	asm("bsf eax, [edx+4] ");			/* else find LSB of high dword */
	asm("lea eax, [eax+32] ");			/* add 32 to eax without changing flags */
	asm("jnz short rwwlec_4 ");			/* skip if high dword nonzero */
	asm("mov eax, 0x7f ");				/* else set EAX = 0x7F */

	asm("rwwlec_4: ");
	asm("cmp cl, al ");					/* check order of this lock against lowest currently held order */
	asm("jl short rwwlec_ok ");			/* if this lock has lower order, OK - signed comparison so EOrderNone always works */
	asm("int 0xff ");					/* ordering violation - die */

	asm("rwwlec_ok: ");
	asm("popfd ");
	asm("pop edx ");
	asm("pop ecx ");
	asm("pop eax ");
	asm("ret ");
	}

extern "C" __NAKED__ void rwspin_wlock_mark_acq()
	{
	/* ecx points to lock */
	asm("push eax ");
	asm("push ecx ");
	asm("push edx ");
	asm("pushfd ");
	asm("cli ");
	asm("mov edx, ds:[%0]" : : "i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID));
	asm("shr edx, 24");
	asm("mov edx, [edx*4+%0]" : : "i"(&SubSchedulerLookupTable));
	asm("cmp edx, 0 ");						/* Skip checks if subschedulers not yet initialised */
	asm("je short rwwlma_ok ");
	asm("test edx, 3 ");					/* Skip checks if subscheduler for this CPU not yet initialised */
	asm("jnz short rwwlma_ok ");
	asm("mov eax, [edx+%0]" : : "i" _FOFF(TSubScheduler, iCpuNum));
	asm("mov [ecx+7], al ");				/* set byte 7 to holding CPU number */
	asm("movzx ecx, byte ptr [ecx+6] ");	/* CL = order */
	asm("cmp ecx, 0x40 ");
	asm("jae short rwwlma_ok ");				/* if EOrderNone, done */
	asm("bts [edx+%0], ecx" : : "i" _FOFF(TSubScheduler, iSpinLockOrderCheck));

	asm("rwwlma_ok: ");
	asm("popfd ");
	asm("pop edx ");
	asm("pop ecx ");
	asm("pop eax ");
	asm("ret ");
	}

extern "C" __NAKED__ void rwspin_wunlock_entry_check()
	{
	/* ecx points to lock */
	asm("push eax ");
	asm("push ecx ");
	asm("push edx ");
	asm("pushfd ");
	asm("cli ");
	asm("mov edx, ds:[%0]" : : "i"(X86_LOCAL_APIC_BASE + X86_LOCAL_APIC_OFFSET_ID));
	asm("shr edx, 24");
	asm("mov edx, [edx*4+%0]" : : "i"(&SubSchedulerLookupTable));
	asm("cmp edx, 0 ");						/* Skip checks if subschedulers not yet initialised */
	asm("je short rwwuec_ok ");
	asm("test edx, 3 ");					/* Skip checks if subscheduler for this CPU not yet initialised */
	asm("jnz short rwwuec_ok ");
	asm("mov eax, [edx+%0]" : : "i" _FOFF(TSubScheduler, iCpuNum));		/* eax = current CPU number */
	asm("shl eax, 8 ");						/* AL = 0, AH = current CPU number */
	asm("xor ax, [ecx+6] ");				/* AL = order, AH = holding CPU ^ current CPU number */
	asm("cmp al, 0x20 ");
	asm("jae short rwwuec_preemption ");		/* This lock requires preemption to be disabled */

	/* check interrupts disabled */
	asm("test dword ptr [esp], 0x200 ");	/* Interrupts enabled? */
	asm("jz short rwwuec_1 ");				/* No - OK */
	asm("int 0xff ");						/* Yes - die */

	asm("rwwuec_preemption: ");
	asm("cmp al, 0xff ");
	asm("je short rwwuec_1 ");			/* EOrderNone - don't check interrupts or preemption */
	asm("cmp dword ptr [edx+%0], 0" : : "i" _FOFF(TSubScheduler, iKernLockCount));
	asm("jnz short rwwuec_1 ");			/* Preemption disabled - OK */
	asm("int 0xff ");					/* Preemption enabled - die */

	asm("rwwuec_1: ");
	asm("cmp ah, 0 ");					/* Check if holding CPU ^ current CPU number == 0 */
	asm("jz short rwwuec_2 ");			/* Already held by this CPU - OK */
	asm("int 0xff ");					/* We don't hold lock - die */

	asm("rwwuec_2: ");
	asm("mov byte ptr [ecx+7], 0xff ");	/* reset holding CPU */
	asm("cmp eax, 0x40 ");				/* EAX = lock order */
	asm("jae short rwwuec_ok ");			/* if EOrderNone, done */
	asm("btr [edx+%0], eax" : : "i" _FOFF(TSubScheduler, iSpinLockOrderCheck));
	asm("jc short rwwuec_ok ");			/* bit should have been set originally */
	asm("int 0xff ");					/* if not, die - something must have got corrupted */

	asm("rwwuec_ok: ");
	asm("popfd ");
	asm("pop edx ");
	asm("pop ecx ");
	asm("pop eax ");
	asm("ret ");
	}
#endif


/*-----------------------------------------------------------------------------
 - Read locks disabling IRQ
 -----------------------------------------------------------------------------*/
__NAKED__ EXPORT_C void TRWSpinLock::LockIrqR()
	{
	THISCALL_PROLOG0()
	asm("cli ");
	RWSPIN_RLOCK_ENTRY_CHECK()
	asm("mov ax, 0x100 ");
	asm("lock xadd [ecx], ax ");			/* ah = in.r++, al = in.w */
	asm("rwl_rlockirq_loop: ");
	asm("cmp al, [ecx+2] ");				/* compare al to out.w */
	asm("jnz short rwl_rlockirq_loop2 ");
	RWSPIN_RLOCK_MARK_ACQ()
	asm("lock add dword ptr [esp], 0 ");	/* make sure subsequent accesses don't happen until lock acquired */
	THISCALL_EPILOG0()

	asm("rwl_rlockirq_loop2: ");
	X86_PAUSE
	asm("jmp short rwl_rlockirq_loop ");
	}

__NAKED__ EXPORT_C void TRWSpinLock::UnlockIrqR()
	{
	THISCALL_PROLOG0()
	RWSPIN_RUNLOCK_ENTRY_CHECK()
	asm("lock add word ptr [ecx+2], 0x100 ");	/* ++out.r */
	asm("sti ");
	THISCALL_EPILOG0()
	}

extern "C" TBool __fastcall rwspin_rlock_flash_irq(TRWSpinLock* a)
	{
	a->UnlockIrqR();
	a->LockIrqR();
	return TRUE;
	}

__NAKED__ EXPORT_C TBool TRWSpinLock::FlashIrqR()
	{
	THISCALL_PROLOG0()
	asm("mov eax, [ecx] ");		/* al=in.w, ah=in.r, byte2=out.w, byte3=out.r */
	asm("mov edx, eax ");
	asm("shr edx, 16 ");		/* dl=out.w */
	asm("xor eax, edx ");		/* al = in.w ^ out.w = 0 if no writers waiting */
	asm("and eax, 0xff ");
	asm("jne %a0" : : "i" (&rwspin_rlock_flash_irq));
	THISCALL_EPILOG0()
	}


/*-----------------------------------------------------------------------------
 - Write locks disabling IRQ
 -----------------------------------------------------------------------------*/
__NAKED__ EXPORT_C void TRWSpinLock::LockIrqW()
	{
	THISCALL_PROLOG0()
	asm("cli ");
	RWSPIN_WLOCK_ENTRY_CHECK()
	asm("mov ax, [ecx] ");					/* ah = in.r, al = in.w */
	asm("rwl_wlockirq_loop3: ");
	asm("mov edx, eax ");
	asm("inc dl ");							/* dh = in.r, dl = in.w+1 */
	asm("lock cmpxchg [ecx], dx ");			/* attempt to update in.w */
	asm("jne short rwl_wlockirq_loop3 ");	/* loop if failed */
	asm("rwl_wlockirq_loop: ");
	asm("cmp ax, [ecx+2] ");				/* compare ax to (out.w,out.r) */
	asm("jnz short rwl_wlockirq_loop2 ");
	RWSPIN_WLOCK_MARK_ACQ()
	asm("lock add dword ptr [esp], 0 ");	/* make sure subsequent accesses don't happen until lock acquired */
	THISCALL_EPILOG0()

	asm("rwl_wlockirq_loop2: ");
	X86_PAUSE
	asm("jmp short rwl_wlockirq_loop ");
	}

__NAKED__ EXPORT_C void TRWSpinLock::UnlockIrqW()
	{
	THISCALL_PROLOG0()
	RWSPIN_WUNLOCK_ENTRY_CHECK()
	asm("mov ax, [ecx+2] ");				/* ah = out.r, al = out.w */
	asm("rwl_wunlockirq_loop: ");
	asm("mov edx, eax ");
	asm("inc dl ");							/* dh = out.r, dl = out.w+1 */
	asm("lock cmpxchg [ecx+2], dx ");		/* attempt to update out.w */
	asm("jne short rwl_wunlockirq_loop ");	/* loop if failed */
	asm("sti ");
	THISCALL_EPILOG0()
	}

extern "C" TBool __fastcall rwspin_wlock_flash_irq(TRWSpinLock* a)
	{
	a->UnlockIrqW();
	a->LockIrqW();
	return TRUE;
	}

__NAKED__ EXPORT_C TBool TRWSpinLock::FlashIrqW()
	{
	THISCALL_PROLOG0()
	asm("mov eax, [ecx] ");		/* al=in.w, ah=in.r, byte2=out.w, byte3=out.r */
	asm("mov edx, eax ");
	asm("shr edx, 16 ");		/* dl=out.w, dh=out.r */
	asm("inc dl ");				/* dx==ax now means no-one else is waiting for lock */
	asm("xor eax, edx ");
	asm("and eax, 0xffff ");
	asm("jne %a0" : : "i" (&rwspin_wlock_flash_irq));
	THISCALL_EPILOG0()
	}



/*-----------------------------------------------------------------------------
 - Read locks leaving IRQ alone
 -----------------------------------------------------------------------------*/
__NAKED__ EXPORT_C void TRWSpinLock::LockOnlyR()
	{
	THISCALL_PROLOG0()
	RWSPIN_RLOCK_ENTRY_CHECK()
	asm("mov ax, 0x100 ");
	asm("lock xadd [ecx], ax ");			/* ah = in.r++, al = in.w */
	asm("rwl_rlockonly_loop: ");
	asm("cmp al, [ecx+2] ");				/* compare al to out.w */
	asm("jnz short rwl_rlockonly_loop2 ");
	RWSPIN_RLOCK_MARK_ACQ()
	asm("lock add dword ptr [esp], 0 ");	/* make sure subsequent accesses don't happen until lock acquired */
	THISCALL_EPILOG0()

	asm("rwl_rlockonly_loop2: ");
	X86_PAUSE
	asm("jmp short rwl_rlockonly_loop ");
	}

__NAKED__ EXPORT_C void TRWSpinLock::UnlockOnlyR()
	{
	THISCALL_PROLOG0()
	RWSPIN_RUNLOCK_ENTRY_CHECK()
	asm("lock add word ptr [ecx+2], 0x100 ");	/* ++out.r */
	THISCALL_EPILOG0()
	}

extern "C" TBool __fastcall rwspin_rlock_flash_only(TRWSpinLock* a)
	{
	a->UnlockOnlyR();
	a->LockOnlyR();
	return TRUE;
	}

__NAKED__ EXPORT_C TBool TRWSpinLock::FlashOnlyR()
	{
	THISCALL_PROLOG0()
	asm("mov eax, [ecx] ");		/* al=in.w, ah=in.r, byte2=out.w, byte3=out.r */
	asm("mov edx, eax ");
	asm("shr edx, 16 ");		/* dl=out.w */
	asm("xor eax, edx ");		/* al = in.w ^ out.w = 0 if no writers waiting */
	asm("and eax, 0xff ");
	asm("jne %a0" : : "i" (&rwspin_rlock_flash_only));
	THISCALL_EPILOG0()
	}


/*-----------------------------------------------------------------------------
 - Write locks leaving IRQ alone
 -----------------------------------------------------------------------------*/
__NAKED__ EXPORT_C void TRWSpinLock::LockOnlyW()
	{
	THISCALL_PROLOG0()
	RWSPIN_WLOCK_ENTRY_CHECK()
	asm("mov ax, [ecx] ");					/* ah = in.r, al = in.w */
	asm("rwl_wlockonly_loop3: ");
	asm("mov edx, eax ");
	asm("inc dl ");							/* dh = in.r, dl = in.w+1 */
	asm("lock cmpxchg [ecx], dx ");			/* attempt to update in.w */
	asm("jne short rwl_wlockonly_loop3 ");	/* loop if failed */
	asm("rwl_wlockonly_loop: ");
	asm("cmp ax, [ecx+2] ");				/* compare ax to (out.w,out.r) */
	asm("jnz short rwl_wlockonly_loop2 ");
	RWSPIN_WLOCK_MARK_ACQ()
	asm("lock add dword ptr [esp], 0 ");	/* make sure subsequent accesses don't happen until lock acquired */
	THISCALL_EPILOG0()

	asm("rwl_wlockonly_loop2: ");
	X86_PAUSE
	asm("jmp short rwl_wlockonly_loop ");
	}

__NAKED__ EXPORT_C void TRWSpinLock::UnlockOnlyW()
	{
	THISCALL_PROLOG0()
	RWSPIN_WUNLOCK_ENTRY_CHECK()
	asm("mov ax, [ecx+2] ");				/* ah = out.r, al = out.w */
	asm("rwl_wunlockonly_loop: ");
	asm("mov edx, eax ");
	asm("inc dl ");							/* dh = out.r, dl = out.w+1 */
	asm("lock cmpxchg [ecx+2], dx ");		/* attempt to update out.w */
	asm("jne short rwl_wunlockonly_loop ");	/* loop if failed */
	THISCALL_EPILOG0()
	}

extern "C" TBool __fastcall rwspin_wlock_flash_only(TRWSpinLock* a)
	{
	a->UnlockOnlyW();
	a->LockOnlyW();
	return TRUE;
	}

__NAKED__ EXPORT_C TBool TRWSpinLock::FlashOnlyW()
	{
	THISCALL_PROLOG0()
	asm("mov eax, [ecx] ");		/* al=in.w, ah=in.r, byte2=out.w, byte3=out.r */
	asm("mov edx, eax ");
	asm("shr edx, 16 ");		/* dl=out.w, dh=out.r */
	asm("inc dl ");				/* dx==ax now means no-one else is waiting for lock */
	asm("xor eax, edx ");
	asm("and eax, 0xffff ");
	asm("jne %a0" : : "i" (&rwspin_wlock_flash_only));
	THISCALL_EPILOG0()
	}



/*-----------------------------------------------------------------------------
 - Read locks disabling IRQ with save/restore IRQ state
 -----------------------------------------------------------------------------*/
__NAKED__ EXPORT_C TInt TRWSpinLock::LockIrqSaveR()
	{
	THISCALL_PROLOG0()
	asm("pushfd ");
	asm("cli ");
	RWSPIN_RLOCK_ENTRY_CHECK()
	asm("mov ax, 0x100 ");
	asm("lock xadd [ecx], ax ");			/* ah = in.r++, al = in.w */
	asm("rwl_rlockirqs_loop: ");
	asm("cmp al, [ecx+2] ");				/* compare al to out.w */
	asm("jnz short rwl_rlockirqs_loop2 ");
	RWSPIN_RLOCK_MARK_ACQ()
	asm("lock add dword ptr [esp], 0 ");	/* make sure subsequent accesses don't happen until lock acquired */
	asm("pop eax ");						/* retrieve saved EFLAGS */
	asm("and eax, 0x200 ");					/* return just interrupt mask bit */
	THISCALL_EPILOG0()

	asm("rwl_rlockirqs_loop2: ");
	X86_PAUSE
	asm("jmp short rwl_rlockirqs_loop ");
	}

__NAKED__ EXPORT_C void TRWSpinLock::UnlockIrqRestoreR(TInt)
	{
	THISCALL_PROLOG1()
	RWSPIN_RUNLOCK_ENTRY_CHECK()
	asm("lock add word ptr [ecx+2], 0x100 ");	/* ++out.r */
	asm("test dword ptr [esp+4], 0x200 ");
	asm("jz short rwl_runlockirqr_1 ");
	asm("sti ");
	asm("rwl_runlockirqr_1: ");
	THISCALL_EPILOG1()
	}

__NAKED__ EXPORT_C TBool TRWSpinLock::FlashIrqRestoreR(TInt)
	{
	/* don't mess with stacked args, yet */
	THISCALL_PROLOG0()
	asm("mov eax, [ecx] ");		/* al=in.w, ah=in.r, byte2=out.w, byte3=out.r */
	asm("mov edx, eax ");
	asm("shr edx, 16 ");		/* dl=out.w */
	asm("xor eax, edx ");		/* al = in.w ^ out.w = 0 if no writers waiting */
	asm("and eax, 0xff ");
	asm("jne short rwl_rflashirqr_1 ");

	/* now we can remove stacked arg since we don't need it */
	THISCALL_EPILOG1()

	asm("rwl_rflashirqr_1: ");
	THISCALL_PROLOG1()
	asm("test dword ptr [esp+4], 0x200 ");
	asm("jnz short rwl_rflashirqr_2 ");
	asm("call %a0" : : "i" (&rwspin_rlock_flash_only));
	asm("jmp short rwl_rflashirqr_3 ");
	asm("rwl_rflashirqr_2: ");
	asm("call %a0" : : "i" (&rwspin_rlock_flash_irq));
	asm("rwl_rflashirqr_3: ");
	THISCALL_EPILOG1()
	}


/*-----------------------------------------------------------------------------
 - Write locks disabling IRQ with save/restore IRQ state
 -----------------------------------------------------------------------------*/
__NAKED__ EXPORT_C TInt TRWSpinLock::LockIrqSaveW()
	{
	THISCALL_PROLOG0()
	asm("pushfd ");
	asm("cli ");
	RWSPIN_WLOCK_ENTRY_CHECK()
	asm("mov ax, [ecx] ");					/* ah = in.r, al = in.w */
	asm("rwl_wlockirqs_loop3: ");
	asm("mov edx, eax ");
	asm("inc dl ");							/* dh = in.r, dl = in.w+1 */
	asm("lock cmpxchg [ecx], dx ");			/* attempt to update in.w */
	asm("jne short rwl_wlockirqs_loop3 ");	/* loop if failed */
	asm("rwl_wlockirqs_loop: ");
	asm("cmp ax, [ecx+2] ");				/* compare ax to (out.w,out.r) */
	asm("jnz short rwl_wlockirqs_loop2 ");
	RWSPIN_WLOCK_MARK_ACQ()
	asm("lock add dword ptr [esp], 0 ");	/* make sure subsequent accesses don't happen until lock acquired */
	asm("pop eax ");						/* retrieve saved EFLAGS */
	asm("and eax, 0x200 ");					/* return just interrupt mask bit */
	THISCALL_EPILOG0()

	asm("rwl_wlockirqs_loop2: ");
	X86_PAUSE
	asm("jmp short rwl_wlockirqs_loop ");
	}

__NAKED__ EXPORT_C void TRWSpinLock::UnlockIrqRestoreW(TInt)
	{
	THISCALL_PROLOG1()
	RWSPIN_WUNLOCK_ENTRY_CHECK()
	asm("mov ax, [ecx+2] ");				/* ah = out.r, al = out.w */
	asm("rwl_wunlockirqr_loop: ");
	asm("mov edx, eax ");
	asm("inc dl ");							/* dh = out.r, dl = out.w+1 */
	asm("lock cmpxchg [ecx+2], dx ");		/* attempt to update out.w */
	asm("jne short rwl_wunlockirqr_loop ");	/* loop if failed */
	asm("test dword ptr [esp+4], 0x200 ");
	asm("jz short rwl_wunlockirqr_1 ");
	asm("sti ");
	asm("rwl_wunlockirqr_1: ");
	THISCALL_EPILOG1()
	}

__NAKED__ EXPORT_C TBool TRWSpinLock::FlashIrqRestoreW(TInt)
	{
	/* don't mess with stacked args, yet */
	THISCALL_PROLOG0()
	asm("mov eax, [ecx] ");		/* al=in.w, ah=in.r, byte2=out.w, byte3=out.r */
	asm("mov edx, eax ");
	asm("shr edx, 16 ");		/* dl=out.w, dh=out.r */
	asm("inc dl ");				/* dx==ax now means no-one else is waiting for lock */
	asm("xor eax, edx ");
	asm("and eax, 0xffff ");
	asm("jne short rwl_wflashirqr_1 ");

	/* now we can remove stacked arg since we don't need it */
	THISCALL_EPILOG1()

	asm("rwl_wflashirqr_1: ");
	THISCALL_PROLOG1()
	asm("test dword ptr [esp+4], 0x200 ");
	asm("jnz short rwl_wflashirqr_2 ");
	asm("call %a0" : : "i" (&rwspin_wlock_flash_only));
	asm("jmp short rwl_wflashirqr_3 ");
	asm("rwl_wflashirqr_2: ");
	asm("call %a0" : : "i" (&rwspin_wlock_flash_irq));
	asm("rwl_wflashirqr_3: ");
	THISCALL_EPILOG1()
	}


/*-----------------------------------------------------------------------------
 - Read lock flash allowing preemption
 -----------------------------------------------------------------------------*/
extern "C" TBool __fastcall rwspin_rlock_flash_preempt(TRWSpinLock* a)
	{
	a->UnlockOnlyR();
	NKern::PreemptionPoint();
	a->LockOnlyR();
	return TRUE;
	}

__NAKED__ EXPORT_C TBool TRWSpinLock::FlashPreemptR()
	{
	THISCALL_PROLOG0()
	asm("mov eax, [ecx] ");		/* al=in.w, ah=in.r, byte2=out.w, byte3=out.r */
	asm("mov edx, eax ");
	asm("shr edx, 16 ");		/* dl=out.w */
	asm("xor eax, edx ");		/* al = in.w ^ out.w = 0 if no writers waiting */
	asm("and eax, 0xff ");
	asm("jne %a0" : : "i" (&rwspin_rlock_flash_preempt));
	THISCALL_EPILOG0()
	}


/*-----------------------------------------------------------------------------
 - Write lock flash allowing preemption
 -----------------------------------------------------------------------------*/
extern "C" TBool __fastcall rwspin_wlock_flash_preempt(TRWSpinLock* a)
	{
	a->UnlockOnlyW();
	NKern::PreemptionPoint();
	a->LockOnlyW();
	return TRUE;
	}

__NAKED__ EXPORT_C TBool TRWSpinLock::FlashPreemptW()
	{
	THISCALL_PROLOG0()
	asm("mov eax, [ecx] ");		/* al=in.w, ah=in.r, byte2=out.w, byte3=out.r */
	asm("mov edx, eax ");
	asm("shr edx, 16 ");		/* dl=out.w, dh=out.r */
	asm("inc dl ");				/* dx==ax now means no-one else is waiting for lock */
	asm("xor eax, edx ");
	asm("and eax, 0xffff ");
	asm("jne %a0" : : "i" (&rwspin_wlock_flash_preempt));
	THISCALL_EPILOG0()
	}