//------------------------------------------------------------------
// file: vec_memcpy.S
// AltiVec enabled version of memcpy and bcopy
//------------------------------------------------------------------
//------------------------------------------------------------------
// Copyright Motorola, Inc. 2003
// ALL RIGHTS RESERVED
//
// You are hereby granted a copyright license to use, modify, and
// distribute the SOFTWARE so long as this entire notice is retained
// without alteration in any modified and/or redistributed versions,
// and that such modified versions are clearly identified as such.
// No licenses are granted by implication, estoppel or otherwise under
// any patents or trademarks of Motorola, Inc.
//
// The SOFTWARE is provided on an "AS IS" basis and without warranty.
// To the maximum extent permitted by applicable law, MOTOROLA DISCLAIMS
// ALL WARRANTIES WHETHER EXPRESS OR IMPLIED, INCLUDING IMPLIED
// WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR
// PURPOSE AND ANY WARRANTY AGAINST INFRINGEMENT WITH
// REGARD TO THE SOFTWARE (INCLUDING ANY MODIFIED VERSIONS
// THEREOF) AND ANY ACCOMPANYING WRITTEN MATERIALS.
//
// To the maximum extent permitted by applicable law, IN NO EVENT SHALL
// MOTOROLA BE LIABLE FOR ANY DAMAGES WHATSOEVER
// (INCLUDING WITHOUT LIMITATION, DAMAGES FOR LOSS OF
// BUSINESS PROFITS, BUSINESS INTERRUPTION, LOSS OF BUSINESS
// INFORMATION, OR OTHER PECUNIARY LOSS) ARISING OF THE USE OR
// INABILITY TO USE THE SOFTWARE. Motorola assumes no responsibility
// for the maintenance and support of the SOFTWARE.
//------------------------------------------------------------------
//------------------------------------------------------------------
// extern void * memcpy(void *dst, const void *src, size_t len);
// Returns:
// void *dst
//------------------------------------------------------------------
//------------------------------------------------------------------
// extern void * memmove( void *dst, const void *src, size_t len );
// Copies len characters from src to dst and returns the value of
// dst. Works correctly for overlapping memory regions.
// - Harbison&Steele 4th ed (corrected as to return)
// Returns:
// void *dst
//------------------------------------------------------------------
//------------------------------------------------------------------
// extern void * bcopy(const void *src, void *dst, size_t len);
// Returns:
// void *dst
//------------------------------------------------------------------
// memcpy and memmove are combined into one entry point here because of
// the similarity of operation and need to create fool-proof code.
// The following conditions determine what is "fool proof":
//
// if: then single entry:
// (DST-SRC)<0 && (SRC-DST)>=BC && BC>MIN_VEC will b to v_memcpy
// (DST-SRC)<0 && (SRC-DST)< BC && BC>MIN_VEC must b to v_memcpy
// (DST-SRC)<0 && BC<MIN_VEC copy fwd byte-by-byte
// (DST-SRC)==0 || BC==0 will just return
// (DST-SRC)>0 && BC<MIN_VEC copy bkwd byte-by-byte
// (DST-SRC)>0 && (DST-SRC)< BC && BC>MIN_VEC must b to v_memmove
// (DST-SRC)>0 && (SRC-DST)>=BC && BC>MIN_VEC will b to v_memmove
// If you call memmove (or vec_memmove) and |DST-SRC|>=BC,
// this code will branch to v_memcpy anyway for maximum performance.
// Revision History:
// Rev 0.0 Original Chuck Corley 02/03/03
// Can still add dst, 128B loop, and aligned option
// Rev 0.01 Fixed JY's seg-fault violation CJC 02/17/03
// Rev 0.1 Added 128B loop and dst; cndtnlzd dcbz CJC 02/18/03
// (Creating separate path for QW aligned didn't help much)
// Rev 0.11 Small code schdling; chngd dst for memmove CJC 02/23/03
// Rev 0.20 Eliminated alternate entry and cleanup CJC 02/27/03
// Rev 0.21 Inproved loop branch targets for v_mempcy CJC 03/01/03
// Rev 0.22 Experimented with dst (sent to H.) CJC 03/02/03
// Rev 0.23 Substituted dcba for dcbz (sent to JY) CJC 03/08/03
// Rev 0.24 Use two dst streams CJC 03/12/03
// Rev 0.25 Fix for all compilers, cleanup, and release with
// libmotovec.a rev 0.10 CJC 03/14/03
// Rev 0.30 Fix for pre-empted destination (SNDF-DS) CJC 04/02/03
//
// Between Rev 0.25 and 0.30 the code was revised to store elements of
// source at destination when first and/or last vector are less than 16
// bytes. Areviewer at SNDF observed that loading the destination vector
// for merging exposed the "uninvolved" destination bytes to incoherency
// if an interrupt pre-empted this routine and modified the "uninvolved"
// destination vector(s) while held in register for merging. It seems
// like a low possibility but this revision is no longer subject to that
// possibility. (It is also slightly faster than Rev 0.25.)
// This is beta quality code; users are encouraged to make it faster.
// ASSUMPTIONS:
// Code is highly likely to be in the cache; data is not (streaming data)
#define VRSV 256 // VRSAVE spr
// Don't use vectors for BC <= MIN_VEC. Works only if MIN_VEC >= 16 bytes.
#define MIN_VEC 16
// Don't use Big_loop in v_memcpy for |dst-src|<= minimum overlap.
#define MIN_OVL 128
// Register useage
#define Rt r0 // r0 when used as a temporary register
#define DST r3 // entering: dst pointer; exiting: same dst pointer
#define SRC r4 // entering: src ptr; then end of src range index (SRC+BC) in memmove
#define BC r5 // entering: Byte_Count
#define PCS r6 // save for partial checksum entering
#define DMS r7 // dst - src initially
#define BK r7 // BC - 1 +/- (n*16)
// Codewarrior will put an unwelcome space as "lbzu r0,1(r7 )"
// if you don't put the comment right after the r7. CJC 030314
#define SM1 r8// src -1 for byte-by-byte forwards initially
#define S r8 // src[28:31]
#define SMD r8 // src[0:27]-dst[0:27]
#define STR r8 // data stream touch block & stride info for Big_loop
#define DM1 r9// dst -1 for byte-by-byte forwards initially
#define D r9 // dst[28:31]
#define DNX r9 // (dst+n*16)[28:31]
#define BL r9 // second byte_kount index pointer
#define SBC r10// src + byte count initially then src[28:31]
#define BLK r10 // temporary data stream touch block & stride info
#define DR r10 // (dst+16)[0:27]
#define QW r10 // number of quad words (vectors)
#define DBC r11// dst + byte count initially
#define BLL r11 // temporary data stream touch block & stride info
#define SBK r11 // (src+byte_count-1)
#define SBR r11 // (src+byte_count-1)[0:27]
#define DBK r11 // (dst+byte_count-1) then (dst+byte_count-1)[28:31]
#define BIG r11 // QW/8 or 128 byte loop count
#define SP8 r11 // SRC + n*128 (8 QWs) for data streaming after first call
#define RSV r12 // storage for VRSAVE register if used
#define VS0 v0 // src vector for permuting
#define VS1 v1 // src vector for permuting
#define VP3 v2 // d - s permute register
#define VPS0 v3 // permuted source vector to store
#define VPS1 v4 // 2nd permuted source vector to store
#define VPS2 v5 // additional permuted src in Big loop
#define VS2 v6 // src vector for permuting
#define VPS3 v6 // additional permuted src in Big loop
#define VS3 v7 // additional src load in Big loop
#define VPS4 v7 // additional permuted src in Big loop
#define VS4 v8 // additional src load in Big loop
#define VPS5 v8 // additional permuted src in Big loop
#define VS5 v9 // additional src load in Big loop
#define VPS6 v9 // additional permuted src in Big loop
#define VS6 v10 // additional src load in Big loop
#define VPS7 v10 // additional permuted src in Big loop
#define VS7 v11 // additional src load in Big loop
// Conditionalize the use of dcba. It will help if the data is
// not in cache and hurt if it is. Generally, except for small
// benchmarks repeated many times, we assume data is not in cache
// (data streaming) and using dcbz is a performance boost.
#ifndef NO_DCBA
#if defined(__GNUC__) || defined(__MWERKS__) || defined(_DIAB_TOOL)
// gcc and codewarrior and diab don't assemble dcba
#define DCBK .long 0x7c033dec
// dcba r3,r7 or dcba DST,BK
#define DCBL .long 0x7c034dec
// dcba r3,r9 or dcba DST,BL
#else
#ifdef __ghs__
.macro DCBK
.long 0x7c033dec
.endm
.macro DCBL
.long 0x7c034dec
.endm
#else
#define DCBK dcba DST,BK
#define DCBL dcba DST,BL
#endif // __ghs__
#endif // __GNUC__ or __MWERKS__
#else
#define DCBK nop
#define DCBL nop
#endif // NO_DCBA
// Conditionalize the use of dst (data stream touch). It will help
// if the data is not in cache and hurt if it is (though not as badly
// as dcbz). Generally, except for small benchmarks repeated many times,
// we assume data is not in cache (data streaming) and using dst is a
// performance boost.
#ifndef NO_DST
#define STRM_B dst SBC,BLL,0
#define STRM_F dst SRC,BLK,0
#define STRM_1 dst SP8,STR,1
#else
#define STRM_B nop
#define STRM_F nop
#define STRM_1 nop
#endif
// Condition register use
// cr0[0:2] = (dst-src==0)? return: ((dst-src>0)? copy_bkwd, copy_fwd;);
// then cr0[0:2] = (dst[28:31]-src[28:31]<0)? "shifting left", "shifting right";
// cr1[0,2] = (BC == 0)? 1 : 0; (nothing to move)
// then cr1[2] = (DST[28:31] == 0)? 1 : 0; (D0 left justified)
// then cr1[2] = ((DBK = DST+BC-1)[28:31] = 0xF)? 1 : 0; (DN right justified)
// cr5[0,2] = (|DST-SRC|<=MIN_OVL)?1:0; (Overlap too small for Big loop?)
// cr6[1,2] = (DST-SRC>=BC)?1:0; (Okay for v_memmove to copy forward?)
// then cr6[2] = (QW == 0)? 1 : 0; (Any full vectors to move?)
// then cr6[1] = (QW > 4)? 1 : 0; (>4 vectors to move?)
// then cr6[3] = (third store[27] == 1)? 1: 0; (cache line alignment)
// then cr6[3] = (last store[27] == 1)? 1: 0; (last store odd?)
// cr7[2] = (BC>MIN_VEC)?1:0; (BC big enough to warrant vectors)
// then cr7[0:3] = (DST+16)[0:27]-DST (How many bytes (iff <16) in first vector?)
// then cr7[1] = (QW > 14)? 1 : 0; (>14 vectors to move?)
// then cr7[0:3] = (DST+BC)[0:27] (How many bytes (iff <16) in last vector?)
.text
#ifdef __MWERKS__
.align 32
#else
.align 5
#endif
#ifdef LIBMOTOVEC
.globl memmove
memmove:
nop // IU1 Compilers forget first label
.globl memcpy
memcpy:
#else
.globl vec_memmove
vec_memmove:
nop // IU1 Only way I know to preserve both labels
.globl _vec_memcpy
_vec_memcpy:
#endif
subf. DMS,SRC,DST // IU1 Compute dst-src difference
cmpi cr1,0,BC,0 // IU1 Eliminate zero byte count moves
cmpi cr7,0,BC,MIN_VEC // IU1 Check for minimum byte count
addi SM1,SRC,-1 // IU1 Pre-bias and duplicate src for fwd
addi DM1,DST,-1 // IU1 Pre-bias and duplicate destination
add SBC,SRC,BC // IU1 Pre-bias and duplicate src for bkwd
beqlr // return if DST = SRC
add DBC,DST,BC // IU1 Pre-bias and duplicate destination
subf Rt,DST,SRC // IU1 Form |DST-SRC| if DST-SRC<0
beqlr cr1 // return if BC = 0
bgt Cpy_bkwd // b if DST-SRC>0 (have to copy backward)
cmpi cr5,0,Rt,MIN_OVL // IU1 (|DST-SRC|>128)?1:0; for v_memcpy
bgt cr7,v_memcpy // b if BC>MIN_VEC (okay to copy vectors fwd)
// Copy byte-by-byte forwards if DST-SRC<0 and BC<=MIN_VEC
mtctr BC // i=BC; do ...;i--; while (i>0)
Byte_cpy_fwd:
lbzu Rt,1(SM1) // LSU * ++(DST-1) = * ++(SRC-1)
stbu Rt,1(DM1) // LSU
bdnz Byte_cpy_fwd
blr
nop // IU1 Improve next label as branch target
Cpy_bkwd:
cmpi cr5,0,DMS,MIN_OVL // IU1 ((DST-SRC)>128)?1:0; for v_memcpy
cmp cr6,0,DMS,BC // IU1 cr6[1,2]=(DST-SRC>=BC)?1:0;
bgt cr7,v_memmove // b if BC>MIN_VEC (copy vectors bkwd)
// Copy byte-by-byte backwards if DST-SRC>0 and BC<=MIN_VEC
mtctr BC // i=BC; do ...;i--; while (i>0)
Byte_cpy_bwd:
lbzu Rt,-1(SBC) // LSU * --(DST+BC) = * --(SRC+BC)
stbu Rt,-1(DBC) // LSU Store it
bdnz Byte_cpy_bwd
blr
#ifdef __MWERKS__
.align 16
#else
.align 4
#endif
v_memmove:
// Byte count < MIN_VEC bytes will have been copied by scalar code above,
// so this will not deal with small block moves < MIN_VEC.
// For systems using VRSAVE, define VRSAVE=1 when compiling. For systems
// that don't, make sure VRSAVE is undefined.
#ifdef VRSAVE
mfspr RSV,VRSV // IU2 Get current VRSAVE contents
#endif
rlwinm S,SRC,0,28,31 // IU1 Save src address bits s[28:31]
rlwinm D,DST,0,28,31 // IU1 D = dst[28:31]
bge cr6,MC_entry // b to v_memcpy if DST-SRC>=BC (fwd copy OK)
#ifdef VRSAVE
oris Rt,RSV,0xfff0 // IU1 Or in registers used by this routine
#endif
lis BLL,0x010c // IU1 Stream 12 blocks of 16 bytes
subf. SMD,D,S // IU1 if S-D<0 essentially shifting right
#ifdef VRSAVE
mtspr VRSV,Rt // IU2 Save in VRSAVE before first vec op
#endif
lvsr VP3,0,DMS // LSU Permute vector for dst - src shft right
ori BLL,BLL,0xffe0 // IU1 Stream stride -32B
STRM_B // LSU Start data stream at SRC+BC
addi SBK,SBC,-1 // IU1 Address of last src byte
bgt Rt_shft // Bytes from upper vector = (s-d>0)?s-d:16+s-d;
addi SMD,SMD,16 // IU1 Save 16-(d-s)
Rt_shft:
rlwinm SBR,SBK,0,0,27 // IU1 (SRC+BC-1)[0:27]
addi BK,BC,-1 // IU1 Initialize byte index
subf Rt,SBR,SBC // IU1 How many bytes in first source?
add DBK,DST,BK // IU1 Address of last dst byte
addi DR,DST,16 // IU1 Address of second dst vector
subf. SMD,Rt,SMD // IU1 if bytes in 1st src>Bytes in 1st permute
rlwinm Rt,DBK,0,28,31 // IU1 (DST+BC-1)[28:31]
rlwinm DR,DR,0,0,27 // IU1 (DST+16)[0:27]
// If there are more useful bytes in the upper vector of a permute pair than we
// will get in the first permute, the first loaded vector needs to be in the
// lower half of the permute pair. The upper half is a don't care then.
blt Get_bytes_rt // b if shifting left (D-S>=0)
lvx VS1,SRC,BK // LSU Get SN load started
// Comments numbering source and destination assume single path through the
// code executing each instruction once. For vec_memmove, an example would
// be the call memmove(BASE+0x0F, BASE+0x2F, 82). N = 6 in that case.
addi SRC,SRC,-16 // IU1 Decrement src base (to keep BK useful)
Get_bytes_rt: // Come here to get VS0 & Don't care what VS1 is
lvx VS0,SRC,BK // LSU Get SN-1 (SN if D-S<0) in lower vector
subf QW,DR,DBK // IU1 Bytes of full vectors to move (-16)
cmpi cr7,0,Rt,0xF // IU1 Is Dn right justified?
cmpi cr1,0,D,0 // IU1 Is D0 left justified?
rlwinm QW,QW,28,4,31 // IU1 Quad words remaining
add Rt,DST,BC // IU1 Refresh the value of DST+BC
cmpi cr6,0,QW,0 // IU1 Any full vectors to move?
vperm VPS0,VS0,VS1,VP3 // VPU Align SN-1 and SN to DN
vor VS1,VS0,VS0 // VIU1 Move lower vector to upper
beq cr7,Rt_just // b if DN is right justified
mtcrf 0x01,Rt // IU2 Put final vector byte count in cr7
rlwinm DBK,DBK,0,0,27 // IU1 Address of first byte of final vector
li D,0 // IU1 Initialize an index pointer
bnl cr7,Only_1W_bkwd // b if there was only one or zero words to store
stvewx VPS0,DBK,D // LSU store word 1 of two or three
addi D,D,4 // IU1 increment index
stvewx VPS0,DBK,D // LSU store word 2 of two or three
addi D,D,4 // IU1 increment index
Only_1W_bkwd:
bng cr7,Only_2W_bkwd // b if there were only two or zero words to store
stvewx VPS0,DBK,D // LSU store word 3 of three if necessary
addi D,D,4 // IU1 increment index
Only_2W_bkwd:
bne cr7,Only_B_bkwd // b if there are no half words to store
stvehx VPS0,DBK,D // LSU store one halfword if necessary
addi D,D,2 // IU1 increment index
Only_B_bkwd:
bns cr7,All_done_bkwd // b if there are no bytes to store
stvebx VPS0,DBK,D // LSU store one byte if necessary
b All_done_bkwd
Rt_just:
stvx VPS0,DST,BK // LSU Store 16 bytes at DN
All_done_bkwd:
addi BK,BK,-16 // IU1 Decrement destination byte count
ble cr6,Last_load // b if no Quad words to do
mtctr QW // IU2 for (i=0;i<=QW;i++)-execution serializng
cmpi cr6,0,QW,4 // IU1 Check QW>4
QW_loop:
lvx VS0,SRC,BK // LSU Get SN-2 (or SN-1 if ADJ==0)
vperm VPS0,VS0,VS1,VP3 // VPU Align SN-2 and SN-1 to DN-1
vor VS1,VS0,VS0 // VIU1 Move lower vector to upper
stvx VPS0,DST,BK // LSU Store 16 bytes at DN-1
addi BK,BK,-16 // IU1 Decrement byte kount
bdnzf 25,QW_loop // b if 4 or less quad words to do
add DNX,DST,BK // IU1 address of next store (DST+BC-1-16)
bgt cr6,GT_4QW // b if >4 quad words left
Last_load: // if D-S>=0, next load will be from same address as last
blt No_ld_bkwd // b if shifting right (S-D>=0)
addi SRC,SRC,16 // IU1 recorrect source if it was decremented
No_ld_bkwd:
lvx VS0,0,SRC // LSU Get last source SN-6 (guaranteed S0)
// Current 16 bytes is the last; we're done.
dss 0 // Data stream stop
vperm VPS0,VS0,VS1,VP3 // VPU Align SN-6 and SN-5 to DN-6
subfic D,DST,16 // IU1 How many bytes in first destination?
beq cr1,Lt_just // b if last destination is left justified
mtcrf 0x01,D // IU2 Put byte count remaining in cr7
li D,0 // IU1 Initialize index pointer
bns cr7,No_B_bkwd // b if only even number of bytes to store
stvebx VPS0,DST,D // LSU store first byte at DST+0
addi D,D,1 // IU1 increment index
No_B_bkwd:
bne cr7,No_H_bkwd // b if only words to store
stvehx VPS0,DST,D // LSU store halfword at DST+0/1
addi D,D,2 // IU1 increment index
No_H_bkwd:
bng cr7,No_W1_bkwd // b if exactly zero or two words to store
stvewx VPS0,DST,D // LSU store word 1 of one or three
addi D,D,4 // IU1 increment index
No_W1_bkwd:
bnl cr7,No_W2_bkwd // b if there was only one word to store
stvewx VPS0,DST,D // LSU store word 1 of two or 2 of three
addi D,D,4 // IU1 increment index
stvewx VPS0,DST,D // LSU store word 2 of two or 3 of three
b No_W2_bkwd
Lt_just:
stvx VPS0,0,DST // LSU Store 16 bytes at final dst addr D0
No_W2_bkwd:
#ifdef VRSAVE
mtspr VRSV,RSV // IU1 Restore VRSAVE
#endif
blr // Return destination address from entry
GT_4QW: // Do once if next store is to even half of cache line, else twice
lvx VS0,SRC,BK // LSU Get SN-3 (or SN-2)
mtcrf 0x02,DNX // IU2 cr6[3]=((DST+BC-1)[27]==1)?1:0;
vperm VPS0,VS0,VS1,VP3 // VPU Align SN-3 and SN-2 to Dn-2
vor VS1,VS0,VS0 // VIU1 Move lower vector to upper
addi DNX,DNX,-16 // IU1 Prepare to update cr6 next loop
stvx VPS0,DST,BK // LSU Store 16 bytes at DN-2
vor VS3,VS0,VS0 // VIU Make a copy of lower vector
addi BK,BK,-16 // IU1 Decrement byte count by 16
bdnzt 27,GT_4QW // b if next store is to upper (odd) half of CL
// At this point next store will be to even address.
lis STR,0x102 // IU1 Stream 2 blocks of 16 bytes
mtcrf 0x02,DST // IU2 cr6[3]=(DST[27]==1)?1:0; (DST odd?)
addi BL,BK,-16 // IU1 Create an alternate byte count - 16
ori STR,STR,0xffe0 // IU1 Stream stride -32B
addi SP8,SRC,-64 // IU1 Starting address for data stream touch
bso cr6,B32_bkwd // b if DST[27] == 1; i.e, final store is odd
bdnz B32_bkwd // decrement counter for last odd QW store
B32_bkwd: // Should be at least 2 stores remaining and next 2 are cache aligned
lvx VS2,SRC,BK // LSU Get SN-4 (or SN-3)
addi SP8,SP8,-32 // IU1 Next starting address for data stream touch
lvx VS1,SRC,BL // LSU Get SN-5 (or SN-4)
vperm VPS0,VS2,VS3,VP3 // VPU Align SN-4 and SN-3 to DN-3
STRM_1 // LSU Stream 64 byte blocks ahead of loads
DCBL // LSU allocate next cache line
vperm VPS1,VS1,VS2,VP3 // VPU Align SN-5 and SN-4 to DN-4
vor VS3,VS1,VS1 // VIU1 Move SN-5 to SN-3
stvx VPS0,DST,BK // LSU Store 16 bytes at DN-3
addi BK,BL,-16 // IU1 Decrement byte count
bdz Nxt_loc_bkwd // always decrement and branch to next instr
Nxt_loc_bkwd:
stvx VPS1,DST,BL // LSU Store 16 bytes at DN-4
addi BL,BK,-16 // IU1 Decrement alternate byte count
bdnz B32_bkwd // b if there are at least two more QWs to do
bns cr6,One_odd_QW // b if there was one more odd QW to store
b Last_load
// Come here with two more loads and two stores to do
One_odd_QW:
lvx VS1,SRC,BK // LSU Get SN-6 (or SN-5)
vperm VPS1,VS1,VS3,VP3 // VPU Align SN-6 and SN-5 to DN-5
stvx VPS1,DST,BK // LSU Store 16 bytes at DN-5
b Last_load
// End of memmove in AltiVec
#ifdef __MWERKS__
.align 16
#else
.align 4
#endif
v_memcpy:
// Byte count < MIN_VEC bytes will have been copied by scalar code above,
// so this will not deal with small block moves < MIN_VEC.
#ifdef VRSAVE
mfspr RSV,VRSV // IU2 Get current VRSAVE contents
#endif
rlwinm S,SRC,0,28,31 // IU1 Save src address bits s[28:31]
rlwinm D,DST,0,28,31 // IU1 D = dst[28:31]
MC_entry: // enter here from memmove if DST-SRC>=BC; this should be faster
#ifdef VRSAVE
oris Rt,RSV,0xfff0 // IU1 Or in registers used by this routine
#endif
lis BLK,0x010c // IU1 Stream 12 blocks of 16 bytes
subf. S,S,D // IU1 if D-S<0 essentially shifting left
#ifdef VRSAVE
mtspr VRSV,Rt // IU2 Save in VRSAVE before first vec op
#endif
lvsr VP3,0,DMS // LSU Permute vector for dst - src shft right
ori BLK,BLK,32 // IU1 Stream stride 32B
STRM_F // LSU Start data stream 0 at SRC
addi DR,DST,16 // IU1 Address of second dst vector
addi DBK,DBC,-1 // IU1 Address of last dst byte
// If D-S<0 we are "kinda" shifting left with the right shift permute vector
// loaded to VP3 and we need both S0 and S1 to permute. If D-S>=0 then the
// first loaded vector needs to be in the upper half of the permute pair and
// the lower half is a don't care then.
bge Ld_bytes_rt // b if shifting right (D-S>=0)
lvx VS0,0,SRC // LSU Get S0 load started
// Comments numbering source and destination assume single path through the
// code executing each instruction once. For vec_memcpy, an example would
// be the call memcpy(BASE+0x1E, BASE+0x1F, 259). N = 16 in that case.
addi SRC,SRC,16 // IU1 Increment src base (to keep BK useful)
Ld_bytes_rt: // Come here to get VS1 & Don't care what VS0 is
lvx VS1,0,SRC // LSU Get S1 (or S0 if D-S>=0) in upper vector
rlwinm DR,DR,0,0,27 // IU1 (DST+16)[0:27]
cmpi cr1,0,D,0 // IU1 Is D0 left justified?
subf Rt,DST,DR // IU1 How many bytes in first destination?
subf QW,DR,DBK // IU1 Bytes of full vectors to move (-16)
li BK,0 // IU1 Initialize byte kount index
mtcrf 0x01,Rt // IU2 Put bytes in 1st dst in cr7
rlwinm QW,QW,28,4,31 // IU1 Quad words remaining
vperm VPS0,VS0,VS1,VP3 // VPU Align S0 and S1 to D0
vor VS0,VS1,VS1 // VIU1 Move upper vector to lower
beq cr1,Left_just // b if D0 is left justified
bns cr7,No_B_fwd // b if only even number of bytes to store
stvebx VPS0,DST,BK // LSU store first byte at DST+0
addi BK,BK,1 // IU1 increment index
No_B_fwd:
bne cr7,No_H_fwd // b if only words to store
stvehx VPS0,DST,BK // LSU store halfword at DST+0/1
addi BK,BK,2 // IU1 increment index
No_H_fwd:
bng cr7,No_W1_fwd // b if exactly zero or two words to store
stvewx VPS0,DST,BK // LSU store word 1 of one or three
addi BK,BK,4 // IU1 increment index
No_W1_fwd:
bnl cr7,No_W2_fwd // b if there was only one word to store
stvewx VPS0,DST,BK // LSU store word 1 of two or 2 of three
addi BK,BK,4 // IU1 increment index
stvewx VPS0,DST,BK // LSU store word 2 of two or 3 of three
b No_W2_fwd
Left_just:
stvx VPS0,0,DST // LSU Store 16 bytes at D0
No_W2_fwd:
rlwinm Rt,DBK,0,28,31 // IU1 (DBK = DST+BC-1)[28:31]
cmpi cr6,0,QW,0 // IU1 Any full vectors to move?
li BK,16 // IU1 Re-initialize byte kount index
cmpi cr1,0,Rt,0xF // IU1 Is DN right justified?
cmpi cr7,0,QW,14 // IU1 Check QW>14
ble cr6,Last_ld_fwd // b if no Quad words to do
mtctr QW // IU2 for (i=0;i<=QW;i++)
cmpi cr6,0,QW,4 // IU1 Check QW>4
QW_fwd_loop:
lvx VS1,SRC,BK // LSU Get S2 (or S1)
vperm VPS0,VS0,VS1,VP3 // VPU Align S1 and S2 to D1
vor VS0,VS1,VS1 // VIU1 Move upper vector to lower
stvx VPS0,DST,BK // LSU Store 16 bytes at D1(+n*16 where n<4)
addi BK,BK,16 // IU1 Increment byte kount index
bdnzf 25,QW_fwd_loop // b if 4 or less quad words to do
add DNX,DST,BK // IU1 address of next store (DST+32 if QW>4)
addi QW,QW,-1 // IU1 One more QW stored by now
bgt cr6,GT_4QW_fwd // b if >4 quad words left
Last_ld_fwd: // Next 16 bytes is the last; we're done.
add DBC,DST,BC // IU1 Recompute address of last dst byte + 1
add SBC,SRC,BC // IU1 Recompute address of last src byte + 1
bge No_ld_fwd // b if shifting right (D-S>=0)
addi SBC,SBC,-16 // IU1 if D-S>=0 we didn't add 16 to src
No_ld_fwd:
mtcrf 0x01,DBC // IU2 Put final vector byte count in cr7
addi DBK,DBC,-1 // IU1 Recompute address of last dst byte
addi Rt,SBC,-1 // IU1 Recompute address of last src byte
// If D-S<0 we have already loaded all the source vectors.
// If D-S>=0 then the first loaded vector went to the upper half of the permute
// pair and we need one more vector. (This may be a duplicate.)
lvx VS1,0,Rt // LSU Get last source S14 (guaranteed SN)
#ifndef NO_DST
dss 0 // Data stream 0 stop
dss 1 // Data stream 1 stop
#endif
vperm VPS0,VS0,VS1,VP3 // VPU Align S13 and S14 to D14
beq cr1,Rt_just_fwd // b if last destination is right justified
rlwinm DBK,DBK,0,0,27 // IU1 Round to QW addr of last byte
li D,0 // IU1 Initialize index pointer
bnl cr7,Only_1W_fwd // b if there was only one or zero words to store
stvewx VPS0,DBK,D // LSU store word 1 of two or three
addi D,D,4 // IU1 increment index
stvewx VPS0,DBK,D // LSU store word 2 of two or three
addi D,D,4 // IU1 increment index
Only_1W_fwd:
bng cr7,Only_2W_fwd // b if there were only two or zero words to store
stvewx VPS0,DBK,D // LSU store word 3 of three if necessary
addi D,D,4 // IU1 increment index
Only_2W_fwd:
bne cr7,Only_B_fwd // b if there are no half words to store
stvehx VPS0,DBK,D // LSU store one halfword if necessary
addi D,D,2 // IU1 increment index
Only_B_fwd:
bns cr7,All_done_fwd // b if there are no bytes to store
stvebx VPS0,DBK,D // LSU store one byte if necessary
b All_done_fwd
Rt_just_fwd:
stvx VPS0,DST,BK // LSU Store 16 bytes at D14
All_done_fwd:
#ifdef VRSAVE
mtspr VRSV,RSV // IU1 Restore VRSAVE
#endif
blr // Return destination address from entry
#ifdef __MWERKS__
.align 16
#else
.align 4
#endif
GT_4QW_fwd: // Do once if nxt st is to odd half of cache line, else twice
lvx VS1,SRC,BK // LSU Get S3 (or S2)
addi QW,QW,-1 // IU1 Keeping track of QWs stored
mtcrf 0x02,DNX // IU2 cr6[3]=((DST+32)[27]==1)?1:0;
addi DNX,DNX,16 // IU1 Update cr6 for next loop
addi Rt,QW,-2 // IU1 Insure at least 2 QW left after big loop
vperm VPS0,VS0,VS1,VP3 // VPU Align S2 and S3 to D2
vor VS0,VS1,VS1 // VIU1 Move upper vector to lower
stvx VPS0,DST,BK // LSU Store 16 bytes at D2
addi BK,BK,16 // IU1 Increment byte count by 16
bdnzf 27,GT_4QW_fwd // b if next store is to lower (even) half of CL
// At this point next store will be to even address.
mtcrf 0x02,DBK // IU2 cr6[3]=((last store)[27]==1)?1:0; (odd?)
lis STR,0x104 // IU1 Stream 4 blocks of 16 bytes
addi BL,BK,16 // IU1 Create an alternate byte kount + 32
ori STR,STR,32 // IU1 Stream stride 32B
#ifndef NO_BIG_LOOP
rlwinm BIG,Rt,29,3,31 // IU1 QW/8 big loops to do
rlwinm Rt,Rt,0,0,28 // IU1 How many QWs will be done in big loop
bgt cr7,Big_loop // b if QW > 14
#endif
No_big_loop:
// We need the ctr register to reflect an even byte count before entering
// the next block - faster to decrement than to reload.
addi SP8,SRC,256 // IU1 Starting address for data stream touch
xoris STR,STR,0x6 // IU1 Reset stream to 2 blocks of 16 bytes
bns cr6,B32_fwd // b if DST[27] == 0; i.e, final store is even
bdnz B32_fwd // decrement counter for last QW store odd
B32_fwd: // Should be at least 2 stores remaining and next 2 are cache aligned
lvx VS1,SRC,BK // LSU Get S12
addi SP8,SP8,32 // IU1 Next starting address for data stream touch
lvx VS2,SRC,BL // LSU Get S13
vperm VPS1,VS0,VS1,VP3 // VPU Align S11 and S12 to D11
STRM_1 // LSU Stream 64 byte blocks ahead of loads
DCBK // LSU then Kill instead of RWITM
vperm VPS0,VS1,VS2,VP3 // VPU Align S12 and S13 to D12
vor VS0,VS2,VS2 // VIU1 Move S13 to S11
stvx VPS1,DST,BK // LSU Store 16 bytes at D11
addi BK,BL,16 // IU1 Increment byte count
bdz Nxt_loc_fwd // always decrement and branch to next instr
Nxt_loc_fwd:
stvx VPS0,DST,BL // LSU Store 16 bytes at D12
addi BL,BK,16 // IU1 Increment alternate byte count
bdnz B32_fwd // b if there are at least two more QWs to do
bso cr6,One_even_QW // b if there is one even and one odd QW to store
b Last_ld_fwd // b if last store is to even address
// Come here with two more loads and two stores to do
One_even_QW:
lvx VS1,SRC,BK // LSU Get S14 (or S13 if if D-S>=0)
vperm VPS0,VS0,VS1,VP3 // VPU Align S13 and S14 to D13
vor VS0,VS1,VS1 // VIU1 Move upper vector to lower
stvx VPS0,DST,BK // LSU Store 16 bytes at D13
addi BK,BK,16 // IU1 Increment byte count
b Last_ld_fwd
#ifdef __MWERKS__
.align 16
#else
.align 4
#endif
Big_loop:
subf QW,Rt,QW // IU1 Should be 2-7 QWs left after big loop
blt cr5,No_big_loop // b back if |DST-SRC|<128; Big_loop won't work.
mtctr BIG // IU2 loop for as many 128B loops as possible
addi SP8,SRC,256 // IU1 Starting address for data stream touch
Loop_of_128B: // Come here with QW>=10 and next store even; VS0 last load
lvx VS1,SRC,BK // LSU Get S4 (or S3 if D-S>=0)
addi BL,BK,32 // IU1 Increment Byte_Kount+16 by 32
addi SP8,SP8,128 // IU1 increment address for data stream touch
lvx VS3,SRC,BL // LSU Get S6 (or S5)
addi BL,BL,32 // IU1 Increment Byte_Kount+48 by 32
lvx VS5,SRC,BL // LSU Get S8 (or S7)
addi BL,BL,32 // IU1 Increment Byte_Kount+80 by 32
lvx VS7,SRC,BL // LSU Get S10 (or S9)
addi BL,BK,16 // IU1 Increment Byte_Kount+16 by 16
lvx VS2,SRC,BL // LSU Get S5 (or S4)
addi BL,BL,32 // IU1 Increment Byte_Kount+32 by 32
lvx VS4,SRC,BL // LSU Get S7 (or S6)
addi BL,BL,32 // IU1 Increment Byte_Kount+64 by 32
lvx VS6,SRC,BL // LSU Get S9 (or S8)
addi BL,BL,32 // IU1 Increment Byte_Kount+96 by 32
vperm VPS0,VS0,VS1,VP3 // VPU
lvx VS0,SRC,BL // LSU Get S11 (or S10)
vperm VPS1,VS1,VS2,VP3 // VPU
STRM_1 // LSU Stream 4 32B blocks, stride 32B
DCBK // LSU then Kill instead of RWITM
stvx VPS0,DST,BK // LSU Store D3
addi BK,BK,16 // IU1 Increment Byte_Kount+16 by 16
vperm VPS2,VS2,VS3,VP3 // VPU
stvx VPS1,DST,BK // LSU Store D4
addi BK,BK,16 // IU1 Increment Byte_Kount+32 by 16
vperm VPS3,VS3,VS4,VP3 // VPU
DCBK // LSU then Kill instead of RWITM
stvx VPS2,DST,BK // LSU Store D5
addi BK,BK,16 // IU1 Increment Byte_Kount+48 by 16
vperm VPS4,VS4,VS5,VP3 // VPU
stvx VPS3,DST,BK // LSU Store D6
addi BK,BK,16 // IU1 Increment Byte_Kount+64 by 16
vperm VPS5,VS5,VS6,VP3 // VPU
DCBK // LSU then Kill instead of RWITM
stvx VPS4,DST,BK // LSU Store D7
addi BK,BK,16 // IU1 Increment Byte_Kount+80 by 16
vperm VPS6,VS6,VS7,VP3 // VPU
stvx VPS5,DST,BK // LSU Store D8
addi BK,BK,16 // IU1 Increment Byte_Kount+96 by 16
vperm VPS7,VS7,VS0,VP3 // VPU
DCBK // LSU then Kill instead of RWITM
stvx VPS6,DST,BK // LSU Store D9
addi BK,BK,16 // IU1 Increment Byte_Kount+112 by 16
stvx VPS7,DST,BK // LSU Store D10
addi BK,BK,16 // IU1 Increment Byte_Kount+128 by 16
bdnz Loop_of_128B // b if ctr > 0 (QW/8 still > 0)
mtctr QW // IU1 Restore QW remaining to counter
addi BL,BK,16 // IU1 Create an alternate byte kount + 16
bns cr6,B32_fwd // b if DST[27] == 0; i.e, final store is even
bdnz B32_fwd // b and decrement counter for last QW store odd
// One of the above branches should have taken
// End of memcpy in AltiVec
// bcopy works like memcpy, but the source and destination operands are reversed.
// Following will just reverse the operands and branch to memcpy.
#ifdef LIBMOTOVEC
.globl bcopy
bcopy:
#else
.globl vec_bcopy
vec_bcopy:
#endif
mr Rt,DST // temp storage for what is really source address (r3)
mr DST,SRC // swap destination address to r3 to match memcpy dst
mr SRC,Rt // Complete swap of destination and source for memcpy
#ifdef LIBMOTOVEC
b memcpy // b to memcpy with correct args in r3 and r4
#else
b _vec_memcpy // b to vec_memcpy with correct args in r3 and r4
#endif
// End of bcopy in AltiVec