1 /*

     2  * ARM NEON vector operations.

     3  *

     4  * Copyright (c) 2007, 2008 CodeSourcery.

     5  * Written by Paul Brook

     6  *

     7  * This code is licenced under the GNU GPL v2.

     8  */

     9 #include <stdlib.h>

    10 #include <stdio.h>

    11 

    12 #include "cpu.h"

    13 #include "exec-all.h"

    14 #include "helpers.h"

    15 

    16 #define SIGNBIT (uint32_t)0x80000000

    17 #define SIGNBIT64 ((uint64_t)1 << 63)

    18 

    19 #define SET_QC() env->vfp.xregs[ARM_VFP_FPSCR] = CPSR_Q

    20 

    21 static float_status neon_float_status;

    22 #define NFS &neon_float_status

    23 

    24 /* Helper routines to perform bitwise copies between float and int.  */

    25 static inline float32 vfp_itos(uint32_t i)

    26 {

    27     union {

    28         uint32_t i;

    29         float32 s;

    30     } v;

    31 

    32     v.i = i;

    33     return v.s;

    34 }

    35 

    36 static inline uint32_t vfp_stoi(float32 s)

    37 {

    38     union {

    39         uint32_t i;

    40         float32 s;

    41     } v;

    42 

    43     v.s = s;

    44     return v.i;

    45 }

    46 

    47 #define NEON_TYPE1(name, type) \

    48 typedef struct \

    49 { \

    50     type v1; \

    51 } neon_##name;

    52 #ifdef WORDS_BIGENDIAN

    53 #define NEON_TYPE2(name, type) \

    54 typedef struct \

    55 { \

    56     type v2; \

    57     type v1; \

    58 } neon_##name;

    59 #define NEON_TYPE4(name, type) \

    60 typedef struct \

    61 { \

    62     type v4; \

    63     type v3; \

    64     type v2; \

    65     type v1; \

    66 } neon_##name;

    67 #else

    68 #define NEON_TYPE2(name, type) \

    69 typedef struct \

    70 { \

    71     type v1; \

    72     type v2; \

    73 } neon_##name;

    74 #define NEON_TYPE4(name, type) \

    75 typedef struct \

    76 { \

    77     type v1; \

    78     type v2; \

    79     type v3; \

    80     type v4; \

    81 } neon_##name;

    82 #endif

    83 

    84 NEON_TYPE4(s8, int8_t)

    85 NEON_TYPE4(u8, uint8_t)

    86 NEON_TYPE2(s16, int16_t)

    87 NEON_TYPE2(u16, uint16_t)

    88 NEON_TYPE1(s32, int32_t)

    89 NEON_TYPE1(u32, uint32_t)

    90 #undef NEON_TYPE4

    91 #undef NEON_TYPE2

    92 #undef NEON_TYPE1

    93 

    94 /* Copy from a uint32_t to a vector structure type.  */

    95 #define NEON_UNPACK(vtype, dest, val) do { \

    96     union { \

    97         vtype v; \

    98         uint32_t i; \

    99     } conv_u; \

   100     conv_u.i = (val); \

   101     dest = conv_u.v; \

   102     } while(0)

   103 

   104 /* Copy from a vector structure type to a uint32_t.  */

   105 #define NEON_PACK(vtype, dest, val) do { \

   106     union { \

   107         vtype v; \

   108         uint32_t i; \

   109     } conv_u; \

   110     conv_u.v = (val); \

   111     dest = conv_u.i; \

   112     } while(0)

   113 

   114 #define NEON_DO1 \

   115     NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1);

   116 #define NEON_DO2 \

   117     NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \

   118     NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2);

   119 #define NEON_DO4 \

   120     NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \

   121     NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \

   122     NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \

   123     NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4);

   124 

   125 #define NEON_VOP_BODY(vtype, n) \

   126 { \

   127     uint32_t res; \

   128     vtype vsrc1; \

   129     vtype vsrc2; \

   130     vtype vdest; \

   131     NEON_UNPACK(vtype, vsrc1, arg1); \

   132     NEON_UNPACK(vtype, vsrc2, arg2); \

   133     NEON_DO##n; \

   134     NEON_PACK(vtype, res, vdest); \

   135     return res; \

   136 }

   137 

   138 #define NEON_VOP(name, vtype, n) \

   139 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \

   140 NEON_VOP_BODY(vtype, n)

   141 

   142 #define NEON_VOP_ENV(name, vtype, n) \

   143 uint32_t HELPER(glue(neon_,name))(CPUState *env, uint32_t arg1, uint32_t arg2) \

   144 NEON_VOP_BODY(vtype, n)

   145 

   146 /* Pairwise operations.  */

   147 /* For 32-bit elements each segment only contains a single element, so

   148    the elementwise and pairwise operations are the same.  */

   149 #define NEON_PDO2 \

   150     NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \

   151     NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2);

   152 #define NEON_PDO4 \

   153     NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \

   154     NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \

   155     NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \

   156     NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \

   157 

   158 #define NEON_POP(name, vtype, n) \

   159 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \

   160 { \

   161     uint32_t res; \

   162     vtype vsrc1; \

   163     vtype vsrc2; \

   164     vtype vdest; \

   165     NEON_UNPACK(vtype, vsrc1, arg1); \

   166     NEON_UNPACK(vtype, vsrc2, arg2); \

   167     NEON_PDO##n; \

   168     NEON_PACK(vtype, res, vdest); \

   169     return res; \

   170 }

   171 

   172 /* Unary operators.  */

   173 #define NEON_VOP1(name, vtype, n) \

   174 uint32_t HELPER(glue(neon_,name))(uint32_t arg) \

   175 { \

   176     vtype vsrc1; \

   177     vtype vdest; \

   178     NEON_UNPACK(vtype, vsrc1, arg); \

   179     NEON_DO##n; \

   180     NEON_PACK(vtype, arg, vdest); \

   181     return arg; \

   182 }

   183 

   184 

   185 #define NEON_USAT(dest, src1, src2, type) do { \

   186     uint32_t tmp = (uint32_t)src1 + (uint32_t)src2; \

   187     if (tmp != (type)tmp) { \

   188         SET_QC(); \

   189         dest = ~0; \

   190     } else { \

   191         dest = tmp; \

   192     }} while(0)

   193 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)

   194 NEON_VOP_ENV(qadd_u8, neon_u8, 4)

   195 #undef NEON_FN

   196 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)

   197 NEON_VOP_ENV(qadd_u16, neon_u16, 2)

   198 #undef NEON_FN

   199 #undef NEON_USAT

   200 

   201 #define NEON_SSAT(dest, src1, src2, type) do { \

   202     int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \

   203     if (tmp != (type)tmp) { \

   204         SET_QC(); \

   205         if (src2 > 0) { \

   206             tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \

   207         } else { \

   208             tmp = 1 << (sizeof(type) * 8 - 1); \

   209         } \

   210     } \

   211     dest = tmp; \

   212     } while(0)

   213 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)

   214 NEON_VOP_ENV(qadd_s8, neon_s8, 4)

   215 #undef NEON_FN

   216 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)

   217 NEON_VOP_ENV(qadd_s16, neon_s16, 2)

   218 #undef NEON_FN

   219 #undef NEON_SSAT

   220 

   221 #define NEON_USAT(dest, src1, src2, type) do { \

   222     uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \

   223     if (tmp != (type)tmp) { \

   224         SET_QC(); \

   225         dest = 0; \

   226     } else { \

   227         dest = tmp; \

   228     }} while(0)

   229 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)

   230 NEON_VOP_ENV(qsub_u8, neon_u8, 4)

   231 #undef NEON_FN

   232 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)

   233 NEON_VOP_ENV(qsub_u16, neon_u16, 2)

   234 #undef NEON_FN

   235 #undef NEON_USAT

   236 

   237 #define NEON_SSAT(dest, src1, src2, type) do { \

   238     int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \

   239     if (tmp != (type)tmp) { \

   240         SET_QC(); \

   241         if (src2 < 0) { \

   242             tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \

   243         } else { \

   244             tmp = 1 << (sizeof(type) * 8 - 1); \

   245         } \

   246     } \

   247     dest = tmp; \

   248     } while(0)

   249 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)

   250 NEON_VOP_ENV(qsub_s8, neon_s8, 4)

   251 #undef NEON_FN

   252 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)

   253 NEON_VOP_ENV(qsub_s16, neon_s16, 2)

   254 #undef NEON_FN

   255 #undef NEON_SSAT

   256 

   257 #define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1

   258 NEON_VOP(hadd_s8, neon_s8, 4)

   259 NEON_VOP(hadd_u8, neon_u8, 4)

   260 NEON_VOP(hadd_s16, neon_s16, 2)

   261 NEON_VOP(hadd_u16, neon_u16, 2)

   262 #undef NEON_FN

   263 

   264 int32_t HELPER(neon_hadd_s32)(int32_t src1, int32_t src2)

   265 {

   266     int32_t dest;

   267 

   268     dest = (src1 >> 1) + (src2 >> 1);

   269     if (src1 & src2 & 1)

   270         dest++;

   271     return dest;

   272 }

   273 

   274 uint32_t HELPER(neon_hadd_u32)(uint32_t src1, uint32_t src2)

   275 {

   276     uint32_t dest;

   277 

   278     dest = (src1 >> 1) + (src2 >> 1);

   279     if (src1 & src2 & 1)

   280         dest++;

   281     return dest;

   282 }

   283 

   284 #define NEON_FN(dest, src1, src2) dest = (src1 + src2 + 1) >> 1

   285 NEON_VOP(rhadd_s8, neon_s8, 4)

   286 NEON_VOP(rhadd_u8, neon_u8, 4)

   287 NEON_VOP(rhadd_s16, neon_s16, 2)

   288 NEON_VOP(rhadd_u16, neon_u16, 2)

   289 #undef NEON_FN

   290 

   291 int32_t HELPER(neon_rhadd_s32)(int32_t src1, int32_t src2)

   292 {

   293     int32_t dest;

   294 

   295     dest = (src1 >> 1) + (src2 >> 1);

   296     if ((src1 | src2) & 1)

   297         dest++;

   298     return dest;

   299 }

   300 

   301 uint32_t HELPER(neon_rhadd_u32)(uint32_t src1, uint32_t src2)

   302 {

   303     uint32_t dest;

   304 

   305     dest = (src1 >> 1) + (src2 >> 1);

   306     if ((src1 | src2) & 1)

   307         dest++;

   308     return dest;

   309 }

   310 

   311 #define NEON_FN(dest, src1, src2) dest = (src1 - src2) >> 1

   312 NEON_VOP(hsub_s8, neon_s8, 4)

   313 NEON_VOP(hsub_u8, neon_u8, 4)

   314 NEON_VOP(hsub_s16, neon_s16, 2)

   315 NEON_VOP(hsub_u16, neon_u16, 2)

   316 #undef NEON_FN

   317 

   318 int32_t HELPER(neon_hsub_s32)(int32_t src1, int32_t src2)

   319 {

   320     int32_t dest;

   321 

   322     dest = (src1 >> 1) - (src2 >> 1);

   323     if ((~src1) & src2 & 1)

   324         dest--;

   325     return dest;

   326 }

   327 

   328 uint32_t HELPER(neon_hsub_u32)(uint32_t src1, uint32_t src2)

   329 {

   330     uint32_t dest;

   331 

   332     dest = (src1 >> 1) - (src2 >> 1);

   333     if ((~src1) & src2 & 1)

   334         dest--;

   335     return dest;

   336 }

   337 

   338 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? ~0 : 0

   339 NEON_VOP(cgt_s8, neon_s8, 4)

   340 NEON_VOP(cgt_u8, neon_u8, 4)

   341 NEON_VOP(cgt_s16, neon_s16, 2)

   342 NEON_VOP(cgt_u16, neon_u16, 2)

   343 NEON_VOP(cgt_s32, neon_s32, 1)

   344 NEON_VOP(cgt_u32, neon_u32, 1)

   345 #undef NEON_FN

   346 

   347 #define NEON_FN(dest, src1, src2) dest = (src1 >= src2) ? ~0 : 0

   348 NEON_VOP(cge_s8, neon_s8, 4)

   349 NEON_VOP(cge_u8, neon_u8, 4)

   350 NEON_VOP(cge_s16, neon_s16, 2)

   351 NEON_VOP(cge_u16, neon_u16, 2)

   352 NEON_VOP(cge_s32, neon_s32, 1)

   353 NEON_VOP(cge_u32, neon_u32, 1)

   354 #undef NEON_FN

   355 

   356 #define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2

   357 NEON_VOP(min_s8, neon_s8, 4)

   358 NEON_VOP(min_u8, neon_u8, 4)

   359 NEON_VOP(min_s16, neon_s16, 2)

   360 NEON_VOP(min_u16, neon_u16, 2)

   361 NEON_VOP(min_s32, neon_s32, 1)

   362 NEON_VOP(min_u32, neon_u32, 1)

   363 NEON_POP(pmin_s8, neon_s8, 4)

   364 NEON_POP(pmin_u8, neon_u8, 4)

   365 NEON_POP(pmin_s16, neon_s16, 2)

   366 NEON_POP(pmin_u16, neon_u16, 2)

   367 #undef NEON_FN

   368 

   369 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2

   370 NEON_VOP(max_s8, neon_s8, 4)

   371 NEON_VOP(max_u8, neon_u8, 4)

   372 NEON_VOP(max_s16, neon_s16, 2)

   373 NEON_VOP(max_u16, neon_u16, 2)

   374 NEON_VOP(max_s32, neon_s32, 1)

   375 NEON_VOP(max_u32, neon_u32, 1)

   376 NEON_POP(pmax_s8, neon_s8, 4)

   377 NEON_POP(pmax_u8, neon_u8, 4)

   378 NEON_POP(pmax_s16, neon_s16, 2)

   379 NEON_POP(pmax_u16, neon_u16, 2)

   380 #undef NEON_FN

   381 

   382 #define NEON_FN(dest, src1, src2) \

   383     dest = (src1 > src2) ? (src1 - src2) : (src2 - src1)

   384 NEON_VOP(abd_s8, neon_s8, 4)

   385 NEON_VOP(abd_u8, neon_u8, 4)

   386 NEON_VOP(abd_s16, neon_s16, 2)

   387 NEON_VOP(abd_u16, neon_u16, 2)

   388 NEON_VOP(abd_s32, neon_s32, 1)

   389 NEON_VOP(abd_u32, neon_u32, 1)

   390 #undef NEON_FN

   391 

   392 #define NEON_FN(dest, src1, src2) do { \

   393     int8_t tmp; \

   394     tmp = (int8_t)src2; \

   395     if (abs(tmp) >= sizeof(src1) * 8) { \

   396         dest = 0; \

   397     } else if (tmp < 0) { \

   398         dest = src1 >> -tmp; \

   399     } else { \

   400         dest = src1 << tmp; \

   401     }} while (0)

   402 NEON_VOP(shl_u8, neon_u8, 4)

   403 NEON_VOP(shl_u16, neon_u16, 2)

   404 NEON_VOP(shl_u32, neon_u32, 1)

   405 #undef NEON_FN

   406 

   407 uint64_t HELPER(neon_shl_u64)(uint64_t val, uint64_t shiftop)

   408 {

   409     int8_t shift = (int8_t)shiftop;

   410     if (shift >= 64 || shift <= -64) {

   411         val = 0;

   412     } else if (shift < 0) {

   413         val >>= -shift;

   414     } else {

   415         val <<= shift;

   416     }

   417     return val;

   418 }

   419 

   420 #define NEON_FN(dest, src1, src2) do { \

   421     int8_t tmp; \

   422     tmp = (int8_t)src2; \

   423     if (tmp >= sizeof(src1) * 8) { \

   424         dest = 0; \

   425     } else if (tmp <= -(int) (sizeof(src1) * 8)) { \

   426         dest = src1 >> (sizeof(src1) * 8 - 1); \

   427     } else if (tmp < 0) { \

   428         dest = src1 >> -tmp; \

   429     } else { \

   430         dest = src1 << tmp; \

   431     }} while (0)

   432 NEON_VOP(shl_s8, neon_s8, 4)

   433 NEON_VOP(shl_s16, neon_s16, 2)

   434 NEON_VOP(shl_s32, neon_s32, 1)

   435 #undef NEON_FN

   436 

   437 uint64_t HELPER(neon_shl_s64)(uint64_t valop, uint64_t shiftop)

   438 {

   439     int8_t shift = (int8_t)shiftop;

   440     int64_t val = valop;

   441     if (shift >= 64) {

   442         val = 0;

   443     } else if (shift <= -64) {

   444         val >>= 63;

   445     } else if (shift < 0) {

   446         val >>= -shift;

   447     } else {

   448         val <<= shift;

   449     }

   450     return val;

   451 }

   452 

   453 #define NEON_FN(dest, src1, src2) do { \

   454     int8_t tmp; \

   455     tmp = (int8_t)src2; \

   456     if (tmp >= sizeof(src1) * 8) { \

   457         dest = 0; \

   458     } else if (tmp < -(int) (sizeof(src1) * 8)) { \

   459         dest = src1 >> (sizeof(src1) * 8 - 1); \

   460     } else if (tmp == -(int) (sizeof(src1) * 8)) { \

   461         dest = src1 >> (tmp - 1); \

   462         dest++; \

   463         src2 >>= 1; \

   464     } else if (tmp < 0) { \

   465         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \

   466     } else { \

   467         dest = src1 << tmp; \

   468     }} while (0)

   469 NEON_VOP(rshl_s8, neon_s8, 4)

   470 NEON_VOP(rshl_s16, neon_s16, 2)

   471 NEON_VOP(rshl_s32, neon_s32, 1)

   472 #undef NEON_FN

   473 

   474 uint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop)

   475 {

   476     int8_t shift = (int8_t)shiftop;

   477     int64_t val = valop;

   478     if (shift >= 64) {

   479         val = 0;

   480     } else if (shift < -64) {

   481         val >>= 63;

   482     } else if (shift == -63) {

   483         val >>= 63;

   484         val++;

   485         val >>= 1;

   486     } else if (shift < 0) {

   487         val = (val + ((int64_t)1 << (-1 - shift))) >> -shift;

   488     } else {

   489         val <<= shift;

   490     }

   491     return val;

   492 }

   493 

   494 #define NEON_FN(dest, src1, src2) do { \

   495     int8_t tmp; \

   496     tmp = (int8_t)src2; \

   497     if (abs(tmp) >= sizeof(src1) * 8) { \

   498         dest = 0; \

   499     } else if (tmp == -(int) (sizeof(src1) * 8)) { \

   500         dest = src1 >> (tmp - 1); \

   501     } else if (tmp < 0) { \

   502         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \

   503     } else { \

   504         dest = src1 << tmp; \

   505     }} while (0)

   506 NEON_VOP(rshl_u8, neon_u8, 4)

   507 NEON_VOP(rshl_u16, neon_u16, 2)

   508 NEON_VOP(rshl_u32, neon_u32, 1)

   509 #undef NEON_FN

   510 

   511 uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop)

   512 {

   513     int8_t shift = (uint8_t)shiftop;

   514     if (shift >= 64 || shift < 64) {

   515         val = 0;

   516     } else if (shift == -64) {

   517         /* Rounding a 1-bit result just preserves that bit.  */

   518         val >>= 63;

   519     } if (shift < 0) {

   520         val = (val + ((uint64_t)1 << (-1 - shift))) >> -shift;

   521         val >>= -shift;

   522     } else {

   523         val <<= shift;

   524     }

   525     return val;

   526 }

   527 

   528 #define NEON_FN(dest, src1, src2) do { \

   529     int8_t tmp; \

   530     tmp = (int8_t)src2; \

   531     if (tmp >= sizeof(src1) * 8) { \

   532         if (src1) { \

   533             SET_QC(); \

   534             dest = ~0; \

   535         } else { \

   536             dest = 0; \

   537         } \

   538     } else if (tmp <= -(int) (sizeof(src1) * 8)) { \

   539         dest = 0; \

   540     } else if (tmp < 0) { \

   541         dest = src1 >> -tmp; \

   542     } else { \

   543         dest = src1 << tmp; \

   544         if ((dest >> tmp) != src1) { \

   545             SET_QC(); \

   546             dest = ~0; \

   547         } \

   548     }} while (0)

   549 NEON_VOP_ENV(qshl_u8, neon_u8, 4)

   550 NEON_VOP_ENV(qshl_u16, neon_u16, 2)

   551 NEON_VOP_ENV(qshl_u32, neon_u32, 1)

   552 #undef NEON_FN

   553 

   554 uint64_t HELPER(neon_qshl_u64)(CPUState *env, uint64_t val, uint64_t shiftop)

   555 {

   556     int8_t shift = (int8_t)shiftop;

   557     if (shift >= 64) {

   558         if (val) {

   559             val = ~(uint64_t)0;

   560             SET_QC();

   561         } else {

   562             val = 0;

   563         }

   564     } else if (shift <= -64) {

   565         val = 0;

   566     } else if (shift < 0) {

   567         val >>= -shift;

   568     } else {

   569         uint64_t tmp = val;

   570         val <<= shift;

   571         if ((val >> shift) != tmp) {

   572             SET_QC();

   573             val = ~(uint64_t)0;

   574         }

   575     }

   576     return val;

   577 }

   578 

   579 #define NEON_FN(dest, src1, src2) do { \

   580     int8_t tmp; \

   581     tmp = (int8_t)src2; \

   582     if (tmp >= sizeof(src1) * 8) { \

   583         if (src1) \

   584             SET_QC(); \

   585         dest = src1 >> 31; \

   586     } else if (tmp <= -(int) (sizeof(src1) * 8)) { \

   587         dest = src1 >> 31; \

   588     } else if (tmp < 0) { \

   589         dest = src1 >> -tmp; \

   590     } else { \

   591         dest = src1 << tmp; \

   592         if ((dest >> tmp) != src1) { \

   593             SET_QC(); \

   594             dest = src2 >> 31; \

   595         } \

   596     }} while (0)

   597 NEON_VOP_ENV(qshl_s8, neon_s8, 4)

   598 NEON_VOP_ENV(qshl_s16, neon_s16, 2)

   599 NEON_VOP_ENV(qshl_s32, neon_s32, 1)

   600 #undef NEON_FN

   601 

   602 uint64_t HELPER(neon_qshl_s64)(CPUState *env, uint64_t valop, uint64_t shiftop)

   603 {

   604     int8_t shift = (uint8_t)shiftop;

   605     int64_t val = valop;

   606     if (shift >= 64) {

   607         if (val) {

   608             SET_QC();

   609             val = (val >> 63) & ~SIGNBIT64;

   610         }

   611     } else if (shift <= 64) {

   612         val >>= 63;

   613     } else if (shift < 0) {

   614         val >>= -shift;

   615     } else {

   616         int64_t tmp = val;

   617         val <<= shift;

   618         if ((val >> shift) != tmp) {

   619             SET_QC();

   620             val = (tmp >> 63) ^ ~SIGNBIT64;

   621         }

   622     }

   623     return val;

   624 }

   625 

   626 

   627 /* FIXME: This is wrong.  */

   628 #define NEON_FN(dest, src1, src2) do { \

   629     int8_t tmp; \

   630     tmp = (int8_t)src2; \

   631     if (tmp < 0) { \

   632         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \

   633     } else { \

   634         dest = src1 << tmp; \

   635         if ((dest >> tmp) != src1) { \

   636             SET_QC(); \

   637             dest = ~0; \

   638         } \

   639     }} while (0)

   640 NEON_VOP_ENV(qrshl_u8, neon_u8, 4)

   641 NEON_VOP_ENV(qrshl_u16, neon_u16, 2)

   642 NEON_VOP_ENV(qrshl_u32, neon_u32, 1)

   643 #undef NEON_FN

   644 

   645 uint64_t HELPER(neon_qrshl_u64)(CPUState *env, uint64_t val, uint64_t shiftop)

   646 {

   647     int8_t shift = (int8_t)shiftop;

   648     if (shift < 0) {

   649         val = (val + (1 << (-1 - shift))) >> -shift;

   650     } else { \

   651         uint64_t tmp = val;

   652         val <<= shift;

   653         if ((val >> shift) != tmp) {

   654             SET_QC();

   655             val = ~0;

   656         }

   657     }

   658     return val;

   659 }

   660 

   661 #define NEON_FN(dest, src1, src2) do { \

   662     int8_t tmp; \

   663     tmp = (int8_t)src2; \

   664     if (tmp < 0) { \

   665         dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \

   666     } else { \

   667         dest = src1 << tmp; \

   668         if ((dest >> tmp) != src1) { \

   669             SET_QC(); \

   670             dest = src1 >> 31; \

   671         } \

   672     }} while (0)

   673 NEON_VOP_ENV(qrshl_s8, neon_s8, 4)

   674 NEON_VOP_ENV(qrshl_s16, neon_s16, 2)

   675 NEON_VOP_ENV(qrshl_s32, neon_s32, 1)

   676 #undef NEON_FN

   677 

   678 uint64_t HELPER(neon_qrshl_s64)(CPUState *env, uint64_t valop, uint64_t shiftop)

   679 {

   680     int8_t shift = (uint8_t)shiftop;

   681     int64_t val = valop;

   682 

   683     if (shift < 0) {

   684         val = (val + (1 << (-1 - shift))) >> -shift;

   685     } else {

   686         int64_t tmp = val;;

   687         val <<= shift;

   688         if ((val >> shift) != tmp) {

   689             SET_QC();

   690             val = tmp >> 31;

   691         }

   692     }

   693     return val;

   694 }

   695 

   696 uint32_t HELPER(neon_add_u8)(uint32_t a, uint32_t b)

   697 {

   698     uint32_t mask;

   699     mask = (a ^ b) & 0x80808080u;

   700     a &= ~0x80808080u;

   701     b &= ~0x80808080u;

   702     return (a + b) ^ mask;

   703 }

   704 

   705 uint32_t HELPER(neon_add_u16)(uint32_t a, uint32_t b)

   706 {

   707     uint32_t mask;

   708     mask = (a ^ b) & 0x80008000u;

   709     a &= ~0x80008000u;

   710     b &= ~0x80008000u;

   711     return (a + b) ^ mask;

   712 }

   713 

   714 #define NEON_FN(dest, src1, src2) dest = src1 + src2

   715 NEON_POP(padd_u8, neon_u8, 4)

   716 NEON_POP(padd_u16, neon_u16, 2)

   717 #undef NEON_FN

   718 

   719 #define NEON_FN(dest, src1, src2) dest = src1 - src2

   720 NEON_VOP(sub_u8, neon_u8, 4)

   721 NEON_VOP(sub_u16, neon_u16, 2)

   722 #undef NEON_FN

   723 

   724 #define NEON_FN(dest, src1, src2) dest = src1 * src2

   725 NEON_VOP(mul_u8, neon_u8, 4)

   726 NEON_VOP(mul_u16, neon_u16, 2)

   727 #undef NEON_FN

   728 

   729 /* Polynomial multiplication is like integer multiplication except the

   730    partial products are XORed, not added.  */

   731 uint32_t HELPER(neon_mul_p8)(uint32_t op1, uint32_t op2)

   732 {

   733     uint32_t mask;

   734     uint32_t result;

   735     result = 0;

   736     while (op1) {

   737         mask = 0;

   738         if (op1 & 1)

   739             mask |= 0xff;

   740         if (op1 & (1 << 8))

   741             mask |= (0xff << 8);

   742         if (op1 & (1 << 16))

   743             mask |= (0xff << 16);

   744         if (op1 & (1 << 24))

   745             mask |= (0xff << 24);

   746         result ^= op2 & mask;

   747         op1 = (op1 >> 1) & 0x7f7f7f7f;

   748         op2 = (op2 << 1) & 0xfefefefe;

   749     }

   750     return result;

   751 }

   752 

   753 #define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0

   754 NEON_VOP(tst_u8, neon_u8, 4)

   755 NEON_VOP(tst_u16, neon_u16, 2)

   756 NEON_VOP(tst_u32, neon_u32, 1)

   757 #undef NEON_FN

   758 

   759 #define NEON_FN(dest, src1, src2) dest = (src1 == src2) ? -1 : 0

   760 NEON_VOP(ceq_u8, neon_u8, 4)

   761 NEON_VOP(ceq_u16, neon_u16, 2)

   762 NEON_VOP(ceq_u32, neon_u32, 1)

   763 #undef NEON_FN

   764 

   765 #define NEON_FN(dest, src, dummy) dest = (src < 0) ? -src : src

   766 NEON_VOP1(abs_s8, neon_s8, 4)

   767 NEON_VOP1(abs_s16, neon_s16, 2)

   768 #undef NEON_FN

   769 

   770 /* Count Leading Sign/Zero Bits.  */

   771 static inline int do_clz8(uint8_t x)

   772 {

   773     int n;

   774     for (n = 8; x; n--)

   775         x >>= 1;

   776     return n;

   777 }

   778 

   779 static inline int do_clz16(uint16_t x)

   780 {

   781     int n;

   782     for (n = 16; x; n--)

   783         x >>= 1;

   784     return n;

   785 }

   786 

   787 #define NEON_FN(dest, src, dummy) dest = do_clz8(src)

   788 NEON_VOP1(clz_u8, neon_u8, 4)

   789 #undef NEON_FN

   790 

   791 #define NEON_FN(dest, src, dummy) dest = do_clz16(src)

   792 NEON_VOP1(clz_u16, neon_u16, 2)

   793 #undef NEON_FN

   794 

   795 #define NEON_FN(dest, src, dummy) dest = do_clz8((src < 0) ? ~src : src) - 1

   796 NEON_VOP1(cls_s8, neon_s8, 4)

   797 #undef NEON_FN

   798 

   799 #define NEON_FN(dest, src, dummy) dest = do_clz16((src < 0) ? ~src : src) - 1

   800 NEON_VOP1(cls_s16, neon_s16, 2)

   801 #undef NEON_FN

   802 

   803 uint32_t HELPER(neon_cls_s32)(uint32_t x)

   804 {

   805     int count;

   806     if ((int32_t)x < 0)

   807         x = ~x;

   808     for (count = 32; x; count--)

   809         x = x >> 1;

   810     return count - 1;

   811 }

   812 

   813 /* Bit count.  */

   814 uint32_t HELPER(neon_cnt_u8)(uint32_t x)

   815 {

   816     x = (x & 0x55555555) + ((x >>  1) & 0x55555555);

   817     x = (x & 0x33333333) + ((x >>  2) & 0x33333333);

   818     x = (x & 0x0f0f0f0f) + ((x >>  4) & 0x0f0f0f0f);

   819     return x;

   820 }

   821 

   822 #define NEON_QDMULH16(dest, src1, src2, round) do { \

   823     uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \

   824     if ((tmp ^ (tmp << 1)) & SIGNBIT) { \

   825         SET_QC(); \

   826         tmp = (tmp >> 31) ^ ~SIGNBIT; \

   827     } \

   828     tmp <<= 1; \

   829     if (round) { \

   830         int32_t old = tmp; \

   831         tmp += 1 << 15; \

   832         if ((int32_t)tmp < old) { \

   833             SET_QC(); \

   834             tmp = SIGNBIT - 1; \

   835         } \

   836     } \

   837     dest = tmp >> 16; \

   838     } while(0)

   839 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)

   840 NEON_VOP_ENV(qdmulh_s16, neon_s16, 2)

   841 #undef NEON_FN

   842 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)

   843 NEON_VOP_ENV(qrdmulh_s16, neon_s16, 2)

   844 #undef NEON_FN

   845 #undef NEON_QDMULH16

   846 

   847 #define NEON_QDMULH32(dest, src1, src2, round) do { \

   848     uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \

   849     if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \

   850         SET_QC(); \

   851         tmp = (tmp >> 63) ^ ~SIGNBIT64; \

   852     } else { \

   853         tmp <<= 1; \

   854     } \

   855     if (round) { \

   856         int64_t old = tmp; \

   857         tmp += (int64_t)1 << 31; \

   858         if ((int64_t)tmp < old) { \

   859             SET_QC(); \

   860             tmp = SIGNBIT64 - 1; \

   861         } \

   862     } \

   863     dest = tmp >> 32; \

   864     } while(0)

   865 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)

   866 NEON_VOP_ENV(qdmulh_s32, neon_s32, 1)

   867 #undef NEON_FN

   868 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)

   869 NEON_VOP_ENV(qrdmulh_s32, neon_s32, 1)

   870 #undef NEON_FN

   871 #undef NEON_QDMULH32

   872 

   873 uint32_t HELPER(neon_narrow_u8)(uint64_t x)

   874 {

   875     return (x & 0xffu) | ((x >> 8) & 0xff00u) | ((x >> 16) & 0xff0000u)

   876            | ((x >> 24) & 0xff000000u);

   877 }

   878 

   879 uint32_t HELPER(neon_narrow_u16)(uint64_t x)

   880 {

   881     return (x & 0xffffu) | ((x >> 16) & 0xffff0000u);

   882 }

   883 

   884 uint32_t HELPER(neon_narrow_high_u8)(uint64_t x)

   885 {

   886     return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)

   887             | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);

   888 }

   889 

   890 uint32_t HELPER(neon_narrow_high_u16)(uint64_t x)

   891 {

   892     return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);

   893 }

   894 

   895 uint32_t HELPER(neon_narrow_round_high_u8)(uint64_t x)

   896 {

   897     x &= 0xff80ff80ff80ff80ull;

   898     x += 0x0080008000800080ull;

   899     return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)

   900             | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);

   901 }

   902 

   903 uint32_t HELPER(neon_narrow_round_high_u16)(uint64_t x)

   904 {

   905     x &= 0xffff8000ffff8000ull;

   906     x += 0x0000800000008000ull;

   907     return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);

   908 }

   909 

   910 uint32_t HELPER(neon_narrow_sat_u8)(CPUState *env, uint64_t x)

   911 {

   912     uint16_t s;

   913     uint8_t d;

   914     uint32_t res = 0;

   915 #define SAT8(n) \

   916     s = x >> n; \

   917     if (s > 0xff) { \

   918         d = 0xff; \

   919         SET_QC(); \

   920     } else  { \

   921         d = s; \

   922     } \

   923     res |= (uint32_t)d << (n / 2);

   924 

   925     SAT8(0);

   926     SAT8(16);

   927     SAT8(32);

   928     SAT8(48);

   929 #undef SAT8

   930     return res;

   931 }

   932 

   933 uint32_t HELPER(neon_narrow_sat_s8)(CPUState *env, uint64_t x)

   934 {

   935     int16_t s;

   936     uint8_t d;

   937     uint32_t res = 0;

   938 #define SAT8(n) \

   939     s = x >> n; \

   940     if (s != (int8_t)s) { \

   941         d = (s >> 15) ^ 0x7f; \

   942         SET_QC(); \

   943     } else  { \

   944         d = s; \

   945     } \

   946     res |= (uint32_t)d << (n / 2);

   947 

   948     SAT8(0);

   949     SAT8(16);

   950     SAT8(32);

   951     SAT8(48);

   952 #undef SAT8

   953     return res;

   954 }

   955 

   956 uint32_t HELPER(neon_narrow_sat_u16)(CPUState *env, uint64_t x)

   957 {

   958     uint32_t high;

   959     uint32_t low;

   960     low = x;

   961     if (low > 0xffff) {

   962         low = 0xffff;

   963         SET_QC();

   964     }

   965     high = x >> 32;

   966     if (high > 0xffff) {

   967         high = 0xffff;

   968         SET_QC();

   969     }

   970     return low | (high << 16);

   971 }

   972 

   973 uint32_t HELPER(neon_narrow_sat_s16)(CPUState *env, uint64_t x)

   974 {

   975     int32_t low;

   976     int32_t high;

   977     low = x;

   978     if (low != (int16_t)low) {

   979         low = (low >> 31) ^ 0x7fff;

   980         SET_QC();

   981     }

   982     high = x >> 32;

   983     if (high != (int16_t)high) {

   984         high = (high >> 31) ^ 0x7fff;

   985         SET_QC();

   986     }

   987     return (uint16_t)low | (high << 16);

   988 }

   989 

   990 uint32_t HELPER(neon_narrow_sat_u32)(CPUState *env, uint64_t x)

   991 {

   992     if (x > 0xffffffffu) {

   993         SET_QC();

   994         return 0xffffffffu;

   995     }

   996     return x;

   997 }

   998 

   999 uint32_t HELPER(neon_narrow_sat_s32)(CPUState *env, uint64_t x)

  1000 {

  1001     if ((int64_t)x != (int32_t)x) {

  1002         SET_QC();

  1003         return (x >> 63) ^ 0x7fffffff;

  1004     }

  1005     return x;

  1006 }

  1007 

  1008 uint64_t HELPER(neon_widen_u8)(uint32_t x)

  1009 {

  1010     uint64_t tmp;

  1011     uint64_t ret;

  1012     ret = (uint8_t)x;

  1013     tmp = (uint8_t)(x >> 8);

  1014     ret |= tmp << 16;

  1015     tmp = (uint8_t)(x >> 16);

  1016     ret |= tmp << 32;

  1017     tmp = (uint8_t)(x >> 24);

  1018     ret |= tmp << 48;

  1019     return ret;

  1020 }

  1021 

  1022 uint64_t HELPER(neon_widen_s8)(uint32_t x)

  1023 {

  1024     uint64_t tmp;

  1025     uint64_t ret;

  1026     ret = (uint16_t)(int8_t)x;

  1027     tmp = (uint16_t)(int8_t)(x >> 8);

  1028     ret |= tmp << 16;

  1029     tmp = (uint16_t)(int8_t)(x >> 16);

  1030     ret |= tmp << 32;

  1031     tmp = (uint16_t)(int8_t)(x >> 24);

  1032     ret |= tmp << 48;

  1033     return ret;

  1034 }

  1035 

  1036 uint64_t HELPER(neon_widen_u16)(uint32_t x)

  1037 {

  1038     uint64_t high = (uint16_t)(x >> 16);

  1039     return ((uint16_t)x) | (high << 32);

  1040 }

  1041 

  1042 uint64_t HELPER(neon_widen_s16)(uint32_t x)

  1043 {

  1044     uint64_t high = (int16_t)(x >> 16);

  1045     return ((uint32_t)(int16_t)x) | (high << 32);

  1046 }

  1047 

  1048 uint64_t HELPER(neon_addl_u16)(uint64_t a, uint64_t b)

  1049 {

  1050     uint64_t mask;

  1051     mask = (a ^ b) & 0x8000800080008000ull;

  1052     a &= ~0x8000800080008000ull;

  1053     b &= ~0x8000800080008000ull;

  1054     return (a + b) ^ mask;

  1055 }

  1056 

  1057 uint64_t HELPER(neon_addl_u32)(uint64_t a, uint64_t b)

  1058 {

  1059     uint64_t mask;

  1060     mask = (a ^ b) & 0x8000000080000000ull;

  1061     a &= ~0x8000000080000000ull;

  1062     b &= ~0x8000000080000000ull;

  1063     return (a + b) ^ mask;

  1064 }

  1065 

  1066 uint64_t HELPER(neon_paddl_u16)(uint64_t a, uint64_t b)

  1067 {

  1068     uint64_t tmp;

  1069     uint64_t tmp2;

  1070 

  1071     tmp = a & 0x0000ffff0000ffffull;

  1072     tmp += (a >> 16) & 0x0000ffff0000ffffull;

  1073     tmp2 = b & 0xffff0000ffff0000ull;

  1074     tmp2 += (b << 16) & 0xffff0000ffff0000ull;

  1075     return    ( tmp         & 0xffff)

  1076             | ((tmp  >> 16) & 0xffff0000ull)

  1077             | ((tmp2 << 16) & 0xffff00000000ull)

  1078             | ( tmp2        & 0xffff000000000000ull);

  1079 }

  1080 

  1081 uint64_t HELPER(neon_paddl_u32)(uint64_t a, uint64_t b)

  1082 {

  1083     uint32_t low = a + (a >> 32);

  1084     uint32_t high = b + (b >> 32);

  1085     return low + ((uint64_t)high << 32);

  1086 }

  1087 

  1088 uint64_t HELPER(neon_subl_u16)(uint64_t a, uint64_t b)

  1089 {

  1090     uint64_t mask;

  1091     mask = (a ^ ~b) & 0x8000800080008000ull;

  1092     a |= 0x8000800080008000ull;

  1093     b &= ~0x8000800080008000ull;

  1094     return (a - b) ^ mask;

  1095 }

  1096 

  1097 uint64_t HELPER(neon_subl_u32)(uint64_t a, uint64_t b)

  1098 {

  1099     uint64_t mask;

  1100     mask = (a ^ ~b) & 0x8000000080000000ull;

  1101     a |= 0x8000000080000000ull;

  1102     b &= ~0x8000000080000000ull;

  1103     return (a - b) ^ mask;

  1104 }

  1105 

  1106 uint64_t HELPER(neon_addl_saturate_s32)(CPUState *env, uint64_t a, uint64_t b)

  1107 {

  1108     uint32_t x, y;

  1109     uint32_t low, high;

  1110 

  1111     x = a;

  1112     y = b;

  1113     low = x + y;

  1114     if (((low ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {

  1115         SET_QC();

  1116         low = ((int32_t)x >> 31) ^ ~SIGNBIT;

  1117     }

  1118     x = a >> 32;

  1119     y = b >> 32;

  1120     high = x + y;

  1121     if (((high ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {

  1122         SET_QC();

  1123         high = ((int32_t)x >> 31) ^ ~SIGNBIT;

  1124     }

  1125     return low | ((uint64_t)high << 32);

  1126 }

  1127 

  1128 uint64_t HELPER(neon_addl_saturate_s64)(CPUState *env, uint64_t a, uint64_t b)

  1129 {

  1130     uint64_t result;

  1131 

  1132     result = a + b;

  1133     if (((result ^ a) & SIGNBIT64) && !((a ^ b) & SIGNBIT64)) {

  1134         SET_QC();

  1135         result = ((int64_t)a >> 63) ^ ~SIGNBIT64;