1 /*

     2  * Copyright © 2004 Red Hat, Inc.

     3  * Copyright © 2004 Nicholas Miell

     4  * Copyright © 2005 Trolltech AS

     5  *

     6  * Permission to use, copy, modify, distribute, and sell this software and its

     7  * documentation for any purpose is hereby granted without fee, provided that

     8  * the above copyright notice appear in all copies and that both that

     9  * copyright notice and this permission notice appear in supporting

    10  * documentation, and that the name of Red Hat not be used in advertising or

    11  * publicity pertaining to distribution of the software without specific,

    12  * written prior permission.  Red Hat makes no representations about the

    13  * suitability of this software for any purpose.  It is provided "as is"

    14  * without express or implied warranty.

    15  *

    16  * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS

    17  * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND

    18  * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY

    19  * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES

    20  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN

    21  * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING

    22  * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS

    23  * SOFTWARE.

    24  *

    25  * Author:  Søren Sandmann (sandmann@redhat.com)

    26  * Minor Improvements: Nicholas Miell (nmiell@gmail.com)

    27  * MMX code paths for fbcompose.c by Lars Knoll (lars@trolltech.com) 

    28  *

    29  * Based on work by Owen Taylor

    30  */

    31 //Portions Copyright (c)  2008-2009 Nokia Corporation and/or its subsidiary(-ies). All rights reserved. 

    32 

    33 #ifdef HAVE_CONFIG_H

    34 #include "config.h"

    35 #endif

    36 

    37 #include <liboil/liboil.h>

    38 #include <liboil/liboilfunction.h>

    39  

    40 #include <xmmintrin.h> /* for _mm_shuffle_pi16 and _MM_SHUFFLE */

    41 

    42 typedef uint32_t CARD32;

    43 typedef uint16_t CARD16;

    44 typedef int16_t INT16;

    45 typedef uint8_t CARD8;

    46 typedef uint64_t ullong;

    47 typedef CARD32* PicturePtr;

    48 typedef CARD32* FbBits;

    49 typedef int FbStride;

    50 

    51 

    52 #include "fbmmx.h"

    53 #include "fbpict.h"

    54 

    55 #define CHECKPOINT()

    56 

    57 OIL_DECLARE_CLASS (composite_in_argb);

    58 OIL_DECLARE_CLASS (composite_in_argb_const_src);

    59 OIL_DECLARE_CLASS (composite_in_argb_const_mask);

    60 OIL_DECLARE_CLASS (composite_over_argb);

    61 OIL_DECLARE_CLASS (composite_over_argb_const_src);

    62 OIL_DECLARE_CLASS (composite_add_argb);

    63 OIL_DECLARE_CLASS (composite_add_argb_const_src);

    64 OIL_DECLARE_CLASS (composite_in_over_argb);

    65 OIL_DECLARE_CLASS (composite_in_over_argb_const_src);

    66 OIL_DECLARE_CLASS (composite_in_over_argb_const_mask);

    67 OIL_DECLARE_CLASS (composite_over_u8);

    68 OIL_DECLARE_CLASS (composite_add_u8);

    69 

    70 

    71 /* --------------- MMX code patch for fbcompose.c --------------------- */

    72 

    73 #if 0

    74 static void

    75 mmxCombineMaskU (uint32_t *dest, const uint32_t *src, const uint8_t *mask, int width)

    76 {

    77     const __m64 mmx_0 = _mm_setzero_si64();

    78     const __m64 mmx_4x0080 = (__m64) 0x0080008000800080ULL;

    79     

    80     const uint32_t *end = mask + width;

    81     while (mask < end) {

    82         __m64 a = MmxTo(*mask);

    83         __m64 s = MmxTo(*src);

    84         a = MmxAlpha(a);

    85         MmxMul(s, a);

    86         *dest = MmxFrom(s);

    87         ++src;

    88         ++dest;

    89         ++mask;

    90     }

    91     _mm_empty();

    92 }

    93 #endif

    94 

    95 #ifdef ENABLE_BROKEN_IMPLS

    96 static void

    97 mmxCombineOverU (uint32_t *dest, const uint32_t *src, int width)

    98 {

    99     const __m64 mmx_0 = _mm_setzero_si64();

   100     const __m64 mmx_4x0080 = (__m64) 0x0080008000800080ULL;

   101     const __m64 mmx_4x00ff = (__m64) 0x00ff00ff00ff00ffULL;

   102 

   103     const uint32_t *end = dest + width;

   104 

   105     while (dest < end) {

   106         __m64 x, y, a;

   107         x = MmxTo(*src);

   108         y = MmxTo(*dest);

   109         a = MmxAlpha(x);

   110         a = MmxNegate(a);

   111         MmxMulAdd(y, a, x);

   112         *dest = MmxFrom(y);

   113         ++dest;

   114         ++src;

   115     }

   116     _mm_empty();

   117 }

   118 OIL_DEFINE_IMPL_FULL(mmxCombineOverU, composite_over_argb, OIL_IMPL_FLAG_MMX);

   119 #endif

   120 

   121 #if 0

   122 static FASTCALL void

   123 mmxCombineOverReverseU (CARD32 *dest, const CARD32 *src, int width)

   124 {

   125     const __m64 mmx_0 = _mm_setzero_si64();

   126     const __m64 mmx_4x0080 = (__m64) 0x0080008000800080ULL;

   127     const __m64 mmx_4x00ff = (__m64) 0x00ff00ff00ff00ffULL;

   128 

   129     const CARD32 *end = dest + width;

   130 

   131     while (dest < end) {

   132         __m64 x, y, a;

   133         x = MmxTo(*dest);

   134         y = MmxTo(*src);

   135         a = MmxAlpha(x);

   136         a = MmxNegate(a);

   137         MmxMulAdd(y, a, x);

   138         *dest = MmxFrom(y);

   139         ++dest;

   140         ++src;

   141     }

   142     _mm_empty();

   143 }

   144 #endif

   145 

   146 #if 0

   147 static void

   148 mmxCombineInU (CARD32 *dest, const CARD32 *src, int width)

   149 {

   150     const __m64 mmx_0 = _mm_setzero_si64();

   151     const __m64 mmx_4x0080 = (__m64) 0x0080008000800080ULL;

   152 

   153     const CARD32 *end = dest + width;

   154 

   155     while (dest < end) {

   156         __m64 x, a;

   157         x = MmxTo(*src);

   158         a = MmxTo(*dest);

   159         a = MmxAlpha(a);

   160         MmxMul(x, a);

   161         *dest = MmxFrom(x);

   162         ++dest;

   163         ++src;

   164     }

   165     _mm_empty();

   166 }

   167 #endif

   168 

   169 #if 0

   170 static FASTCALL void

   171 mmxCombineInReverseU (CARD32 *dest, const CARD32 *src, int width)

   172 {

   173     const __m64 mmx_0 = _mm_setzero_si64();

   174     const __m64 mmx_4x0080 = (__m64) 0x0080008000800080ULL;

   175 

   176     const CARD32 *end = dest + width;

   177 

   178     while (dest < end) {

   179         __m64 x, a;

   180         x = MmxTo(*dest);

   181         a = MmxTo(*src);

   182         a = MmxAlpha(a);

   183         MmxMul(x, a);

   184         *dest = MmxFrom(x);

   185         ++dest;

   186         ++src;

   187     }

   188     _mm_empty();

   189 }

   190 #endif

   191 

   192 #if 0

   193 static FASTCALL void

   194 mmxCombineOutU (CARD32 *dest, const CARD32 *src, int width)

   195 {

   196     const __m64 mmx_0 = _mm_setzero_si64();

   197     const __m64 mmx_4x0080 = (__m64) 0x0080008000800080ULL;

   198     const __m64 mmx_4x00ff = (__m64) 0x00ff00ff00ff00ffULL;

   199 

   200     const CARD32 *end = dest + width;

   201 

   202     while (dest < end) {

   203         __m64 x, a;

   204         x = MmxTo(*src);

   205         a = MmxTo(*dest);

   206         a = MmxAlpha(a);

   207         a = MmxNegate(a);

   208         MmxMul(x, a);

   209         *dest = MmxFrom(x);

   210         ++dest;

   211         ++src;

   212     }

   213     _mm_empty();

   214 }

   215 #endif

   216 

   217 #if 0

   218 static FASTCALL void

   219 mmxCombineOutReverseU (CARD32 *dest, const CARD32 *src, int width)

   220 {

   221     const __m64 mmx_0 = _mm_setzero_si64();

   222     const __m64 mmx_4x0080 = (__m64) 0x0080008000800080ULL;

   223     const __m64 mmx_4x00ff = (__m64) 0x00ff00ff00ff00ffULL;

   224 

   225     const CARD32 *end = dest + width;

   226 

   227     while (dest < end) {

   228         __m64 x, a;

   229         x = MmxTo(*dest);

   230         a = MmxTo(*src);

   231         a = MmxAlpha(a);

   232         a = MmxNegate(a);

   233         MmxMul(x, a);

   234         *dest = MmxFrom(x);

   235         ++dest;

   236         ++src;

   237     }

   238     _mm_empty();

   239 }

   240 

   241 static FASTCALL void

   242 mmxCombineAtopU (CARD32 *dest, const CARD32 *src, int width)

   243 {

   244     const __m64 mmx_0 = _mm_setzero_si64();

   245     const __m64 mmx_4x0080 = (__m64) 0x0080008000800080ULL;

   246     const __m64 mmx_4x00ff = (__m64) 0x00ff00ff00ff00ffULL;

   247 

   248     const CARD32 *end = dest + width;

   249 

   250     while (dest < end) {

   251         __m64 s, da, d, sia;

   252         s = MmxTo(*src);

   253         d = MmxTo(*dest);

   254         sia = MmxAlpha(s);

   255         sia = MmxNegate(sia);

   256         da = MmxAlpha(d);

   257         MmxAddMul(s, da, d, sia);

   258         *dest = MmxFrom(s);

   259         ++dest;

   260         ++src;

   261     }

   262     _mm_empty();

   263 }

   264 

   265 static FASTCALL void

   266 mmxCombineAtopReverseU (CARD32 *dest, const CARD32 *src, int width)

   267 {

   268     const __m64 mmx_0 = _mm_setzero_si64();

   269     const __m64 mmx_4x0080 = (__m64) 0x0080008000800080ULL;

   270     const __m64 mmx_4x00ff = (__m64) 0x00ff00ff00ff00ffULL;

   271 

   272     const CARD32 *end;

   273 

   274     end = dest + width;

   275 

   276     while (dest < end) {

   277         __m64 s, dia, d, sa;

   278         s = MmxTo(*src);

   279         d = MmxTo(*dest);

   280         sa = MmxAlpha(s);

   281         dia = MmxAlpha(d);

   282         dia = MmxNegate(dia);

   283         MmxAddMul(s, dia, d, sa);

   284         *dest = MmxFrom(s);

   285         ++dest;

   286         ++src;

   287     }

   288     _mm_empty();

   289 }

   290 

   291 static FASTCALL void

   292 mmxCombineXorU (CARD32 *dest, const CARD32 *src, int width)

   293 {

   294     const __m64 mmx_0 = _mm_setzero_si64();

   295     const __m64 mmx_4x0080 = (__m64) 0x0080008000800080ULL;

   296     const __m64 mmx_4x00ff = (__m64) 0x00ff00ff00ff00ffULL;

   297 

   298     const CARD32 *end = dest + width;

   299 

   300     while (dest < end) {

   301         __m64 s, dia, d, sia;

   302         s = MmxTo(*src);

   303         d = MmxTo(*dest);

   304         sia = MmxAlpha(s);

   305         dia = MmxAlpha(d);

   306         sia = MmxNegate(sia);

   307         dia = MmxNegate(dia);

   308         MmxAddMul(s, dia, d, sia);

   309         *dest = MmxFrom(s);

   310         ++dest;

   311         ++src;

   312     }

   313     _mm_empty();

   314 }

   315 #endif

   316 

   317 static void

   318 mmxCombineAddU (uint32_t *dest, const uint32_t *src, int width)

   319 {

   320     const __m64 mmx_0 = _mm_setzero_si64();

   321 

   322     const uint32_t *end = dest + width;

   323     while (dest < end) {

   324         __m64 s, d;

   325         s = MmxTo(*src);

   326         d = MmxTo(*dest);

   327         s = MmxAdd(s, d);

   328         *dest = MmxFrom(s);

   329         ++dest;

   330         ++src;

   331     }

   332     _mm_empty();

   333 }

   334 OIL_DEFINE_IMPL_FULL(mmxCombineAddU, composite_add_argb, OIL_IMPL_FLAG_MMX | OIL_IMPL_FLAG_SSE);

   335 

   336 #if 0

   337 static FASTCALL void

   338 mmxCombineSaturateU (CARD32 *dest, const CARD32 *src, int width)

   339 {

   340     const __m64 mmx_0 = _mm_setzero_si64();

   341     const __m64 mmx_4x0080 = (__m64) 0x0080008000800080ULL;

   342 

   343     const CARD32 *end = dest + width;

   344     while (dest < end) {

   345         CARD32 s = *src;

   346         CARD32 d = *dest;

   347         __m64 ms = MmxTo(s);

   348         __m64 md = MmxTo(d);

   349         CARD32 sa = s >> 24;

   350         CARD32 da = ~d >> 24;

   351 

   352         if (sa > da) {

   353             __m64 msa = MmxTo(FbIntDiv(da, sa));

   354             msa = MmxAlpha(msa);

   355             MmxMul(ms, msa);

   356         }

   357         MmxAdd(md, ms);

   358         *dest = MmxFrom(md);

   359         ++src;

   360         ++dest;

   361     }

   362     _mm_empty();

   363 }

   364 

   365 

   366 static FASTCALL void

   367 mmxCombineSrcC (CARD32 *dest, CARD32 *src, CARD32 *mask, int width)

   368 {

   369     const __m64 mmx_0 = _mm_setzero_si64();

   370     const __m64 mmx_4x0080 = (__m64) 0x0080008000800080ULL;

   371 

   372     const CARD32 *end = src + width;

   373     while (src < end) {

   374         __m64 a = MmxTo(*mask);

   375         __m64 s = MmxTo(*src);

   376         MmxMul(s, a);

   377         *dest = MmxFrom(s);

   378         ++src;

   379         ++mask;

   380         ++dest;

   381     }

   382     _mm_empty();

   383 }

   384 

   385 static FASTCALL void

   386 mmxCombineOverC (CARD32 *dest, CARD32 *src, CARD32 *mask, int width)

   387 {

   388     const __m64 mmx_0 = _mm_setzero_si64();

   389     const __m64 mmx_4x0080 = (__m64) 0x0080008000800080ULL;

   390     const __m64 mmx_4x00ff = (__m64) 0x00ff00ff00ff00ffULL;

   391     

   392     const CARD32 *end = src + width;

   393     while (src < end) {

   394         __m64 a = MmxTo(*mask);

   395         __m64 s = MmxTo(*src);

   396         __m64 d = MmxTo(*dest);

   397         __m64 sa = MmxAlpha(s);

   398         MmxMul(s, a);

   399         MmxMul(a, sa);

   400         a = MmxNegate(a);

   401         MmxMulAdd(d, a, s);

   402         *dest = MmxFrom(d);

   403         ++src;

   404         ++dest;

   405         ++mask;

   406     }

   407     _mm_empty();

   408 }

   409 

   410 static FASTCALL void

   411 mmxCombineOverReverseC (CARD32 *dest, CARD32 *src, CARD32 *mask, int width)

   412 {

   413     const __m64 mmx_0 = _mm_setzero_si64();

   414     const __m64 mmx_4x0080 = (__m64) 0x0080008000800080ULL;

   415     const __m64 mmx_4x00ff = (__m64) 0x00ff00ff00ff00ffULL;

   416     

   417     const CARD32 *end = src + width;

   418     while (src < end) {

   419         __m64 a = MmxTo(*mask);

   420         __m64 s = MmxTo(*src);

   421         __m64 d = MmxTo(*dest);

   422         __m64 da = MmxAlpha(d);

   423         da = MmxNegate(da);

   424         MmxMul(s, a);

   425         MmxMulAdd(s, da, d);

   426         *dest = MmxFrom(s);

   427         ++src;

   428         ++dest;

   429         ++mask;

   430     }

   431     _mm_empty();

   432 }

   433 

   434 

   435 static FASTCALL void

   436 mmxCombineInC (CARD32 *dest, CARD32 *src, CARD32 *mask, int width)

   437 {

   438     const __m64 mmx_0 = _mm_setzero_si64();

   439     const __m64 mmx_4x0080 = (__m64) 0x0080008000800080ULL;

   440     

   441     const CARD32 *end = src + width;

   442     while (src < end) {

   443         __m64 a = MmxTo(*mask);

   444         __m64 s = MmxTo(*src);

   445         __m64 d = MmxTo(*dest);

   446         __m64 da = MmxAlpha(d);

   447         MmxMul(s, a);

   448         MmxMul(s, da);

   449         *dest = MmxFrom(s);

   450         ++src;

   451         ++dest;

   452         ++mask;

   453     }

   454     _mm_empty();

   455 }

   456 

   457 static FASTCALL void

   458 mmxCombineInReverseC (CARD32 *dest, CARD32 *src, CARD32 *mask, int width)

   459 {

   460     const __m64 mmx_0 = _mm_setzero_si64();

   461     const __m64 mmx_4x0080 = (__m64) 0x0080008000800080ULL;

   462     

   463     const CARD32 *end = src + width;

   464     while (src < end) {

   465         __m64 a = MmxTo(*mask);

   466         __m64 s = MmxTo(*src);

   467         __m64 d = MmxTo(*dest);

   468         __m64 sa = MmxAlpha(s);

   469         MmxMul(a, sa);

   470         MmxMul(d, a);

   471         *dest = MmxFrom(d);

   472         ++src;

   473         ++dest;

   474         ++mask;

   475     }

   476     _mm_empty();

   477 }

   478 

   479 static FASTCALL void

   480 mmxCombineOutC (CARD32 *dest, CARD32 *src, CARD32 *mask, int width)

   481 {

   482     const __m64 mmx_0 = _mm_setzero_si64();

   483     const __m64 mmx_4x0080 = (__m64) 0x0080008000800080ULL;

   484     const __m64 mmx_4x00ff = (__m64) 0x00ff00ff00ff00ffULL;

   485     

   486     const CARD32 *end = src + width;

   487     while (src < end) {

   488         __m64 a = MmxTo(*mask);

   489         __m64 s = MmxTo(*src);

   490         __m64 d = MmxTo(*dest);

   491         __m64 da = MmxAlpha(d);

   492         da = MmxNegate(da);

   493         MmxMul(s, a);

   494         MmxMul(s, da);

   495         *dest = MmxFrom(s);

   496         ++src;

   497         ++dest;

   498         ++mask;

   499     }

   500     _mm_empty();

   501 }

   502 

   503 static FASTCALL void

   504 mmxCombineOutReverseC (CARD32 *dest, CARD32 *src, CARD32 *mask, int width)

   505 {

   506     const __m64 mmx_0 = _mm_setzero_si64();

   507     const __m64 mmx_4x0080 = (__m64) 0x0080008000800080ULL;

   508     const __m64 mmx_4x00ff = (__m64) 0x00ff00ff00ff00ffULL;

   509     

   510     const CARD32 *end = src + width;

   511     while (src < end) {

   512         __m64 a = MmxTo(*mask);

   513         __m64 s = MmxTo(*src);

   514         __m64 d = MmxTo(*dest);

   515         __m64 sa = MmxAlpha(s);

   516         MmxMul(a, sa);

   517         a = MmxNegate(a);

   518         MmxMul(d, a);

   519         *dest = MmxFrom(d);

   520         ++src;

   521         ++dest;

   522         ++mask;

   523     }

   524     _mm_empty();

   525 }

   526 

   527 static FASTCALL void

   528 mmxCombineAtopC (CARD32 *dest, CARD32 *src, CARD32 *mask, int width)

   529 {

   530     const __m64 mmx_0 = _mm_setzero_si64();

   531     const __m64 mmx_4x0080 = (__m64) 0x0080008000800080ULL;

   532     const __m64 mmx_4x00ff = (__m64) 0x00ff00ff00ff00ffULL;

   533     

   534     const CARD32 *end = src + width;

   535     while (src < end) {

   536         __m64 a = MmxTo(*mask);

   537         __m64 s = MmxTo(*src);

   538         __m64 d = MmxTo(*dest);

   539         __m64 da = MmxAlpha(d);

   540         __m64 sa = MmxAlpha(s); 

   541         MmxMul(s, a);

   542         MmxMul(a, sa);

   543         a = MmxNegate(a);

   544         MmxAddMul(d, a, s, da);

   545         *dest = MmxFrom(d);

   546         ++src;

   547         ++dest;

   548         ++mask;

   549     }

   550     _mm_empty();

   551 }

   552 

   553 static FASTCALL void

   554 mmxCombineAtopReverseC (CARD32 *dest, CARD32 *src, CARD32 *mask, int width)

   555 {

   556     const __m64 mmx_0 = _mm_setzero_si64();

   557     const __m64 mmx_4x0080 = (__m64) 0x0080008000800080ULL;

   558     const __m64 mmx_4x00ff = (__m64) 0x00ff00ff00ff00ffULL;

   559     

   560     const CARD32 *end = src + width;

   561     while (src < end) {

   562         __m64 a = MmxTo(*mask);

   563         __m64 s = MmxTo(*src);

   564         __m64 d = MmxTo(*dest);

   565         __m64 da = MmxAlpha(d);

   566         __m64 sa = MmxAlpha(s)

   567         MmxMul(s, a);

   568         MmxMul(a, sa);

   569         da = MmxNegate(da);

   570         MmxAddMul(d, a, s, da);

   571         *dest = MmxFrom(d);

   572         ++src;

   573         ++dest;

   574         ++mask;

   575     }

   576     _mm_empty();

   577 }

   578 

   579 static FASTCALL void

   580 mmxCombineXorC (CARD32 *dest, CARD32 *src, CARD32 *mask, int width)

   581 {

   582     const __m64 mmx_0 = _mm_setzero_si64();

   583     const __m64 mmx_4x0080 = (__m64) 0x0080008000800080ULL;

   584     const __m64 mmx_4x00ff = (__m64) 0x00ff00ff00ff00ffULL;

   585     

   586     const CARD32 *end = src + width;

   587     while (src < end) {

   588         __m64 a = MmxTo(*mask);

   589         __m64 s = MmxTo(*src);

   590         __m64 d = MmxTo(*dest);

   591         __m64 da = MmxAlpha(d);

   592         __m64 sa = MmxAlpha(s);

   593         MmxMul(s, a);

   594         MmxMul(a, sa);

   595         da = MmxNegate(da);

   596         a = MmxNegate(a);

   597         MmxAddMul(d, a, s, da);

   598         *dest = MmxFrom(d);

   599         ++src;

   600         ++dest;

   601         ++mask;

   602     }

   603     _mm_empty();

   604 }

   605 

   606 static FASTCALL void

   607 mmxCombineAddC (CARD32 *dest, CARD32 *src, CARD32 *mask, int width)

   608 {

   609     const __m64 mmx_0 = _mm_setzero_si64();

   610     const __m64 mmx_4x0080 = (__m64) 0x0080008000800080ULL;

   611     

   612     const CARD32 *end = src + width;

   613     while (src < end) {

   614         __m64 a = MmxTo(*mask);

   615         __m64 s = MmxTo(*src);

   616         __m64 d = MmxTo(*dest);

   617         MmxMul(s, a);

   618         d = MmxAdd(s, d);

   619         *dest = MmxFrom(d);

   620         ++src;

   621         ++dest;

   622         ++mask;

   623     }

   624     _mm_empty();

   625 }

   626 

   627 extern FbComposeFunctions composeFunctions;

   628 

   629 void fbComposeSetupMMX(void)

   630 {

   631     /* check if we have MMX support and initialize accordingly */

   632     if (fbHaveMMX()) {

   633         composeFunctions.combineU[PictOpOver] = mmxCombineOverU;

   634         composeFunctions.combineU[PictOpOverReverse] = mmxCombineOverReverseU;

   635         composeFunctions.combineU[PictOpIn] = mmxCombineInU;

   636         composeFunctions.combineU[PictOpInReverse] = mmxCombineInReverseU;

   637         composeFunctions.combineU[PictOpOut] = mmxCombineOutU;

   638         composeFunctions.combineU[PictOpOutReverse] = mmxCombineOutReverseU;

   639         composeFunctions.combineU[PictOpAtop] = mmxCombineAtopU;

   640         composeFunctions.combineU[PictOpAtopReverse] = mmxCombineAtopReverseU;

   641         composeFunctions.combineU[PictOpXor] = mmxCombineXorU;

   642         composeFunctions.combineU[PictOpAdd] = mmxCombineAddU;

   643         composeFunctions.combineU[PictOpSaturate] = mmxCombineSaturateU;

   644 

   645         composeFunctions.combineC[PictOpSrc] = mmxCombineSrcC;

   646         composeFunctions.combineC[PictOpOver] = mmxCombineOverC;

   647         composeFunctions.combineC[PictOpOverReverse] = mmxCombineOverReverseC;

   648         composeFunctions.combineC[PictOpIn] = mmxCombineInC;

   649         composeFunctions.combineC[PictOpInReverse] = mmxCombineInReverseC;

   650         composeFunctions.combineC[PictOpOut] = mmxCombineOutC;

   651         composeFunctions.combineC[PictOpOutReverse] = mmxCombineOutReverseC;

   652         composeFunctions.combineC[PictOpAtop] = mmxCombineAtopC;

   653         composeFunctions.combineC[PictOpAtopReverse] = mmxCombineAtopReverseC;

   654         composeFunctions.combineC[PictOpXor] = mmxCombineXorC;

   655         composeFunctions.combineC[PictOpAdd] = mmxCombineAddC;

   656 

   657         composeFunctions.combineMaskU = mmxCombineMaskU;

   658     } 

   659 }

   660 #endif

   661 

   662 

   663 /* ------------------ MMX code paths called from fbpict.c ----------------------- */

   664 

   665 typedef union {

   666   __m64 m64;

   667   uint64_t ull;

   668 } m64_ull;

   669 

   670 typedef struct

   671 {

   672     m64_ull mmx_4x00ff;

   673     m64_ull mmx_4x0080;

   674     m64_ull mmx_565_rgb;

   675     m64_ull mmx_565_unpack_multiplier;

   676     m64_ull mmx_565_r;

   677     m64_ull mmx_565_g;

   678     m64_ull mmx_565_b;

   679     m64_ull mmx_mask_0;

   680     m64_ull mmx_mask_1;

   681     m64_ull mmx_mask_2;

   682     m64_ull mmx_mask_3;

   683     m64_ull mmx_full_alpha;

   684     m64_ull mmx_ffff0000ffff0000;

   685     m64_ull mmx_0000ffff00000000;

   686     m64_ull mmx_000000000000ffff;

   687 } MMXData;

   688 

   689 static const MMXData c =

   690 {

   691     .mmx_4x00ff.ull =			0x00ff00ff00ff00ffULL,

   692     .mmx_4x0080.ull =			0x0080008000800080ULL,

   693     .mmx_565_rgb.ull =			0x000001f0003f001fULL,

   694     .mmx_565_r.ull =			0x000000f800000000ULL,

   695     .mmx_565_g.ull =			0x0000000000fc0000ULL,

   696     .mmx_565_b.ull =			0x00000000000000f8ULL,

   697     .mmx_mask_0.ull =			0xffffffffffff0000ULL,

   698     .mmx_mask_1.ull =			0xffffffff0000ffffULL,

   699     .mmx_mask_2.ull =			0xffff0000ffffffffULL,

   700     .mmx_mask_3.ull =			0x0000ffffffffffffULL,

   701     .mmx_full_alpha.ull =			0x00ff000000000000ULL,

   702     .mmx_565_unpack_multiplier.ull =	0x0000008404100840ULL,

   703     .mmx_ffff0000ffff0000.ull =		0xffff0000ffff0000ULL,

   704     .mmx_0000ffff00000000.ull =		0x0000ffff00000000ULL,

   705     .mmx_000000000000ffff.ull =		0x000000000000ffffULL,

   706 };

   707 

   708 #define MC(x) ((__m64) c.mmx_##x.m64)

   709 

   710 static __inline__ __m64

   711 shift (__m64 v, int s)

   712 {

   713     if (s > 0)

   714 	return _mm_slli_si64 (v, s);

   715     else if (s < 0)

   716 	return _mm_srli_si64 (v, -s);

   717     else

   718 	return v;

   719 }

   720 

   721 static __inline__ __m64

   722 negate (__m64 mask)

   723 {

   724     return _mm_xor_si64 (mask, MC(4x00ff));

   725 }

   726 

   727 static __inline__ __m64

   728 pix_multiply (__m64 a, __m64 b)

   729 {

   730     __m64 res;

   731     

   732     res = _mm_mullo_pi16 (a, b);

   733     res = _mm_adds_pu16 (res, MC(4x0080));

   734     res = _mm_adds_pu16 (res, _mm_srli_pi16 (res, 8));

   735     res = _mm_srli_pi16 (res, 8);

   736     

   737     return res;

   738 }

   739 

   740 static __inline__ __m64

   741 expand_alpha (__m64 pixel)

   742 {

   743     return _mm_shuffle_pi16 (pixel, _MM_SHUFFLE(3, 3, 3, 3));

   744 }

   745 

   746 static __inline__ __m64

   747 expand_alpha_rev (__m64 pixel)

   748 {

   749     return _mm_shuffle_pi16 (pixel, _MM_SHUFFLE(0, 0, 0, 0));

   750 }    

   751 

   752 static __inline__ __m64

   753 invert_colors (__m64 pixel)

   754 {

   755     return _mm_shuffle_pi16 (pixel, _MM_SHUFFLE(3, 0, 1, 2));

   756 }

   757 

   758 /* Notes about writing mmx code

   759  *

   760  * give memory operands as the second operand. If you give it as the

   761  * first, gcc will first load it into a register, then use that

   762  * register

   763  *

   764  *   ie. use

   765  *

   766  *         _mm_mullo_pi16 (x, mmx_constant);

   767  *

   768  *   not

   769  *

   770  *         _mm_mullo_pi16 (mmx_constant, x);

   771  *

   772  * Also try to minimize dependencies. i.e. when you need a value, try

   773  * to calculate it from a value that was calculated as early as

   774  * possible.

   775  */

   776 

   777 static __inline__ __m64

   778 over (__m64 src, __m64 srca, __m64 dest)

   779 {

   780     return  _mm_adds_pu8 (src, pix_multiply(dest, negate(srca)));

   781 }

   782 

   783 static __inline__ __m64

   784 over_rev_non_pre (__m64 src, __m64 dest)

   785 {

   786     __m64 srca = expand_alpha (src);

   787     __m64 srcfaaa = _mm_or_si64 (srca, MC(full_alpha));

   788     

   789     return over(pix_multiply(invert_colors(src), srcfaaa), srca, dest);

   790 }

   791 

   792 static __inline__ __m64

   793 in (__m64 src,

   794     __m64 mask)

   795 {

   796     return pix_multiply (src, mask);

   797 }

   798 

   799 static __inline__ __m64

   800 in_over (__m64 src,

   801 	 __m64 srca,

   802 	 __m64 mask,

   803 	 __m64 dest)

   804 {

   805     return over(in(src, mask), pix_multiply(srca, mask), dest);

   806 }

   807 

   808 static __inline__ __m64

   809 load8888 (CARD32 v)

   810 {

   811     return _mm_unpacklo_pi8 (_mm_cvtsi32_si64 (v), _mm_setzero_si64());

   812 }

   813 

   814 static __inline__ __m64

   815 pack8888 (__m64 lo, __m64 hi)

   816 {

   817     __m64 r;

   818     r = _mm_packs_pu16 (lo, hi);

   819     return r;

   820 }

   821 

   822 static __inline__ CARD32

   823 store8888 (__m64 v)

   824 {

   825     return _mm_cvtsi64_si32(pack8888(v, _mm_setzero_si64()));

   826 }

   827 

   828 /* Expand 16 bits positioned at @pos (0-3) of a mmx register into

   829  *

   830  *    00RR00GG00BB

   831  * 

   832  * --- Expanding 565 in the low word ---

   833  * 

   834  * m = (m << (32 - 3)) | (m << (16 - 5)) | m;

   835  * m = m & (01f0003f001f);

   836  * m = m * (008404100840);

   837  * m = m >> 8;

   838  * 

   839  * Note the trick here - the top word is shifted by another nibble to

   840  * avoid it bumping into the middle word

   841  */

   842 static __inline__ __m64

   843 expand565 (__m64 pixel, int pos)

   844 {

   845     __m64 p = pixel;

   846     __m64 t1, t2;

   847     

   848     /* move pixel to low 16 bit and zero the rest */

   849     p = shift (shift (p, (3 - pos) * 16), -48); 

   850     

   851     t1 = shift (p, 36 - 11);

   852     t2 = shift (p, 16 - 5);

   853     

   854     p = _mm_or_si64 (t1, p);

   855     p = _mm_or_si64 (t2, p);

   856     p = _mm_and_si64 (p, MC(565_rgb));

   857     

   858     pixel = _mm_mullo_pi16 (p, MC(565_unpack_multiplier));

   859     return _mm_srli_pi16 (pixel, 8);

   860 }

   861 

   862 static __inline__ __m64

   863 expand8888 (__m64 in, int pos)

   864 {

   865     if (pos == 0)

   866 	return _mm_unpacklo_pi8 (in, _mm_setzero_si64());

   867     else

   868 	return _mm_unpackhi_pi8 (in, _mm_setzero_si64());

   869 }

   870 

   871 static __inline__ __m64

   872 pack565 (__m64 pixel, __m64 target, int pos)

   873 {

   874     __m64 p = pixel;

   875     __m64 t = target;

   876     __m64 r, g, b;

   877     

   878     r = _mm_and_si64 (p, MC(565_r));

   879     g = _mm_and_si64 (p, MC(565_g));

   880     b = _mm_and_si64 (p, MC(565_b));

   881     

   882     r = shift (r, - (32 - 8) + pos * 16);

   883     g = shift (g, - (16 - 3) + pos * 16);

   884     b = shift (b, - (0  + 3) + pos * 16);

   885     

   886     if (pos == 0)

   887 	t = _mm_and_si64 (t, MC(mask_0));

   888     else if (pos == 1)

   889 	t = _mm_and_si64 (t, MC(mask_1));

   890     else if (pos == 2)

   891 	t = _mm_and_si64 (t, MC(mask_2));

   892     else if (pos == 3)

   893 	t = _mm_and_si64 (t, MC(mask_3));

   894     

   895     p = _mm_or_si64 (r, t);

   896     p = _mm_or_si64 (g, p);

   897     

   898     return _mm_or_si64 (b, p);

   899 }

   900 

   901 #ifdef ENABLE_BROKEN_IMPLS

   902 /* broken.  See Debian bug #340932 */

   903 static void

   904 fbCompositeSolid_nx8888mmx (uint32_t *dst, uint32_t *src, int w)

   905 {

   906     __m64	vsrc, vsrca;

   907 

   908     vsrc = load8888 (*src);

   909     vsrca = expand_alpha (vsrc);

   910 

   911     while (w && (unsigned long)dst & 7)

   912     {

   913         *dst = store8888(over(vsrc, vsrca, load8888(*dst)));

   914         

   915         w--;

   916         dst++;

   917     }

   918     

   919     while (w >= 2)

   920     {

   921         __m64 vdest;

   922         __m64 dest0, dest1;

   923         

   924         vdest = *(__m64 *)dst;

   925         

   926         dest0 = over(vsrc, vsrca, expand8888(vdest, 0));

   927         dest1 = over(vsrc, vsrca, expand8888(vdest, 1));

   928         

   929         *(__m64 *)dst = pack8888(dest0, dest1);

   930         

   931         dst += 2;

   932         w -= 2;

   933     }

   934     

   935     while (w)

   936     {

   937         *dst = store8888(over(vsrc, vsrca, load8888(*dst)));

   938         

   939         w--;

   940         dst++;

   941     }

   942     

   943     _mm_empty();

   944 }

   945 OIL_DEFINE_IMPL_FULL(fbCompositeSolid_nx8888mmx, composite_over_argb_const_src,

   946     OIL_IMPL_FLAG_MMX| OIL_IMPL_FLAG_MMXEXT);

   947 #endif

   948 

   949 #if 0

   950 void

   951 fbCompositeSolid_nx0565mmx (CARD8	op,

   952 			    PicturePtr pSrc,

   953 			    PicturePtr pMask,

   954 			    PicturePtr pDst,

   955 			    INT16	xSrc,

   956 			    INT16	ySrc,

   957 			    INT16	xMask,

   958 			    INT16	yMask,

   959 			    INT16	xDst,

   960 			    INT16	yDst,

   961 			    CARD16	width,

   962 			    CARD16	height)

   963 {

   964     CARD32	src;

   965     CARD16	*dstLine, *dst;

   966     CARD16	w;

   967     FbStride	dstStride;

   968     __m64	vsrc, vsrca;

   969     

   970     CHECKPOINT();

   971     

   972     fbComposeGetSolid(pSrc, src, pDst->format);

   973     

   974     if (src >> 24 == 0)

   975 	return;

   976     

   977     fbComposeGetStart (pDst, xDst, yDst, CARD16, dstStride, dstLine, 1);

   978     

   979     vsrc = load8888 (src);

   980     vsrca = expand_alpha (vsrc);

   981     

   982     while (height--)

   983     {

   984 	dst = dstLine;

   985 	dstLine += dstStride;

   986 	w = width;

   987 	

   988 	CHECKPOINT();

   989 	

   990 	while (w && (unsigned long)dst & 7)

   991 	{

   992 	    ullong d = *dst;

   993 	    __m64 vdest = expand565 ((__m64)d, 0);

   994 	    vdest = pack565(over(vsrc, vsrca, vdest), vdest, 0);

   995 	    *dst = (ullong)vdest;

   996 	    

   997 	    w--;

   998 	    dst++;

   999 	}

  1000 	

  1001 	while (w >= 4)

  1002 	{

  1003 	    __m64 vdest;

  1004 	    

  1005 	    vdest = *(__m64 *)dst;

  1006 	    

  1007 	    vdest = pack565 (over(vsrc, vsrca, expand565(vdest, 0)), vdest, 0);

  1008 	    vdest = pack565 (over(vsrc, vsrca, expand565(vdest, 1)), vdest, 1);

  1009 	    vdest = pack565 (over(vsrc, vsrca, expand565(vdest, 2)), vdest, 2);

  1010 	    vdest = pack565 (over(vsrc, vsrca, expand565(vdest, 3)), vdest, 3);

  1011 	    

  1012 	    *(__m64 *)dst = vdest;

  1013 	    

  1014 	    dst += 4;

  1015 	    w -= 4;

  1016 	}

  1017 	

  1018 	CHECKPOINT();

  1019 	

  1020 	while (w)

  1021 	{

  1022 	    ullong d = *dst;

  1023 	    __m64 vdest = expand565 ((__m64)d, 0);

  1024 	    vdest = pack565(over(vsrc, vsrca, vdest), vdest, 0);

  1025 	    *dst = (ullong)vdest;

  1026 	    

  1027 	    w--;

  1028 	    dst++;

  1029 	}

  1030     }

  1031     

  1032     _mm_empty();

  1033 }

  1034 #endif

  1035 

  1036 #if 0

  1037 static void

  1038 fbCompositeSolidMask_nx8888x8888Cmmx (uint32_t *dst, uint32_t *src, uint8_t *mask, int w)

  1039 {

  1040     CARD32	src, srca;

  1041     CARD32	*dstLine;

  1042     CARD32	*maskLine;

  1043     FbStride	dstStride, maskStride;

  1044     __m64	vsrc, vsrca;

  1045     

  1046     

  1047     while (twidth && (unsigned long)q & 7)

  1048     {

  1049         CARD32 m = *(CARD32 *)p;

  1050         

  1051         if (m)

  1052         {

  1053             __m64 vdest = load8888(*q);

  1054             vdest = in_over(vsrc, vsrca, load8888(m), vdest);

  1055             *q = (ullong)pack8888(vdest, _mm_setzero_si64());

  1056         }

  1057         

  1058         twidth--;

  1059         p++;

  1060         q++;

  1061     }

  1062     

  1063     while (twidth >= 2)

  1064     {

  1065         CARD32 m0, m1;

  1066         m0 = *p;

  1067         m1 = *(p + 1);

  1068         

  1069         if (m0 | m1)

  1070         {

  1071             __m64 dest0, dest1;

  1072             __m64 vdest = *(__m64 *)q;

  1073             

  1074             dest0 = in_over(vsrc, vsrca, load8888(m0),

  1075                             expand8888 (vdest, 0));

  1076             dest1 = in_over(vsrc, vsrca, load8888(m1),

  1077                             expand8888 (vdest, 1));

  1078             

  1079             *(__m64 *)q = pack8888(dest0, dest1);

  1080         }

  1081         

  1082         p += 2;

  1083         q += 2;

  1084         twidth -= 2;

  1085     }

  1086     

  1087     while (twidth)

  1088     {

  1089         CARD32 m = *(CARD32 *)p;

  1090         

  1091         if (m)

  1092         {

  1093             __m64 vdest = load8888(*q);

  1094             vdest = in_over(vsrc, vsrca, load8888(m), vdest);

  1095             *q = (ullong)pack8888(vdest, _mm_setzero_si64());

  1096         }

  1097         

  1098         twidth--;

  1099         p++;

  1100         q++;

  1101     }

  1102     

  1103     _mm_empty();

  1104 }

  1105 #endif

  1106 

  1107 #if 0

  1108 static void

  1109 fbCompositeSrc_8888x8x8888mmx (uint32_t *dest, uint32_t *src, uint8_t *mask,

  1110     int width)

  1111 {

  1112 

  1113     mask = *maskLine << 24 | *maskLine << 16 | *maskLine << 8 | *maskLine;

  1114     vmask = load8888 (mask);

  1115     srca = MC(4x00ff);

  1116     

  1117     while (height--)

  1118     {

  1119 	dst = dstLine;

  1120 	dstLine += dstStride;

  1121 	src = srcLine;

  1122 	srcLine += srcStride;

  1123 	w = width;

  1124 

  1125 	while (w && (unsigned long)dst & 7)

  1126 	{

  1127 	    __m64 s = load8888 (*src);

  1128 	    __m64 d = load8888 (*dst);

  1129 	    

  1130 	    *dst = (ullong)pack8888 (in_over (s, srca, vmask, d), (__m64)_mm_setzero_si64());

  1131 	    

  1132 	    w--;

  1133 	    dst++;

  1134 	    src++;

  1135 	}