1 /* 

     2  * Support for VIA PadLock Advanced Cryptography Engine (ACE)

     3  * Written by Michal Ludvig <michal@logix.cz>

     4  *            http://www.logix.cz/michal

     5  *

     6  * Big thanks to Andy Polyakov for a help with optimization, 

     7  * assembler fixes, port to MS Windows and a lot of other 

     8  * valuable work on this engine!

     9  */

    10 

    11 /* ====================================================================

    12  * Copyright (c) 1999-2001 The OpenSSL Project.  All rights reserved.

    13  *

    14  * Redistribution and use in source and binary forms, with or without

    15  * modification, are permitted provided that the following conditions

    16  * are met:

    17  *

    18  * 1. Redistributions of source code must retain the above copyright

    19  *    notice, this list of conditions and the following disclaimer.

    20  *

    21  * 2. Redistributions in binary form must reproduce the above copyright

    22  *    notice, this list of conditions and the following disclaimer in

    23  *    the documentation and/or other materials provided with the

    24  *    distribution.

    25  *

    26  * 3. All advertising materials mentioning features or use of this

    27  *    software must display the following acknowledgment:

    28  *    "This product includes software developed by the OpenSSL Project

    29  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"

    30  *

    31  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to

    32  *    endorse or promote products derived from this software without

    33  *    prior written permission. For written permission, please contact

    34  *    licensing@OpenSSL.org.

    35  *

    36  * 5. Products derived from this software may not be called "OpenSSL"

    37  *    nor may "OpenSSL" appear in their names without prior written

    38  *    permission of the OpenSSL Project.

    39  *

    40  * 6. Redistributions of any form whatsoever must retain the following

    41  *    acknowledgment:

    42  *    "This product includes software developed by the OpenSSL Project

    43  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"

    44  *

    45  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY

    46  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

    47  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

    48  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR

    49  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

    50  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT

    51  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;

    52  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)

    53  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,

    54  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)

    55  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED

    56  * OF THE POSSIBILITY OF SUCH DAMAGE.

    57  * ====================================================================

    58  *

    59  * This product includes cryptographic software written by Eric Young

    60  * (eay@cryptsoft.com).  This product includes software written by Tim

    61  * Hudson (tjh@cryptsoft.com).

    62  *

    63  */

    64 

    65 

    66 #include <stdio.h>

    67 #include <string.h>

    68 

    69 #include <openssl/opensslconf.h>

    70 #include <openssl/crypto.h>

    71 #include <openssl/dso.h>

    72 #include <openssl/engine.h>

    73 #include <openssl/evp.h>

    74 #ifndef OPENSSL_NO_AES

    75 #include <openssl/aes.h>

    76 #endif

    77 #include <openssl/rand.h>

    78 #include <openssl/err.h>

    79 

    80 #ifndef OPENSSL_NO_HW

    81 #ifndef OPENSSL_NO_HW_PADLOCK

    82 

    83 /* Attempt to have a single source for both 0.9.7 and 0.9.8 :-) */

    84 #if (OPENSSL_VERSION_NUMBER >= 0x00908000L)

    85 #  ifndef OPENSSL_NO_DYNAMIC_ENGINE

    86 #    define DYNAMIC_ENGINE

    87 #  endif

    88 #elif (OPENSSL_VERSION_NUMBER >= 0x00907000L)

    89 #  ifdef ENGINE_DYNAMIC_SUPPORT

    90 #    define DYNAMIC_ENGINE

    91 #  endif

    92 #else

    93 #  error "Only OpenSSL >= 0.9.7 is supported"

    94 #endif

    95 

    96 /* VIA PadLock AES is available *ONLY* on some x86 CPUs.

    97    Not only that it doesn't exist elsewhere, but it

    98    even can't be compiled on other platforms!

    99  

   100    In addition, because of the heavy use of inline assembler,

   101    compiler choice is limited to GCC and Microsoft C. */

   102 #undef COMPILE_HW_PADLOCK

   103 #if !defined(I386_ONLY) && !defined(OPENSSL_NO_INLINE_ASM)

   104 # if (defined(__GNUC__) && (defined(__i386__) || defined(__i386))) || \

   105      (defined(_MSC_VER) && defined(_M_IX86))

   106 #  define COMPILE_HW_PADLOCK

   107 static ENGINE *ENGINE_padlock (void);

   108 # endif

   109 #endif

   110 

   111 EXPORT_C void ENGINE_load_padlock (void)

   112 {

   113 /* On non-x86 CPUs it just returns. */

   114 #ifdef COMPILE_HW_PADLOCK

   115 	ENGINE *toadd = ENGINE_padlock ();

   116 	if (!toadd) return;

   117 	ENGINE_add (toadd);

   118 	ENGINE_free (toadd);

   119 	ERR_clear_error ();

   120 #endif

   121 }

   122 

   123 #ifdef COMPILE_HW_PADLOCK

   124 /* We do these includes here to avoid header problems on platforms that

   125    do not have the VIA padlock anyway... */

   126 #ifdef _MSC_VER

   127 # include <malloc.h>

   128 # define alloca _alloca

   129 #else

   130 # include <stdlib.h>

   131 #endif

   132 

   133 /* Function for ENGINE detection and control */

   134 static int padlock_available(void);

   135 static int padlock_init(ENGINE *e);

   136 

   137 /* RNG Stuff */

   138 static RAND_METHOD padlock_rand;

   139 

   140 /* Cipher Stuff */

   141 #ifndef OPENSSL_NO_AES

   142 static int padlock_ciphers(ENGINE *e, const EVP_CIPHER **cipher, const int **nids, int nid);

   143 #endif

   144 

   145 /* Engine names */

   146 static const char *padlock_id = "padlock";

   147 static char padlock_name[100];

   148 

   149 /* Available features */

   150 static int padlock_use_ace = 0;	/* Advanced Cryptography Engine */

   151 static int padlock_use_rng = 0;	/* Random Number Generator */

   152 #ifndef OPENSSL_NO_AES

   153 static int padlock_aes_align_required = 1;

   154 #endif

   155 

   156 /* ===== Engine "management" functions ===== */

   157 

   158 /* Prepare the ENGINE structure for registration */

   159 static int

   160 padlock_bind_helper(ENGINE *e)

   161 {

   162 	/* Check available features */

   163 	padlock_available();

   164 

   165 #if 1	/* disable RNG for now, see commentary in vicinity of RNG code */

   166 	padlock_use_rng=0;

   167 #endif

   168 

   169 	/* Generate a nice engine name with available features */

   170 	BIO_snprintf(padlock_name, sizeof(padlock_name),

   171 		"VIA PadLock (%s, %s)", 

   172 		 padlock_use_rng ? "RNG" : "no-RNG",

   173 		 padlock_use_ace ? "ACE" : "no-ACE");

   174 

   175 	/* Register everything or return with an error */ 

   176 	if (!ENGINE_set_id(e, padlock_id) ||

   177 	    !ENGINE_set_name(e, padlock_name) ||

   178 

   179 	    !ENGINE_set_init_function(e, padlock_init) ||

   180 #ifndef OPENSSL_NO_AES

   181 	    (padlock_use_ace && !ENGINE_set_ciphers (e, padlock_ciphers)) ||

   182 #endif

   183 	    (padlock_use_rng && !ENGINE_set_RAND (e, &padlock_rand))) {

   184 		return 0;

   185 	}

   186 

   187 	/* Everything looks good */

   188 	return 1;

   189 }

   190 

   191 /* Constructor */

   192 static ENGINE *

   193 ENGINE_padlock(void)

   194 {

   195 	ENGINE *eng = ENGINE_new();

   196 

   197 	if (!eng) {

   198 		return NULL;

   199 	}

   200 

   201 	if (!padlock_bind_helper(eng)) {

   202 		ENGINE_free(eng);

   203 		return NULL;

   204 	}

   205 

   206 	return eng;

   207 }

   208 

   209 /* Check availability of the engine */

   210 static int

   211 padlock_init(ENGINE *e)

   212 {

   213 	return (padlock_use_rng || padlock_use_ace);

   214 }

   215 

   216 /* This stuff is needed if this ENGINE is being compiled into a self-contained

   217  * shared-library.

   218  */

   219 #ifdef DYNAMIC_ENGINE

   220 static int

   221 padlock_bind_fn(ENGINE *e, const char *id)

   222 {

   223 	if (id && (strcmp(id, padlock_id) != 0)) {

   224 		return 0;

   225 	}

   226 

   227 	if (!padlock_bind_helper(e))  {

   228 		return 0;

   229 	}

   230 

   231 	return 1;

   232 }

   233 

   234 IMPLEMENT_DYNAMIC_CHECK_FN ();

   235 IMPLEMENT_DYNAMIC_BIND_FN (padlock_bind_fn);

   236 #endif /* DYNAMIC_ENGINE */

   237 

   238 /* ===== Here comes the "real" engine ===== */

   239 

   240 #ifndef OPENSSL_NO_AES

   241 /* Some AES-related constants */

   242 #define AES_BLOCK_SIZE		16

   243 #define AES_KEY_SIZE_128	16

   244 #define AES_KEY_SIZE_192	24

   245 #define AES_KEY_SIZE_256	32

   246 

   247 /* Here we store the status information relevant to the 

   248    current context. */

   249 /* BIG FAT WARNING:

   250  * 	Inline assembler in PADLOCK_XCRYPT_ASM()

   251  * 	depends on the order of items in this structure.

   252  * 	Don't blindly modify, reorder, etc!

   253  */

   254 struct padlock_cipher_data

   255 {

   256 	unsigned char iv[AES_BLOCK_SIZE];	/* Initialization vector */

   257 	union {	unsigned int pad[4];

   258 		struct {

   259 			int rounds:4;

   260 			int dgst:1;	/* n/a in C3 */

   261 			int align:1;	/* n/a in C3 */

   262 			int ciphr:1;	/* n/a in C3 */

   263 			unsigned int keygen:1;

   264 			int interm:1;

   265 			unsigned int encdec:1;

   266 			int ksize:2;

   267 		} b;

   268 	} cword;		/* Control word */

   269 	AES_KEY ks;		/* Encryption key */

   270 };

   271 

   272 /*

   273  * Essentially this variable belongs in thread local storage.

   274  * Having this variable global on the other hand can only cause

   275  * few bogus key reloads [if any at all on single-CPU system],

   276  * so we accept the penatly...

   277  */

   278 static volatile struct padlock_cipher_data *padlock_saved_context;

   279 #endif

   280 

   281 /*

   282  * =======================================================

   283  * Inline assembler section(s).

   284  * =======================================================

   285  * Order of arguments is chosen to facilitate Windows port

   286  * using __fastcall calling convention. If you wish to add

   287  * more routines, keep in mind that first __fastcall

   288  * argument is passed in %ecx and second - in %edx.

   289  * =======================================================

   290  */

   291 #if defined(__GNUC__) && __GNUC__>=2

   292 /*

   293  * As for excessive "push %ebx"/"pop %ebx" found all over.

   294  * When generating position-independent code GCC won't let

   295  * us use "b" in assembler templates nor even respect "ebx"

   296  * in "clobber description." Therefore the trouble...

   297  */

   298 

   299 /* Helper function - check if a CPUID instruction

   300    is available on this CPU */

   301 static int

   302 padlock_insn_cpuid_available(void)

   303 {

   304 	int result = -1;

   305 

   306 	/* We're checking if the bit #21 of EFLAGS 

   307 	   can be toggled. If yes = CPUID is available. */

   308 	asm volatile (

   309 		"pushf\n"

   310 		"popl %%eax\n"

   311 		"xorl $0x200000, %%eax\n"

   312 		"movl %%eax, %%ecx\n"

   313 		"andl $0x200000, %%ecx\n"

   314 		"pushl %%eax\n"

   315 		"popf\n"

   316 		"pushf\n"

   317 		"popl %%eax\n"

   318 		"andl $0x200000, %%eax\n"

   319 		"xorl %%eax, %%ecx\n"

   320 		"movl %%ecx, %0\n"

   321 		: "=r" (result) : : "eax", "ecx");

   322 	

   323 	return (result == 0);

   324 }

   325 

   326 /* Load supported features of the CPU to see if

   327    the PadLock is available. */

   328 static int

   329 padlock_available(void)

   330 {

   331 	char vendor_string[16];

   332 	unsigned int eax, edx;

   333 

   334 	/* First check if the CPUID instruction is available at all... */

   335 	if (! padlock_insn_cpuid_available())

   336 		return 0;

   337 

   338 	/* Are we running on the Centaur (VIA) CPU? */

   339 	eax = 0x00000000;

   340 	vendor_string[12] = 0;

   341 	asm volatile (

   342 		"pushl	%%ebx\n"

   343 		"cpuid\n"

   344 		"movl	%%ebx,(%%edi)\n"

   345 		"movl	%%edx,4(%%edi)\n"

   346 		"movl	%%ecx,8(%%edi)\n"

   347 		"popl	%%ebx"

   348 		: "+a"(eax) : "D"(vendor_string) : "ecx", "edx");

   349 	if (strcmp(vendor_string, "CentaurHauls") != 0)

   350 		return 0;

   351 

   352 	/* Check for Centaur Extended Feature Flags presence */

   353 	eax = 0xC0000000;

   354 	asm volatile ("pushl %%ebx; cpuid; popl	%%ebx"

   355 		: "+a"(eax) : : "ecx", "edx");

   356 	if (eax < 0xC0000001)

   357 		return 0;

   358 

   359 	/* Read the Centaur Extended Feature Flags */

   360 	eax = 0xC0000001;

   361 	asm volatile ("pushl %%ebx; cpuid; popl %%ebx"

   362 		: "+a"(eax), "=d"(edx) : : "ecx");

   363 

   364 	/* Fill up some flags */

   365 	padlock_use_ace = ((edx & (0x3<<6)) == (0x3<<6));

   366 	padlock_use_rng = ((edx & (0x3<<2)) == (0x3<<2));

   367 

   368 	return padlock_use_ace + padlock_use_rng;

   369 }

   370 

   371 #ifndef OPENSSL_NO_AES

   372 /* Our own htonl()/ntohl() */

   373 static inline void

   374 padlock_bswapl(AES_KEY *ks)

   375 {

   376 	size_t i = sizeof(ks->rd_key)/sizeof(ks->rd_key[0]);

   377 	unsigned int *key = ks->rd_key;

   378 

   379 	while (i--) {

   380 		asm volatile ("bswapl %0" : "+r"(*key));

   381 		key++;

   382 	}

   383 }

   384 #endif

   385 

   386 /* Force key reload from memory to the CPU microcode.

   387    Loading EFLAGS from the stack clears EFLAGS[30] 

   388    which does the trick. */

   389 static inline void

   390 padlock_reload_key(void)

   391 {

   392 	asm volatile ("pushfl; popfl");

   393 }

   394 

   395 #ifndef OPENSSL_NO_AES

   396 /*

   397  * This is heuristic key context tracing. At first one

   398  * believes that one should use atomic swap instructions,

   399  * but it's not actually necessary. Point is that if

   400  * padlock_saved_context was changed by another thread

   401  * after we've read it and before we compare it with cdata,

   402  * our key *shall* be reloaded upon thread context switch

   403  * and we are therefore set in either case...

   404  */

   405 static inline void

   406 padlock_verify_context(struct padlock_cipher_data *cdata)

   407 {

   408 	asm volatile (

   409 	"pushfl\n"

   410 "	btl	$30,(%%esp)\n"

   411 "	jnc	1f\n"

   412 "	cmpl	%2,%1\n"

   413 "	je	1f\n"

   414 "	popfl\n"

   415 "	subl	$4,%%esp\n"

   416 "1:	addl	$4,%%esp\n"

   417 "	movl	%2,%0"

   418 	:"+m"(padlock_saved_context)

   419 	: "r"(padlock_saved_context), "r"(cdata) : "cc");

   420 }

   421 

   422 /* Template for padlock_xcrypt_* modes */

   423 /* BIG FAT WARNING: 

   424  * 	The offsets used with 'leal' instructions

   425  * 	describe items of the 'padlock_cipher_data'

   426  * 	structure.

   427  */

   428 #define PADLOCK_XCRYPT_ASM(name,rep_xcrypt)	\

   429 static inline void *name(size_t cnt,		\

   430 	struct padlock_cipher_data *cdata,	\

   431 	void *out, const void *inp) 		\

   432 {	void *iv; 				\

   433 	asm volatile ( "pushl	%%ebx\n"	\

   434 		"	leal	16(%0),%%edx\n"	\

   435 		"	leal	32(%0),%%ebx\n"	\

   436 			rep_xcrypt "\n"		\

   437 		"	popl	%%ebx"		\

   438 		: "=a"(iv), "=c"(cnt), "=D"(out), "=S"(inp) \

   439 		: "0"(cdata), "1"(cnt), "2"(out), "3"(inp) \

   440 		: "edx", "cc", "memory");	\

   441 	return iv;				\

   442 }

   443 

   444 /* Generate all functions with appropriate opcodes */

   445 PADLOCK_XCRYPT_ASM(padlock_xcrypt_ecb, ".byte 0xf3,0x0f,0xa7,0xc8")	/* rep xcryptecb */

   446 PADLOCK_XCRYPT_ASM(padlock_xcrypt_cbc, ".byte 0xf3,0x0f,0xa7,0xd0")	/* rep xcryptcbc */

   447 PADLOCK_XCRYPT_ASM(padlock_xcrypt_cfb, ".byte 0xf3,0x0f,0xa7,0xe0")	/* rep xcryptcfb */

   448 PADLOCK_XCRYPT_ASM(padlock_xcrypt_ofb, ".byte 0xf3,0x0f,0xa7,0xe8")	/* rep xcryptofb */

   449 #endif

   450 

   451 /* The RNG call itself */

   452 static inline unsigned int

   453 padlock_xstore(void *addr, unsigned int edx_in)

   454 {

   455 	unsigned int eax_out;

   456 

   457 	asm volatile (".byte 0x0f,0xa7,0xc0"	/* xstore */

   458 	    : "=a"(eax_out),"=m"(*(unsigned *)addr)

   459 	    : "D"(addr), "d" (edx_in)

   460 	    );

   461 

   462 	return eax_out;

   463 }

   464 

   465 /* Why not inline 'rep movsd'? I failed to find information on what

   466  * value in Direction Flag one can expect and consequently have to

   467  * apply "better-safe-than-sorry" approach and assume "undefined."

   468  * I could explicitly clear it and restore the original value upon

   469  * return from padlock_aes_cipher, but it's presumably too much

   470  * trouble for too little gain...

   471  *

   472  * In case you wonder 'rep xcrypt*' instructions above are *not*

   473  * affected by the Direction Flag and pointers advance toward

   474  * larger addresses unconditionally.

   475  */ 

   476 static inline unsigned char *

   477 padlock_memcpy(void *dst,const void *src,size_t n)

   478 {

   479 	long       *d=dst;

   480 	const long *s=src;

   481 

   482 	n /= sizeof(*d);

   483 	do { *d++ = *s++; } while (--n);

   484 

   485 	return dst;

   486 }

   487 

   488 #elif defined(_MSC_VER)

   489 /*

   490  * Unlike GCC these are real functions. In order to minimize impact

   491  * on performance we adhere to __fastcall calling convention in

   492  * order to get two first arguments passed through %ecx and %edx.

   493  * Which kind of suits very well, as instructions in question use

   494  * both %ecx and %edx as input:-)

   495  */

   496 #define REP_XCRYPT(code)		\

   497 	_asm _emit 0xf3			\

   498 	_asm _emit 0x0f _asm _emit 0xa7	\

   499 	_asm _emit code

   500 

   501 /* BIG FAT WARNING: 

   502  * 	The offsets used with 'lea' instructions

   503  * 	describe items of the 'padlock_cipher_data'

   504  * 	structure.

   505  */

   506 #define PADLOCK_XCRYPT_ASM(name,code)	\

   507 static void * __fastcall 		\

   508 	name (size_t cnt, void *cdata,	\

   509 	void *outp, const void *inp)	\

   510 {	_asm	mov	eax,edx		\

   511 	_asm	lea	edx,[eax+16]	\

   512 	_asm	lea	ebx,[eax+32]	\

   513 	_asm	mov	edi,outp	\

   514 	_asm	mov	esi,inp		\

   515 	REP_XCRYPT(code)		\

   516 }

   517 

   518 PADLOCK_XCRYPT_ASM(padlock_xcrypt_ecb,0xc8)

   519 PADLOCK_XCRYPT_ASM(padlock_xcrypt_cbc,0xd0)

   520 PADLOCK_XCRYPT_ASM(padlock_xcrypt_cfb,0xe0)

   521 PADLOCK_XCRYPT_ASM(padlock_xcrypt_ofb,0xe8)

   522 

   523 static int __fastcall

   524 padlock_xstore(void *outp,unsigned int code)

   525 {	_asm	mov	edi,ecx

   526 	_asm _emit 0x0f _asm _emit 0xa7 _asm _emit 0xc0

   527 }

   528 

   529 static void __fastcall

   530 padlock_reload_key(void)

   531 {	_asm pushfd _asm popfd		}

   532 

   533 static void __fastcall

   534 padlock_verify_context(void *cdata)

   535 {	_asm	{

   536 		pushfd

   537 		bt	DWORD PTR[esp],30

   538 		jnc	skip

   539 		cmp	ecx,padlock_saved_context

   540 		je	skip

   541 		popfd

   542 		sub	esp,4

   543 	skip:	add	esp,4

   544 		mov	padlock_saved_context,ecx

   545 		}

   546 }

   547 

   548 static int

   549 padlock_available(void)

   550 {	_asm	{

   551 		pushfd

   552 		pop	eax

   553 		mov	ecx,eax

   554 		xor	eax,1<<21

   555 		push	eax

   556 		popfd

   557 		pushfd

   558 		pop	eax

   559 		xor	eax,ecx

   560 		bt	eax,21

   561 		jnc	noluck

   562 		mov	eax,0

   563 		cpuid

   564 		xor	eax,eax

   565 		cmp	ebx,'tneC'

   566 		jne	noluck

   567 		cmp	edx,'Hrua'

   568 		jne	noluck

   569 		cmp	ecx,'slua'

   570 		jne	noluck

   571 		mov	eax,0xC0000000

   572 		cpuid

   573 		mov	edx,eax

   574 		xor	eax,eax

   575 		cmp	edx,0xC0000001

   576 		jb	noluck

   577 		mov	eax,0xC0000001

   578 		cpuid

   579 		xor	eax,eax

   580 		bt	edx,6

   581 		jnc	skip_a

   582 		bt	edx,7

   583 		jnc	skip_a

   584 		mov	padlock_use_ace,1

   585 		inc	eax

   586 	skip_a:	bt	edx,2

   587 		jnc	skip_r

   588 		bt	edx,3

   589 		jnc	skip_r

   590 		mov	padlock_use_rng,1

   591 		inc	eax

   592 	skip_r:

   593 	noluck:

   594 		}

   595 }

   596 

   597 static void __fastcall

   598 padlock_bswapl(void *key)

   599 {	_asm	{

   600 		pushfd

   601 		cld

   602 		mov	esi,ecx

   603 		mov	edi,ecx

   604 		mov	ecx,60

   605 	up:	lodsd

   606 		bswap	eax

   607 		stosd

   608 		loop	up

   609 		popfd

   610 		}

   611 }

   612 

   613 /* MS actually specifies status of Direction Flag and compiler even

   614  * manages to compile following as 'rep movsd' all by itself...

   615  */

   616 #define padlock_memcpy(o,i,n) ((unsigned char *)memcpy((o),(i),(n)&~3U))

   617 #endif

   618 

   619 /* ===== AES encryption/decryption ===== */

   620 #ifndef OPENSSL_NO_AES

   621 

   622 #if defined(NID_aes_128_cfb128) && ! defined (NID_aes_128_cfb)

   623 #define NID_aes_128_cfb	NID_aes_128_cfb128

   624 #endif

   625 

   626 #if defined(NID_aes_128_ofb128) && ! defined (NID_aes_128_ofb)

   627 #define NID_aes_128_ofb	NID_aes_128_ofb128

   628 #endif

   629 

   630 #if defined(NID_aes_192_cfb128) && ! defined (NID_aes_192_cfb)

   631 #define NID_aes_192_cfb	NID_aes_192_cfb128

   632 #endif

   633 

   634 #if defined(NID_aes_192_ofb128) && ! defined (NID_aes_192_ofb)

   635 #define NID_aes_192_ofb	NID_aes_192_ofb128

   636 #endif

   637 

   638 #if defined(NID_aes_256_cfb128) && ! defined (NID_aes_256_cfb)

   639 #define NID_aes_256_cfb	NID_aes_256_cfb128

   640 #endif

   641 

   642 #if defined(NID_aes_256_ofb128) && ! defined (NID_aes_256_ofb)

   643 #define NID_aes_256_ofb	NID_aes_256_ofb128

   644 #endif

   645 

   646 /* List of supported ciphers. */

   647 static int padlock_cipher_nids[] = {

   648 	NID_aes_128_ecb,

   649 	NID_aes_128_cbc,

   650 	NID_aes_128_cfb,

   651 	NID_aes_128_ofb,

   652 

   653 	NID_aes_192_ecb,

   654 	NID_aes_192_cbc,

   655 #if 0

   656 	NID_aes_192_cfb,	/* FIXME: AES192/256 CFB/OFB don't work. */

   657 	NID_aes_192_ofb,

   658 #endif

   659 

   660 	NID_aes_256_ecb,

   661 	NID_aes_256_cbc,

   662 #if 0

   663 	NID_aes_256_cfb,

   664 	NID_aes_256_ofb,

   665 #endif

   666 };

   667 static int padlock_cipher_nids_num = (sizeof(padlock_cipher_nids)/

   668 				      sizeof(padlock_cipher_nids[0]));

   669 

   670 /* Function prototypes ... */

   671 static int padlock_aes_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,

   672 				const unsigned char *iv, int enc);

   673 static int padlock_aes_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,

   674 			      const unsigned char *in, size_t nbytes);

   675 

   676 #define NEAREST_ALIGNED(ptr) ( (unsigned char *)(ptr) +		\

   677 	( (0x10 - ((size_t)(ptr) & 0x0F)) & 0x0F )	)

   678 #define ALIGNED_CIPHER_DATA(ctx) ((struct padlock_cipher_data *)\

   679 	NEAREST_ALIGNED(ctx->cipher_data))

   680 

   681 #define EVP_CIPHER_block_size_ECB	AES_BLOCK_SIZE

   682 #define EVP_CIPHER_block_size_CBC	AES_BLOCK_SIZE

   683 #define EVP_CIPHER_block_size_OFB	1

   684 #define EVP_CIPHER_block_size_CFB	1

   685 /* Declaring so many ciphers by hand would be a pain.

   686    Instead introduce a bit of preprocessor magic :-) */

   687 #define	DECLARE_AES_EVP(ksize,lmode,umode)	\

   688 static const EVP_CIPHER padlock_aes_##ksize##_##lmode = {	\

   689 	NID_aes_##ksize##_##lmode,		\

   690 	EVP_CIPHER_block_size_##umode,	\

   691 	AES_KEY_SIZE_##ksize,		\

   692 	AES_BLOCK_SIZE,			\

   693 	0 | EVP_CIPH_##umode##_MODE,	\

   694 	padlock_aes_init_key,		\

   695 	padlock_aes_cipher,		\

   696 	NULL,				\

   697 	sizeof(struct padlock_cipher_data) + 16,	\

   698 	EVP_CIPHER_set_asn1_iv,		\

   699 	EVP_CIPHER_get_asn1_iv,		\

   700 	NULL,				\

   701 	NULL				\

   702 }

   703 

   704 DECLARE_AES_EVP(128,ecb,ECB);

   705 DECLARE_AES_EVP(128,cbc,CBC);

   706 DECLARE_AES_EVP(128,cfb,CFB);

   707 DECLARE_AES_EVP(128,ofb,OFB);

   708 

   709 DECLARE_AES_EVP(192,ecb,ECB);

   710 DECLARE_AES_EVP(192,cbc,CBC);

   711 DECLARE_AES_EVP(192,cfb,CFB);

   712 DECLARE_AES_EVP(192,ofb,OFB);

   713 

   714 DECLARE_AES_EVP(256,ecb,ECB);

   715 DECLARE_AES_EVP(256,cbc,CBC);

   716 DECLARE_AES_EVP(256,cfb,CFB);

   717 DECLARE_AES_EVP(256,ofb,OFB);

   718 

   719 static int

   720 padlock_ciphers (ENGINE *e, const EVP_CIPHER **cipher, const int **nids, int nid)

   721 {

   722 	/* No specific cipher => return a list of supported nids ... */

   723 	if (!cipher) {

   724 		*nids = padlock_cipher_nids;

   725 		return padlock_cipher_nids_num;

   726 	}

   727 

   728 	/* ... or the requested "cipher" otherwise */

   729 	switch (nid) {

   730 	  case NID_aes_128_ecb:

   731 	    *cipher = &padlock_aes_128_ecb;

   732 	    break;

   733 	  case NID_aes_128_cbc:

   734 	    *cipher = &padlock_aes_128_cbc;

   735 	    break;

   736 	  case NID_aes_128_cfb:

   737 	    *cipher = &padlock_aes_128_cfb;

   738 	    break;

   739 	  case NID_aes_128_ofb:

   740 	    *cipher = &padlock_aes_128_ofb;

   741 	    break;

   742 

   743 	  case NID_aes_192_ecb:

   744 	    *cipher = &padlock_aes_192_ecb;

   745 	    break;

   746 	  case NID_aes_192_cbc:

   747 	    *cipher = &padlock_aes_192_cbc;

   748 	    break;

   749 	  case NID_aes_192_cfb:

   750 	    *cipher = &padlock_aes_192_cfb;

   751 	    break;

   752 	  case NID_aes_192_ofb:

   753 	    *cipher = &padlock_aes_192_ofb;

   754 	    break;

   755 

   756 	  case NID_aes_256_ecb:

   757 	    *cipher = &padlock_aes_256_ecb;

   758 	    break;

   759 	  case NID_aes_256_cbc:

   760 	    *cipher = &padlock_aes_256_cbc;

   761 	    break;

   762 	  case NID_aes_256_cfb:

   763 	    *cipher = &padlock_aes_256_cfb;

   764 	    break;

   765 	  case NID_aes_256_ofb:

   766 	    *cipher = &padlock_aes_256_ofb;

   767 	    break;

   768 

   769 	  default:

   770 	    /* Sorry, we don't support this NID */

   771 	    *cipher = NULL;

   772 	    return 0;

   773 	}

   774 

   775 	return 1;

   776 }

   777 

   778 /* Prepare the encryption key for PadLock usage */

   779 static int

   780 padlock_aes_init_key (EVP_CIPHER_CTX *ctx, const unsigned char *key,

   781 		      const unsigned char *iv, int enc)

   782 {

   783 	struct padlock_cipher_data *cdata;

   784 	int key_len = EVP_CIPHER_CTX_key_length(ctx) * 8;

   785 

   786 	if (key==NULL) return 0;	/* ERROR */

   787 

   788 	cdata = ALIGNED_CIPHER_DATA(ctx);

   789 	memset(cdata, 0, sizeof(struct padlock_cipher_data));

   790 

   791 	/* Prepare Control word. */

   792 	if (EVP_CIPHER_CTX_mode(ctx) == EVP_CIPH_OFB_MODE)

   793 		cdata->cword.b.encdec = 0;

   794 	else

   795 	cdata->cword.b.encdec = (ctx->encrypt == 0);

   796 	cdata->cword.b.rounds = 10 + (key_len - 128) / 32;

   797 	cdata->cword.b.ksize = (key_len - 128) / 64;

   798 

   799 	switch(key_len) {

   800 		case 128:

   801 			/* PadLock can generate an extended key for

   802 			   AES128 in hardware */

   803 			memcpy(cdata->ks.rd_key, key, AES_KEY_SIZE_128);

   804 			cdata->cword.b.keygen = 0;

   805 			break;

   806 

   807 		case 192:

   808 		case 256:

   809 			/* Generate an extended AES key in software.

   810 			   Needed for AES192/AES256 */

   811 			/* Well, the above applies to Stepping 8 CPUs

   812 			   and is listed as hardware errata. They most

   813 			   likely will fix it at some point and then

   814 			   a check for stepping would be due here. */

   815 			if (EVP_CIPHER_CTX_mode(ctx) == EVP_CIPH_CFB_MODE ||

   816 			    EVP_CIPHER_CTX_mode(ctx) == EVP_CIPH_OFB_MODE ||

   817 			    enc)

   818 				AES_set_encrypt_key(key, key_len, &cdata->ks);

   819 			else

   820 				AES_set_decrypt_key(key, key_len, &cdata->ks);

   821 #ifndef AES_ASM

   822 			/* OpenSSL C functions use byte-swapped extended key. */

   823 			padlock_bswapl(&cdata->ks);

   824 #endif

   825 			cdata->cword.b.keygen = 1;

   826 			break;

   827 

   828 		default:

   829 			/* ERROR */

   830 			return 0;

   831 	}

   832 

   833 	/*

   834 	 * This is done to cover for cases when user reuses the

   835 	 * context for new key. The catch is that if we don't do

   836 	 * this, padlock_eas_cipher might proceed with old key...

   837 	 */

   838 	padlock_reload_key ();

   839 

   840 	return 1;

   841 }

   842 

   843 /* 

   844  * Simplified version of padlock_aes_cipher() used when

   845  * 1) both input and output buffers are at aligned addresses.

   846  * or when

   847  * 2) running on a newer CPU that doesn't require aligned buffers.

   848  */

   849 static int

   850 padlock_aes_cipher_omnivorous(EVP_CIPHER_CTX *ctx, unsigned char *out_arg,

   851 		const unsigned char *in_arg, size_t nbytes)

   852 {

   853 	struct padlock_cipher_data *cdata;

   854 	void  *iv;

   855 

   856 	cdata = ALIGNED_CIPHER_DATA(ctx);

   857 	padlock_verify_context(cdata);

   858 

   859 	switch (EVP_CIPHER_CTX_mode(ctx)) {

   860 	case EVP_CIPH_ECB_MODE:

   861 		padlock_xcrypt_ecb(nbytes/AES_BLOCK_SIZE, cdata, out_arg, in_arg);

   862 		break;

   863 

   864 	case EVP_CIPH_CBC_MODE:

   865 		memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);

   866 		iv = padlock_xcrypt_cbc(nbytes/AES_BLOCK_SIZE, cdata, out_arg, in_arg);

   867 		memcpy(ctx->iv, iv, AES_BLOCK_SIZE);

   868 		break;

   869 

   870 	case EVP_CIPH_CFB_MODE:

   871 		memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);

   872 		iv = padlock_xcrypt_cfb(nbytes/AES_BLOCK_SIZE, cdata, out_arg, in_arg);

   873 		memcpy(ctx->iv, iv, AES_BLOCK_SIZE);

   874 		break;

   875 

   876 	case EVP_CIPH_OFB_MODE:

   877 		memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);

   878 		padlock_xcrypt_ofb(nbytes/AES_BLOCK_SIZE, cdata, out_arg, in_arg);

   879 		memcpy(ctx->iv, cdata->iv, AES_BLOCK_SIZE);

   880 		break;

   881 

   882 	default:

   883 		return 0;

   884 	}

   885 

   886 	memset(cdata->iv, 0, AES_BLOCK_SIZE);

   887 

   888 	return 1;

   889 }

   890 

   891 #ifndef  PADLOCK_CHUNK

   892 # define PADLOCK_CHUNK	512	/* Must be a power of 2 larger than 16 */

   893 #endif

   894 #if PADLOCK_CHUNK<16 || PADLOCK_CHUNK&(PADLOCK_CHUNK-1)

   895 # error "insane PADLOCK_CHUNK..."

   896 #endif

   897 

   898 /* Re-align the arguments to 16-Bytes boundaries and run the 

   899    encryption function itself. This function is not AES-specific. */

   900 static int

   901 padlock_aes_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out_arg,

   902 		   const unsigned char *in_arg, size_t nbytes)

   903 {

   904 	struct padlock_cipher_data *cdata;

   905 	const  void *inp;

   906 	unsigned char  *out;

   907 	void  *iv;

   908 	int    inp_misaligned, out_misaligned, realign_in_loop;

   909 	size_t chunk, allocated=0;

   910 

   911 	/* ctx->num is maintained in byte-oriented modes,

   912 	   such as CFB and OFB... */

   913 	if ((chunk = ctx->num)) { /* borrow chunk variable */

   914 		unsigned char *ivp=ctx->iv;

   915 

   916 		switch (EVP_CIPHER_CTX_mode(ctx)) {

   917 		case EVP_CIPH_CFB_MODE:

   918 			if (chunk >= AES_BLOCK_SIZE)

   919 				return 0; /* bogus value */

   920 

   921 			if (ctx->encrypt)

   922 				while (chunk<AES_BLOCK_SIZE && nbytes!=0) {

   923 					ivp[chunk] = *(out_arg++) = *(in_arg++) ^ ivp[chunk];

   924 					chunk++, nbytes--;

   925 				}

   926 			else	while (chunk<AES_BLOCK_SIZE && nbytes!=0) {

   927 					unsigned char c = *(in_arg++);

   928 					*(out_arg++) = c ^ ivp[chunk];

   929 					ivp[chunk++] = c, nbytes--;

   930 				}

   931 

   932 			ctx->num = chunk%AES_BLOCK_SIZE;

   933 			break;

   934 		case EVP_CIPH_OFB_MODE:

   935 			if (chunk >= AES_BLOCK_SIZE)

   936 				return 0; /* bogus value */

   937 

   938 			while (chunk<AES_BLOCK_SIZE && nbytes!=0) {

   939 				*(out_arg++) = *(in_arg++) ^ ivp[chunk];

   940 				chunk++, nbytes--;

   941 			}

   942 

   943 			ctx->num = chunk%AES_BLOCK_SIZE;

   944 			break;

   945 		}

   946 	}

   947 

   948 	if (nbytes == 0)

   949 		return 1;

   950 #if 0

   951 	if (nbytes % AES_BLOCK_SIZE)

   952 		return 0; /* are we expected to do tail processing? */

   953 #else

   954 	/* nbytes is always multiple of AES_BLOCK_SIZE in ECB and CBC

   955 	   modes and arbitrary value in byte-oriented modes, such as

   956 	   CFB and OFB... */

   957 #endif

   958 

   959 	/* VIA promises CPUs that won't require alignment in the future.

   960 	   For now padlock_aes_align_required is initialized to 1 and

   961 	   the condition is never met... */

   962 	/* C7 core is capable to manage unaligned input in non-ECB[!]

   963 	   mode, but performance penalties appear to be approximately

   964 	   same as for software alignment below or ~3x. They promise to

   965 	   improve it in the future, but for now we can just as well

   966 	   pretend that it can only handle aligned input... */

   967 	if (!padlock_aes_align_required && (nbytes%AES_BLOCK_SIZE)==0)

   968 		return padlock_aes_cipher_omnivorous(ctx, out_arg, in_arg, nbytes);

   969 

   970 	inp_misaligned = (((size_t)in_arg) & 0x0F);

   971 	out_misaligned = (((size_t)out_arg) & 0x0F);

   972 

   973 	/* Note that even if output is aligned and input not,

   974 	 * I still prefer to loop instead of copy the whole

   975 	 * input and then encrypt in one stroke. This is done

   976 	 * in order to improve L1 cache utilization... */

   977 	realign_in_loop = out_misaligned|inp_misaligned;

   978 

   979 	if (!realign_in_loop && (nbytes%AES_BLOCK_SIZE)==0)

   980 		return padlock_aes_cipher_omnivorous(ctx, out_arg, in_arg, nbytes);

   981 

   982 	/* this takes one "if" out of the loops */

   983 	chunk  = nbytes;

   984 	chunk %= PADLOCK_CHUNK;

   985 	if (chunk==0) chunk = PADLOCK_CHUNK;

   986 

   987 	if (out_misaligned) {

   988 		/* optmize for small input */

   989 		allocated = (chunk<nbytes?PADLOCK_CHUNK:nbytes);

   990 		out = alloca(0x10 + allocated);

   991 		out = NEAREST_ALIGNED(out);

   992 	}

   993 	else

   994 		out = out_arg;

   995 

   996 	cdata = ALIGNED_CIPHER_DATA(ctx);

   997 	padlock_verify_context(cdata);

   998 

   999 	switch (EVP_CIPHER_CTX_mode(ctx)) {

  1000 	case EVP_CIPH_ECB_MODE:

  1001 		do	{

  1002 			if (inp_misaligned)

  1003 				inp = padlock_memcpy(out, in_arg, chunk);

  1004 			else

  1005 				inp = in_arg;

  1006 			in_arg += chunk;

  1007 

  1008 			padlock_xcrypt_ecb(chunk/AES_BLOCK_SIZE, cdata, out, inp);

  1009 

  1010 			if (out_misaligned)

  1011 				out_arg = padlock_memcpy(out_arg, out, chunk) + chunk;

  1012 			else

  1013 				out     = out_arg+=chunk;

  1014 

  1015 			nbytes -= chunk;

  1016 			chunk   = PADLOCK_CHUNK;

  1017 		} while (nbytes);

  1018 		break;

  1019 

  1020 	case EVP_CIPH_CBC_MODE:

  1021 		memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);

  1022 		goto cbc_shortcut;

  1023 		do	{

  1024 			if (iv != cdata->iv)

  1025 				memcpy(cdata->iv, iv, AES_BLOCK_SIZE);

  1026 			chunk = PADLOCK_CHUNK;

  1027 		cbc_shortcut: /* optimize for small input */

  1028 			if (inp_misaligned)

  1029 				inp = padlock_memcpy(out, in_arg, chunk);

  1030 			else

  1031 				inp = in_arg;

  1032 			in_arg += chunk;

  1033 

  1034 			iv = padlock_xcrypt_cbc(chunk/AES_BLOCK_SIZE, cdata, out, inp);

  1035 

  1036 			if (out_misaligned)

  1037 				out_arg = padlock_memcpy(out_arg, out, chunk) + chunk;

  1038 			else

  1039 				out     = out_arg+=chunk;

  1040 

  1041 		} while (nbytes -= chunk);

  1042 		memcpy(ctx->iv, iv, AES_BLOCK_SIZE);

  1043 		break;

  1044 

  1045 	case EVP_CIPH_CFB_MODE:

  1046 		memcpy (iv = cdata->iv, ctx->iv, AES_BLOCK_SIZE);

  1047 		chunk &= ~(AES_BLOCK_SIZE-1);

  1048 		if (chunk)	goto cfb_shortcut;

  1049 		else		goto cfb_skiploop;

  1050 		do	{

  1051 			if (iv != cdata->iv)

  1052 				memcpy(cdata->iv, iv, AES_BLOCK_SIZE);

  1053 			chunk = PADLOCK_CHUNK;

  1054 		cfb_shortcut: /* optimize for small input */

  1055 			if (inp_misaligned)

  1056 				inp = padlock_memcpy(out, in_arg, chunk);

  1057 			else

  1058 				inp = in_arg;

  1059 			in_arg += chunk;

  1060 

  1061 			iv = padlock_xcrypt_cfb(chunk/AES_BLOCK_SIZE, cdata, out, inp);

  1062 

  1063 			if (out_misaligned)

  1064 				out_arg = padlock_memcpy(out_arg, out, chunk) + chunk;

  1065 			else

  1066 				out     = out_arg+=chunk;

  1067 

  1068 			nbytes -= chunk;

  1069 		} while (nbytes >= AES_BLOCK_SIZE);

  1070 

  1071 		cfb_skiploop:

  1072 		if (nbytes) {

  1073 			unsigned char *ivp = cdata->iv;

  1074 

  1075 			if (iv != ivp) {

  1076 				memcpy(ivp, iv, AES_BLOCK_SIZE);

  1077 				iv = ivp;

  1078 			}

  1079 			ctx->num = nbytes;

  1080 			if (cdata->cword.b.encdec) {

  1081 				cdata->cword.b.encdec=0;

  1082 				padlock_reload_key();

  1083 				padlock_xcrypt_ecb(1,cdata,ivp,ivp);

  1084 				cdata->cword.b.encdec=1;

  1085 				padlock_reload_key();

  1086 				while(nbytes) {

  1087 					unsigned char c = *(in_arg++);

  1088 					*(out_arg++) = c ^ *ivp;

  1089 					*(ivp++) = c, nbytes--;

  1090 				}

  1091 			}

  1092 			else {	padlock_reload_key();

  1093 				padlock_xcrypt_ecb(1,cdata,ivp,ivp);

  1094 				padlock_reload_key();

  1095 				while (nbytes) {

  1096 					*ivp = *(out_arg++) = *(in_arg++) ^ *ivp;

  1097 					ivp++, nbytes--;

  1098 				}

  1099 			}

  1100 		}

  1101 		memcpy(ctx->iv, iv, AES_BLOCK_SIZE);

  1102 		break;

  1103 

  1104 	case EVP_CIPH_OFB_MODE:

  1105 		memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);

  1106 		chunk &= ~(AES_BLOCK_SIZE-1);

  1107 		if (chunk) do	{

  1108 			if (inp_misaligned)

  1109 				inp = padlock_memcpy(out, in_arg, chunk);

  1110 			else

  1111 				inp = in_arg;

  1112 			in_arg += chunk;

  1113 

  1114 			padlock_xcrypt_ofb(chunk/AES_BLOCK_SIZE, cdata, out, inp);

  1115 

  1116 			if (out_misaligned)

  1117 				out_arg = padlock_memcpy(out_arg, out, chunk) + chunk;

  1118 			else

  1119 				out     = out_arg+=chunk;

  1120 

  1121 			nbytes -= chunk;

  1122 			chunk   = PADLOCK_CHUNK;

  1123 		} while (nbytes >= AES_BLOCK_SIZE);

  1124 

  1125 		if (nbytes) {

  1126 			unsigned char *ivp = cdata->iv;

  1127 

  1128 			ctx->num = nbytes;

  1129 			padlock_reload_key();	/* empirically found */

  1130 			padlock_xcrypt_ecb(1,cdata,ivp,ivp);

  1131 			padlock_reload_key();	/* empirically found */

  1132 			while (nbytes) {

  1133 				*(out_arg++) = *(in_arg++) ^ *ivp;

  1134 				ivp++, nbytes--;

  1135 			}