ReactOS 0.4.17-dev-1005-g171e1de
aes-xmm.c
Go to the documentation of this file.
1//
2// aes-xmm.c code for AES implementation
3//
4// Copyright (c) Microsoft Corporation. Licensed under the MIT license.
5//
6// All XMM code for AES operations
7// Requires compiler support for ssse3, aesni and pclmulqdq
8//
9
10#include "precomp.h"
11
12#if SYMCRYPT_CPU_X86 | SYMCRYPT_CPU_AMD64
13
14#include "xtsaes_definitions.h"
15#include "ghash_definitions.h"
16
17#ifdef __clang__
18#pragma clang attribute push (__attribute__((target("ssse3,aes,pclmul"))), apply_to=function)
19#else
20#pragma GCC push_options
21#pragma GCC target("ssse3,aes,pclmul")
22#endif
23
24VOID
25SYMCRYPT_CALL
26SymCryptAes4SboxXmm( _In_reads_(4) PCBYTE pIn, _Out_writes_(4) PBYTE pOut )
27{
28 __m128i x;
29 x = _mm_set1_epi32( *(int *) pIn );
30
31 x = _mm_aeskeygenassist_si128( x, 0 );
32
33 // Could use _mm_storeu_si32( pOut, x ) but it is missing from some headers and _mm_store_ss will be as fast
34 _mm_store_ss( (float *) pOut, _mm_castsi128_ps(x) );
35}
36
37VOID
38SYMCRYPT_CALL
39SymCryptAesCreateDecryptionRoundKeyXmm(
40 _In_reads_(16) PCBYTE pEncryptionRoundKey,
41 _Out_writes_(16) PBYTE pDecryptionRoundKey )
42{
43 //
44 // On x86 our key structure is only 4-aligned (the best we can do) so we use unaligned load/stores.
45 // On Amd64 our round keys are aligned, but recent CPUs have fast unaligned load/store if the address is
46 // actually aligned properly.
47 //
48 _mm_storeu_si128( (__m128i *) pDecryptionRoundKey, _mm_aesimc_si128( _mm_loadu_si128( (__m128i *)pEncryptionRoundKey ) ) );
49}
50
51//
52// The latency of AES instruction has increased up to 8 cycles in Ivy Bridge,
53// and back to 7 in Haswell.
54// We use 8-parallel code to expose the maximum parallelism to the CPU.
55// On x86 it will introduce some register spilling, but the load/stores
56// should be able to hide behind the AES instruction latencies.
57// Silvermont x86 CPUs has AES-NI with latency = 8 and throughput = 5, so there
58// the CPU parallelism is low.
59// For things like BitLocker that is fine, but other uses, such as GCM & AES_CTR_DRBG
60// use odd sizes.
61// We try to do 5-8 blocks in 8-parallel code, 2-4 blocks in 4-parallel code, and
62// 1 block in 1-parallel code.
63// This is a compromise; the big cores can do 8 parallel in about the time of a 4-parallel,
64// but Silvermont cannot and would pay a big price on small requests if we only use 8-parallel.
65// Doing only 8-parallel and then 1-parallel would penalize the big cores a lot.
66//
67// We used to have 7-parallel code, but common request sizes are not multiples of 7
68// blocks so we end up doing a lot of extra work. This is especially expensive on
69// Silvermont where the extra work isn't hidden in the latencies.
70//
71
72#define AES_ENCRYPT_1( pExpandedKey, c0 ) \
73{ \
74 const BYTE (*keyPtr)[4][4]; \
75 const BYTE (*keyLimit)[4][4]; \
76 __m128i roundkey; \
77\
78 keyPtr = &pExpandedKey->RoundKey[0]; \
79 keyLimit = pExpandedKey->lastEncRoundKey; \
80\
81 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
82 keyPtr ++; \
83\
84 c0 = _mm_xor_si128( c0, roundkey ); \
85\
86 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
87 keyPtr ++; \
88 c0 = _mm_aesenc_si128( c0, roundkey ); \
89\
90 do \
91 { \
92 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
93 keyPtr ++; \
94 c0 = _mm_aesenc_si128( c0, roundkey ); \
95 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
96 keyPtr ++; \
97 c0 = _mm_aesenc_si128( c0, roundkey ); \
98 } while( keyPtr < keyLimit ); \
99\
100 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
101\
102 c0 = _mm_aesenclast_si128( c0, roundkey ); \
103};
104
105
106// Perform AES encryption without the first round key and with a specified last round key
107//
108// For algorithms where performance is dominated by a chain of dependent AES rounds (i.e. CBC encryption, CCM, CMAC)
109// we can gain a reasonable performance uplift by computing (last round key ^ next plaintext block ^ first round key)
110// off the critical path and using this computed value in place of last round key in AESENCLAST instructions.
111#define AES_ENCRYPT_1_CHAIN( pExpandedKey, cipherState, mergedLastRoundKey ) \
112{ \
113 const BYTE (*keyPtr)[4][4]; \
114 const BYTE (*keyLimit)[4][4]; \
115 __m128i roundkey; \
116\
117 keyPtr = &pExpandedKey->RoundKey[1]; \
118 keyLimit = pExpandedKey->lastEncRoundKey; \
119\
120 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
121 keyPtr ++; \
122\
123 cipherState = _mm_aesenc_si128( cipherState, roundkey ); \
124\
125 do \
126 { \
127 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
128 keyPtr ++; \
129 cipherState = _mm_aesenc_si128( cipherState, roundkey ); \
130 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
131 keyPtr ++; \
132 cipherState = _mm_aesenc_si128( cipherState, roundkey ); \
133 } while( keyPtr < keyLimit ); \
134\
135 cipherState = _mm_aesenclast_si128( cipherState, mergedLastRoundKey ); \
136};
137
138#define AES_ENCRYPT_4( pExpandedKey, c0, c1, c2, c3 ) \
139{ \
140 const BYTE (*keyPtr)[4][4]; \
141 const BYTE (*keyLimit)[4][4]; \
142 __m128i roundkey; \
143\
144 keyPtr = &pExpandedKey->RoundKey[0]; \
145 keyLimit = pExpandedKey->lastEncRoundKey; \
146\
147 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
148 keyPtr ++; \
149\
150 c0 = _mm_xor_si128( c0, roundkey ); \
151 c1 = _mm_xor_si128( c1, roundkey ); \
152 c2 = _mm_xor_si128( c2, roundkey ); \
153 c3 = _mm_xor_si128( c3, roundkey ); \
154\
155 do \
156 { \
157 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
158 keyPtr ++; \
159 c0 = _mm_aesenc_si128( c0, roundkey ); \
160 c1 = _mm_aesenc_si128( c1, roundkey ); \
161 c2 = _mm_aesenc_si128( c2, roundkey ); \
162 c3 = _mm_aesenc_si128( c3, roundkey ); \
163 } while( keyPtr < keyLimit ); \
164\
165 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
166\
167 c0 = _mm_aesenclast_si128( c0, roundkey ); \
168 c1 = _mm_aesenclast_si128( c1, roundkey ); \
169 c2 = _mm_aesenclast_si128( c2, roundkey ); \
170 c3 = _mm_aesenclast_si128( c3, roundkey ); \
171};
172
173#define AES_ENCRYPT_8( pExpandedKey, c0, c1, c2, c3, c4, c5, c6, c7 ) \
174{ \
175 const BYTE (*keyPtr)[4][4]; \
176 const BYTE (*keyLimit)[4][4]; \
177 __m128i roundkey; \
178\
179 keyPtr = &pExpandedKey->RoundKey[0]; \
180 keyLimit = pExpandedKey->lastEncRoundKey; \
181\
182 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
183 keyPtr ++; \
184\
185 c0 = _mm_xor_si128( c0, roundkey ); \
186 c1 = _mm_xor_si128( c1, roundkey ); \
187 c2 = _mm_xor_si128( c2, roundkey ); \
188 c3 = _mm_xor_si128( c3, roundkey ); \
189 c4 = _mm_xor_si128( c4, roundkey ); \
190 c5 = _mm_xor_si128( c5, roundkey ); \
191 c6 = _mm_xor_si128( c6, roundkey ); \
192 c7 = _mm_xor_si128( c7, roundkey ); \
193\
194 do \
195 { \
196 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
197 keyPtr ++; \
198 c0 = _mm_aesenc_si128( c0, roundkey ); \
199 c1 = _mm_aesenc_si128( c1, roundkey ); \
200 c2 = _mm_aesenc_si128( c2, roundkey ); \
201 c3 = _mm_aesenc_si128( c3, roundkey ); \
202 c4 = _mm_aesenc_si128( c4, roundkey ); \
203 c5 = _mm_aesenc_si128( c5, roundkey ); \
204 c6 = _mm_aesenc_si128( c6, roundkey ); \
205 c7 = _mm_aesenc_si128( c7, roundkey ); \
206 } while( keyPtr < keyLimit ); \
207\
208 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
209\
210 c0 = _mm_aesenclast_si128( c0, roundkey ); \
211 c1 = _mm_aesenclast_si128( c1, roundkey ); \
212 c2 = _mm_aesenclast_si128( c2, roundkey ); \
213 c3 = _mm_aesenclast_si128( c3, roundkey ); \
214 c4 = _mm_aesenclast_si128( c4, roundkey ); \
215 c5 = _mm_aesenclast_si128( c5, roundkey ); \
216 c6 = _mm_aesenclast_si128( c6, roundkey ); \
217 c7 = _mm_aesenclast_si128( c7, roundkey ); \
218};
219
220#define AES_DECRYPT_1( pExpandedKey, c0 ) \
221{ \
222 const BYTE (*keyPtr)[4][4]; \
223 const BYTE (*keyLimit)[4][4]; \
224 __m128i roundkey; \
225\
226 keyPtr = pExpandedKey->lastEncRoundKey; \
227 keyLimit = pExpandedKey->lastDecRoundKey; \
228\
229 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
230 keyPtr ++; \
231\
232 c0 = _mm_xor_si128( c0, roundkey ); \
233\
234 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
235 keyPtr ++; \
236 c0 = _mm_aesdec_si128( c0, roundkey ); \
237\
238 do \
239 { \
240 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
241 keyPtr ++; \
242 c0 = _mm_aesdec_si128( c0, roundkey ); \
243 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
244 keyPtr ++; \
245 c0 = _mm_aesdec_si128( c0, roundkey ); \
246 } while( keyPtr < keyLimit ); \
247\
248 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
249\
250 c0 = _mm_aesdeclast_si128( c0, roundkey ); \
251};
252
253#define AES_DECRYPT_4( pExpandedKey, c0, c1, c2, c3 ) \
254{ \
255 const BYTE (*keyPtr)[4][4]; \
256 const BYTE (*keyLimit)[4][4]; \
257 __m128i roundkey; \
258\
259 keyPtr = pExpandedKey->lastEncRoundKey; \
260 keyLimit = pExpandedKey->lastDecRoundKey; \
261\
262 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
263 keyPtr ++; \
264\
265 c0 = _mm_xor_si128( c0, roundkey ); \
266 c1 = _mm_xor_si128( c1, roundkey ); \
267 c2 = _mm_xor_si128( c2, roundkey ); \
268 c3 = _mm_xor_si128( c3, roundkey ); \
269\
270 do \
271 { \
272 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
273 keyPtr ++; \
274 c0 = _mm_aesdec_si128( c0, roundkey ); \
275 c1 = _mm_aesdec_si128( c1, roundkey ); \
276 c2 = _mm_aesdec_si128( c2, roundkey ); \
277 c3 = _mm_aesdec_si128( c3, roundkey ); \
278 } while( keyPtr < keyLimit ); \
279\
280 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
281\
282 c0 = _mm_aesdeclast_si128( c0, roundkey ); \
283 c1 = _mm_aesdeclast_si128( c1, roundkey ); \
284 c2 = _mm_aesdeclast_si128( c2, roundkey ); \
285 c3 = _mm_aesdeclast_si128( c3, roundkey ); \
286};
287
288#define AES_DECRYPT_8( pExpandedKey, c0, c1, c2, c3, c4, c5, c6, c7 ) \
289{ \
290 const BYTE (*keyPtr)[4][4]; \
291 const BYTE (*keyLimit)[4][4]; \
292 __m128i roundkey; \
293\
294 keyPtr = pExpandedKey->lastEncRoundKey; \
295 keyLimit = pExpandedKey->lastDecRoundKey; \
296\
297 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
298 keyPtr ++; \
299\
300 c0 = _mm_xor_si128( c0, roundkey ); \
301 c1 = _mm_xor_si128( c1, roundkey ); \
302 c2 = _mm_xor_si128( c2, roundkey ); \
303 c3 = _mm_xor_si128( c3, roundkey ); \
304 c4 = _mm_xor_si128( c4, roundkey ); \
305 c5 = _mm_xor_si128( c5, roundkey ); \
306 c6 = _mm_xor_si128( c6, roundkey ); \
307 c7 = _mm_xor_si128( c7, roundkey ); \
308\
309 do \
310 { \
311 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
312 keyPtr ++; \
313 c0 = _mm_aesdec_si128( c0, roundkey ); \
314 c1 = _mm_aesdec_si128( c1, roundkey ); \
315 c2 = _mm_aesdec_si128( c2, roundkey ); \
316 c3 = _mm_aesdec_si128( c3, roundkey ); \
317 c4 = _mm_aesdec_si128( c4, roundkey ); \
318 c5 = _mm_aesdec_si128( c5, roundkey ); \
319 c6 = _mm_aesdec_si128( c6, roundkey ); \
320 c7 = _mm_aesdec_si128( c7, roundkey ); \
321 } while( keyPtr < keyLimit ); \
322\
323 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
324\
325 c0 = _mm_aesdeclast_si128( c0, roundkey ); \
326 c1 = _mm_aesdeclast_si128( c1, roundkey ); \
327 c2 = _mm_aesdeclast_si128( c2, roundkey ); \
328 c3 = _mm_aesdeclast_si128( c3, roundkey ); \
329 c4 = _mm_aesdeclast_si128( c4, roundkey ); \
330 c5 = _mm_aesdeclast_si128( c5, roundkey ); \
331 c6 = _mm_aesdeclast_si128( c6, roundkey ); \
332 c7 = _mm_aesdeclast_si128( c7, roundkey ); \
333};
334
335
336//
337// The EncryptXmm code is tested through the CFB mode encryption which has no further optimizations.
338//
339VOID
340SYMCRYPT_CALL
341SymCryptAesEncryptXmm(
342 _In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey,
343 _In_reads_( SYMCRYPT_AES_BLOCK_SIZE ) PCBYTE pbSrc,
344 _Out_writes_( SYMCRYPT_AES_BLOCK_SIZE ) PBYTE pbDst )
345{
346 __m128i c;
347
348 c = _mm_loadu_si128( ( __m128i * ) pbSrc);
349
350 AES_ENCRYPT_1( pExpandedKey, c );
351
352 _mm_storeu_si128( (__m128i *) pbDst, c );
353}
354
355//
356// The DecryptXmm code is tested through the EcbDecrypt calls which has no further optimizations.
357//
358VOID
359SYMCRYPT_CALL
360SymCryptAesDecryptXmm(
361 _In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey,
362 _In_reads_( SYMCRYPT_AES_BLOCK_SIZE ) PCBYTE pbSrc,
363 _Out_writes_( SYMCRYPT_AES_BLOCK_SIZE ) PBYTE pbDst )
364{
365 __m128i c;
366
367 c = _mm_loadu_si128( ( __m128i * ) pbSrc);
368
369 AES_DECRYPT_1( pExpandedKey, c );
370
371 _mm_storeu_si128( (__m128i *) pbDst, c );
372}
373
374// Disable warnings and VC++ runtime checks for use of uninitialized values (by design)
375#pragma warning(push)
376#pragma warning( disable: 6001 4701 )
377#pragma runtime_checks( "u", off )
378VOID
379SYMCRYPT_CALL
380SymCryptAesEcbEncryptXmm(
381 _In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey,
382 _In_reads_( cbData ) PCBYTE pbSrc,
383 _Out_writes_( cbData ) PBYTE pbDst,
384 SIZE_T cbData )
385{
386 __m128i c0, c1, c2, c3, c4, c5, c6, c7;
387
388 while( cbData >= 8 * SYMCRYPT_AES_BLOCK_SIZE )
389 {
390 c0 = _mm_loadu_si128( ( __m128i * ) (pbSrc + 0 ));
391 c1 = _mm_loadu_si128( ( __m128i * ) (pbSrc + 16 ));
392 c2 = _mm_loadu_si128( ( __m128i * ) (pbSrc + 32 ));
393 c3 = _mm_loadu_si128( ( __m128i * ) (pbSrc + 48 ));
394 c4 = _mm_loadu_si128( ( __m128i * ) (pbSrc + 64 ));
395 c5 = _mm_loadu_si128( ( __m128i * ) (pbSrc + 80 ));
396 c6 = _mm_loadu_si128( ( __m128i * ) (pbSrc + 96 ));
397 c7 = _mm_loadu_si128( ( __m128i * ) (pbSrc +112 ));
398
399 AES_ENCRYPT_8( pExpandedKey, c0, c1, c2, c3, c4, c5, c6, c7 );
400
401 _mm_storeu_si128( (__m128i *) (pbDst + 0 ), c0 );
402 _mm_storeu_si128( (__m128i *) (pbDst + 16 ), c1 );
403 _mm_storeu_si128( (__m128i *) (pbDst + 32 ), c2 );
404 _mm_storeu_si128( (__m128i *) (pbDst + 48 ), c3 );
405 _mm_storeu_si128( (__m128i *) (pbDst + 64 ), c4 );
406 _mm_storeu_si128( (__m128i *) (pbDst + 80 ), c5 );
407 _mm_storeu_si128( (__m128i *) (pbDst + 96 ), c6 );
408 _mm_storeu_si128( (__m128i *) (pbDst +112 ), c7 );
409
410 pbSrc += 8 * SYMCRYPT_AES_BLOCK_SIZE;
411 pbDst += 8 * SYMCRYPT_AES_BLOCK_SIZE;
412 cbData -= 8 * SYMCRYPT_AES_BLOCK_SIZE;
413 }
414
415 if( cbData < 16 )
416 {
417 return;
418 }
419
420 c0 = _mm_loadu_si128( ( __m128i * ) (pbSrc + 0 ));
421 if( cbData >= 32 )
422 {
423 c1 = _mm_loadu_si128( ( __m128i * ) (pbSrc + 16 ));
424 if( cbData >= 48 )
425 {
426 c2 = _mm_loadu_si128( ( __m128i * ) (pbSrc + 32 ));
427 if( cbData >= 64 )
428 {
429 c3 = _mm_loadu_si128( ( __m128i * ) (pbSrc + 48 ));
430 if( cbData >= 80 )
431 {
432 c4 = _mm_loadu_si128( ( __m128i * ) (pbSrc + 64 ));
433 if( cbData >= 96 )
434 {
435 c5 = _mm_loadu_si128( ( __m128i * ) (pbSrc + 80 ));
436 if( cbData >= 112 )
437 {
438 c6 = _mm_loadu_si128( ( __m128i * ) (pbSrc + 96 ));
439 }
440 }
441 }
442 }
443 }
444 }
445
446 if( cbData >= 5 * SYMCRYPT_AES_BLOCK_SIZE )
447 {
448 AES_ENCRYPT_8( pExpandedKey, c0, c1, c2, c3, c4, c5, c6, c7 );
449 }
450 else if( cbData >= 2 * SYMCRYPT_AES_BLOCK_SIZE )
451 {
452 AES_ENCRYPT_4( pExpandedKey, c0, c1, c2, c3 );
453 }
454 else
455 {
456 AES_ENCRYPT_1( pExpandedKey, c0 );
457 }
458
459 _mm_storeu_si128( (__m128i *) (pbDst + 0 ), c0 );
460 if( cbData >= 32 )
461 {
462 _mm_storeu_si128( (__m128i *) (pbDst + 16 ), c1 );
463 if( cbData >= 48 )
464 {
465 _mm_storeu_si128( (__m128i *) (pbDst + 32 ), c2 );
466 if( cbData >= 64 )
467 {
468 _mm_storeu_si128( (__m128i *) (pbDst + 48 ), c3 );
469 if( cbData >= 80 )
470 {
471 _mm_storeu_si128( (__m128i *) (pbDst + 64 ), c4 );
472 if( cbData >= 96 )
473 {
474 _mm_storeu_si128( (__m128i *) (pbDst + 80 ), c5 );
475 if( cbData >= 112 )
476 {
477 _mm_storeu_si128( (__m128i *) (pbDst + 96 ), c6 );
478 }
479 }
480 }
481 }
482 }
483 }
484}
485#pragma runtime_checks( "u", restore )
486#pragma warning( pop )
487
488
489
490VOID
491SYMCRYPT_CALL
492SymCryptAesCbcEncryptXmm(
493 _In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey,
494 _Inout_updates_( SYMCRYPT_AES_BLOCK_SIZE ) PBYTE pbChainingValue,
495 _In_reads_( cbData ) PCBYTE pbSrc,
496 _Out_writes_( cbData ) PBYTE pbDst,
497 SIZE_T cbData )
498{
499 __m128i c = _mm_loadu_si128( (__m128i *) pbChainingValue );
500 __m128i rk0 = _mm_loadu_si128( (__m128i *) &pExpandedKey->RoundKey[0] );
501 __m128i rkLast = _mm_loadu_si128( (__m128i *) pExpandedKey->lastEncRoundKey );
502 __m128i d;
503
504 if (cbData < SYMCRYPT_AES_BLOCK_SIZE)
505 return;
506
507 // This algorithm is dominated by chain of dependent AES rounds, so we want to avoid XOR
508 // instructions on the critical path where possible
509 // We can compute (last round key ^ next plaintext block ^ first round key) off the critical
510 // path and use this with AES_ENCRYPT_1_CHAIN so that only AES instructions write to c in
511 // the main loop
512 d = _mm_xor_si128( _mm_loadu_si128( (__m128i *) pbSrc ), rk0 );
513 c = _mm_xor_si128( c, d );
514 pbSrc += SYMCRYPT_AES_BLOCK_SIZE;
515 cbData -= SYMCRYPT_AES_BLOCK_SIZE;
516
517 while( cbData >= SYMCRYPT_AES_BLOCK_SIZE )
518 {
519 d = _mm_xor_si128( _mm_loadu_si128( (__m128i *) pbSrc ), rk0 );
520 AES_ENCRYPT_1_CHAIN( pExpandedKey, c, _mm_xor_si128(d, rkLast ) );
521 _mm_storeu_si128( (__m128i *) pbDst, _mm_xor_si128(c, d) );
522
523 pbSrc += SYMCRYPT_AES_BLOCK_SIZE;
524 pbDst += SYMCRYPT_AES_BLOCK_SIZE;
525 cbData -= SYMCRYPT_AES_BLOCK_SIZE;
526 }
527 AES_ENCRYPT_1_CHAIN( pExpandedKey, c, rkLast );
528 _mm_storeu_si128( (__m128i *) pbDst, c );
529 _mm_storeu_si128( (__m128i *) pbChainingValue, c );
530}
531
532// Disable warnings and VC++ runtime checks for use of uninitialized values (by design)
533#pragma warning(push)
534#pragma warning( disable: 6001 4701 )
535#pragma runtime_checks( "u", off )
536VOID
537SYMCRYPT_CALL
538SymCryptAesCbcDecryptXmm(
539 _In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey,
540 _Inout_updates_( SYMCRYPT_AES_BLOCK_SIZE ) PBYTE pbChainingValue,
541 _In_reads_( cbData ) PCBYTE pbSrc,
542 _Out_writes_( cbData ) PBYTE pbDst,
543 SIZE_T cbData )
544{
545 __m128i chain;
546 __m128i c0, c1, c2, c3, c4, c5, c6, c7;
547 __m128i d0, d1, d2, d3, d4, d5, d6, d7;
548
549 if( cbData < SYMCRYPT_AES_BLOCK_SIZE )
550 {
551 return;
552 }
553
554 chain = _mm_loadu_si128( (__m128i *) pbChainingValue );
555
556 //
557 // First we do all multiples of 8 blocks
558 //
559
560 while( cbData >= 8 * SYMCRYPT_AES_BLOCK_SIZE )
561 {
562 d0 = c0 = _mm_loadu_si128( (__m128i *) (pbSrc + 0 * SYMCRYPT_AES_BLOCK_SIZE ) );
563 d1 = c1 = _mm_loadu_si128( (__m128i *) (pbSrc + 1 * SYMCRYPT_AES_BLOCK_SIZE ) );
564 d2 = c2 = _mm_loadu_si128( (__m128i *) (pbSrc + 2 * SYMCRYPT_AES_BLOCK_SIZE ) );
565 d3 = c3 = _mm_loadu_si128( (__m128i *) (pbSrc + 3 * SYMCRYPT_AES_BLOCK_SIZE ) );
566 d4 = c4 = _mm_loadu_si128( (__m128i *) (pbSrc + 4 * SYMCRYPT_AES_BLOCK_SIZE ) );
567 d5 = c5 = _mm_loadu_si128( (__m128i *) (pbSrc + 5 * SYMCRYPT_AES_BLOCK_SIZE ) );
568 d6 = c6 = _mm_loadu_si128( (__m128i *) (pbSrc + 6 * SYMCRYPT_AES_BLOCK_SIZE ) );
569 d7 = c7 = _mm_loadu_si128( (__m128i *) (pbSrc + 7 * SYMCRYPT_AES_BLOCK_SIZE ) );
570
571 AES_DECRYPT_8( pExpandedKey, c0, c1, c2, c3, c4, c5, c6, c7 );
572
573 c0 = _mm_xor_si128( c0, chain );
574 c1 = _mm_xor_si128( c1, d0 );
575 c2 = _mm_xor_si128( c2, d1 );
576 c3 = _mm_xor_si128( c3, d2 );
577 c4 = _mm_xor_si128( c4, d3 );
578 c5 = _mm_xor_si128( c5, d4 );
579 c6 = _mm_xor_si128( c6, d5 );
580 c7 = _mm_xor_si128( c7, d6 );
581 chain = d7;
582
583 _mm_storeu_si128( (__m128i *) (pbDst + 0 * SYMCRYPT_AES_BLOCK_SIZE ), c0 );
584 _mm_storeu_si128( (__m128i *) (pbDst + 1 * SYMCRYPT_AES_BLOCK_SIZE ), c1 );
585 _mm_storeu_si128( (__m128i *) (pbDst + 2 * SYMCRYPT_AES_BLOCK_SIZE ), c2 );
586 _mm_storeu_si128( (__m128i *) (pbDst + 3 * SYMCRYPT_AES_BLOCK_SIZE ), c3 );
587 _mm_storeu_si128( (__m128i *) (pbDst + 4 * SYMCRYPT_AES_BLOCK_SIZE ), c4 );
588 _mm_storeu_si128( (__m128i *) (pbDst + 5 * SYMCRYPT_AES_BLOCK_SIZE ), c5 );
589 _mm_storeu_si128( (__m128i *) (pbDst + 6 * SYMCRYPT_AES_BLOCK_SIZE ), c6 );
590 _mm_storeu_si128( (__m128i *) (pbDst + 7 * SYMCRYPT_AES_BLOCK_SIZE ), c7 );
591
592 pbSrc += 8 * SYMCRYPT_AES_BLOCK_SIZE;
593 pbDst += 8 * SYMCRYPT_AES_BLOCK_SIZE;
594 cbData -= 8 * SYMCRYPT_AES_BLOCK_SIZE;
595 }
596
597 if( cbData >= 16 )
598 {
599 //
600 // There is remaining work to be done
601 //
602 d0 = c0 = _mm_loadu_si128( (__m128i *) (pbSrc + 0 * SYMCRYPT_AES_BLOCK_SIZE ) );
603 if( cbData >= 32 )
604 {
605 d1 = c1 = _mm_loadu_si128( (__m128i *) (pbSrc + 1 * SYMCRYPT_AES_BLOCK_SIZE ) );
606 if( cbData >= 48 )
607 {
608 d2 = c2 = _mm_loadu_si128( (__m128i *) (pbSrc + 2 * SYMCRYPT_AES_BLOCK_SIZE ) );
609 if( cbData >= 64 )
610 {
611 d3 = c3 = _mm_loadu_si128( (__m128i *) (pbSrc + 3 * SYMCRYPT_AES_BLOCK_SIZE ) );
612 if( cbData >= 80 )
613 {
614 d4 = c4 = _mm_loadu_si128( (__m128i *) (pbSrc + 4 * SYMCRYPT_AES_BLOCK_SIZE ) );
615 if( cbData >= 96 )
616 {
617 d5 = c5 = _mm_loadu_si128( (__m128i *) (pbSrc + 5 * SYMCRYPT_AES_BLOCK_SIZE ) );
618 if( cbData >= 112 )
619 {
620 d6 = c6 = _mm_loadu_si128( (__m128i *) (pbSrc + 6 * SYMCRYPT_AES_BLOCK_SIZE ) );
621 }
622 }
623 }
624 }
625 }
626 }
627
628 //
629 // Decrypt 1, 4, or 8 blocks in AES-CBC mode. This might decrypt uninitialized registers,
630 // but those will not be used when we store the results.
631 //
632 if( cbData > 4 * SYMCRYPT_AES_BLOCK_SIZE )
633 {
634 AES_DECRYPT_8( pExpandedKey, c0, c1, c2, c3, c4, c5, c6, c7 );
635 c0 = _mm_xor_si128( c0, chain );
636 c1 = _mm_xor_si128( c1, d0 );
637 c2 = _mm_xor_si128( c2, d1 );
638 c3 = _mm_xor_si128( c3, d2 );
639 c4 = _mm_xor_si128( c4, d3 );
640 c5 = _mm_xor_si128( c5, d4 );
641 c6 = _mm_xor_si128( c6, d5 );
642 }
643 else if( cbData > SYMCRYPT_AES_BLOCK_SIZE )
644 {
645 AES_DECRYPT_4( pExpandedKey, c0, c1, c2, c3 );
646 c0 = _mm_xor_si128( c0, chain );
647 c1 = _mm_xor_si128( c1, d0 );
648 c2 = _mm_xor_si128( c2, d1 );
649 c3 = _mm_xor_si128( c3, d2 );
650 } else
651 {
652 AES_DECRYPT_1( pExpandedKey, c0 );
653 c0 = _mm_xor_si128( c0, chain );
654 }
655
656 chain = _mm_loadu_si128( (__m128i *) (pbSrc + cbData - SYMCRYPT_AES_BLOCK_SIZE ) );
657 _mm_storeu_si128( (__m128i *) (pbDst + 0 * SYMCRYPT_AES_BLOCK_SIZE ), c0 );
658 if( cbData >= 32 )
659 {
660 _mm_storeu_si128( (__m128i *) (pbDst + 1 * SYMCRYPT_AES_BLOCK_SIZE ), c1 );
661 if( cbData >= 48 )
662 {
663 _mm_storeu_si128( (__m128i *) (pbDst + 2 * SYMCRYPT_AES_BLOCK_SIZE ), c2 );
664 if( cbData >= 64 )
665 {
666 _mm_storeu_si128( (__m128i *) (pbDst + 3 * SYMCRYPT_AES_BLOCK_SIZE ), c3 );
667 if( cbData >= 80 )
668 {
669 _mm_storeu_si128( (__m128i *) (pbDst + 4 * SYMCRYPT_AES_BLOCK_SIZE ), c4 );
670 if( cbData >= 96 )
671 {
672 _mm_storeu_si128( (__m128i *) (pbDst + 5 * SYMCRYPT_AES_BLOCK_SIZE ), c5 );
673 if( cbData >= 112 )
674 {
675 _mm_storeu_si128( (__m128i *) (pbDst + 6 * SYMCRYPT_AES_BLOCK_SIZE ), c6 );
676 }
677 }
678 }
679 }
680 }
681 }
682 }
683
684 _mm_storeu_si128( (__m128i *) pbChainingValue, chain );
685
686 return;
687}
688#pragma runtime_checks( "u", restore )
689#pragma warning( pop )
690
691VOID
692SYMCRYPT_CALL
693SymCryptAesCbcMacXmm(
694 _In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey,
695 _Inout_updates_( SYMCRYPT_AES_BLOCK_SIZE ) PBYTE pbChainingValue,
696 _In_reads_( cbData ) PCBYTE pbData,
697 SIZE_T cbData )
698{
699 __m128i c = _mm_loadu_si128( (__m128i *) pbChainingValue );
700 __m128i rk0 = _mm_loadu_si128( (__m128i *) &pExpandedKey->RoundKey[0] );
701 __m128i rkLast = _mm_loadu_si128( (__m128i *) pExpandedKey->lastEncRoundKey );
702 __m128i d, rk0AndLast;
703
704 if (cbData < SYMCRYPT_AES_BLOCK_SIZE)
705 return;
706
707 // This algorithm is dominated by chain of dependent AES rounds, so we want to avoid XOR
708 // instructions on the critical path where possible
709 // We can compute (last round key ^ next plaintext block ^ first round key) off the critical
710 // path and use this with AES_ENCRYPT_1_CHAIN so that only AES instructions write to c in
711 // the main loop
712 d = _mm_xor_si128( _mm_loadu_si128( (__m128i *) pbData ), rk0 );
713 c = _mm_xor_si128( c, d );
714 pbData += SYMCRYPT_AES_BLOCK_SIZE;
715 cbData -= SYMCRYPT_AES_BLOCK_SIZE;
716
717 // As we don't compute ciphertext here, we only need to XOR rk0 and rkLast once
718 rk0AndLast = _mm_xor_si128( rk0, rkLast );
719
720 while( cbData >= SYMCRYPT_AES_BLOCK_SIZE )
721 {
722 d = _mm_xor_si128( _mm_loadu_si128( (__m128i *) pbData ), rk0AndLast );
723 AES_ENCRYPT_1_CHAIN( pExpandedKey, c, d );
724
725 pbData += SYMCRYPT_AES_BLOCK_SIZE;
726 cbData -= SYMCRYPT_AES_BLOCK_SIZE;
727 }
728 AES_ENCRYPT_1_CHAIN( pExpandedKey, c, rkLast );
729 _mm_storeu_si128( (__m128i *) pbChainingValue, c );
730}
731
732
733#pragma warning(push)
734#pragma warning( disable:4701 ) // "Use of uninitialized variable"
735#pragma runtime_checks( "u", off )
736
737#define SYMCRYPT_AesCtrMsbXxXmm SymCryptAesCtrMsb64Xmm
738#define MM_ADD_EPIXX _mm_add_epi64
739#define MM_SUB_EPIXX _mm_sub_epi64
740
741#include "aes-pattern.c"
742
743#undef MM_SUB_EPIXX
744#undef MM_ADD_EPIXX
745#undef SYMCRYPT_AesCtrMsbXxXmm
746
747#define SYMCRYPT_AesCtrMsbXxXmm SymCryptAesCtrMsb32Xmm
748#define MM_ADD_EPIXX _mm_add_epi32
749#define MM_SUB_EPIXX _mm_sub_epi32
750
751#include "aes-pattern.c"
752
753#undef MM_SUB_EPIXX
754#undef MM_ADD_EPIXX
755#undef SYMCRYPT_AesCtrMsbXxXmm
756
757#pragma runtime_checks( "u", restore )
758#pragma warning(pop)
759
760/*
761 if( cbData >= 16 )
762 {
763 if( cbData >= 32 )
764 {
765 if( cbData >= 48 )
766 {
767 if( cbData >= 64 )
768 {
769 if( cbData >= 80 )
770 {
771 if( cbData >= 96 )
772 {
773 if( cbData >= 112 )
774 {
775 }
776 }
777 }
778 }
779 }
780 }
781 }
782*/
783
784VOID
785SYMCRYPT_CALL
786SymCryptXtsAesEncryptDataUnitXmm(
787 _In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey,
788 _In_reads_( SYMCRYPT_AES_BLOCK_SIZE ) PBYTE pbTweakBlock,
789 _Out_writes_( SYMCRYPT_AES_BLOCK_SIZE*16 ) PBYTE pbScratch,
790 _In_reads_( cbData ) PCBYTE pbSrc,
791 _Out_writes_( cbData ) PBYTE pbDst,
792 SIZE_T cbData )
793{
794 __m128i t0;
795 __m128i c0, c1, c2, c3, c4, c5, c6, c7;
796 __m128i roundkey, firstRoundKey, lastRoundKey;
797 __m128i XTS_ALPHA_MASK = _mm_set_epi32( 1, 1, 1, 0x87 );
798 SYMCRYPT_GF128_ELEMENT* tweakBuffer = (SYMCRYPT_GF128_ELEMENT*) pbScratch;
799
800 const BYTE (*keyPtr)[4][4];
801 const BYTE (*keyLimit)[4][4] = pExpandedKey->lastEncRoundKey;
802 UINT64 lastTweakLow, lastTweakHigh;
803 int aesEncryptXtsLoop;
804
805 SIZE_T cbDataMain; // number of bytes to handle in the main loop
806 SIZE_T cbDataTail; // number of bytes to handle in the tail loop
807
808 SYMCRYPT_ASSERT(cbData >= SYMCRYPT_AES_BLOCK_SIZE);
809
810 // To simplify logic and unusual size processing, we handle all
811 // data not a multiple of 8 blocks in the tail loop
812 cbDataTail = cbData & ((8*SYMCRYPT_AES_BLOCK_SIZE)-1);
813 // Additionally, so that ciphertext stealing logic does not rely on
814 // reading back from the destination buffer, when we have a non-zero
815 // tail, we ensure that we handle at least 1 whole block in the tail
816 //
817 // Note that our caller has ensured we have at least 1 whole block
818 // to process, this is checked in debug build
819 // This means that cbDataTail is in [1,15] at this point iff there are
820 // at least 8 whole blocks to process; so the below does not cause
821 // cbDataTail or cbDataMain to exceed cbData
822 cbDataTail += ((cbDataTail > 0) && (cbDataTail < SYMCRYPT_AES_BLOCK_SIZE)) ? (8*SYMCRYPT_AES_BLOCK_SIZE) : 0;
823 cbDataMain = cbData - cbDataTail;
824
825 SYMCRYPT_ASSERT(cbDataMain <= cbData);
826 SYMCRYPT_ASSERT(cbDataTail <= cbData);
827 SYMCRYPT_ASSERT((cbDataMain & ((8*SYMCRYPT_AES_BLOCK_SIZE)-1)) == 0);
828
829 c0 = _mm_loadu_si128( (__m128i *) pbTweakBlock );
830 XTS_MUL_ALPHA( c0, c1 );
831 XTS_MUL_ALPHA( c1, c2 );
832 XTS_MUL_ALPHA( c2, c3 );
833
834 XTS_MUL_ALPHA4( c0, c4 );
835 XTS_MUL_ALPHA ( c4, c5 );
836 XTS_MUL_ALPHA ( c5, c6 );
837 XTS_MUL_ALPHA ( c6, c7 );
838
839 tweakBuffer[0].m128i = c0;
840 tweakBuffer[1].m128i = c1;
841 tweakBuffer[2].m128i = c2;
842 tweakBuffer[3].m128i = c3;
843 tweakBuffer[4].m128i = c4;
844 tweakBuffer[5].m128i = c5;
845 tweakBuffer[6].m128i = c6;
846 tweakBuffer[7].m128i = c7;
847 lastTweakLow = tweakBuffer[7].ull[0];
848 lastTweakHigh = tweakBuffer[7].ull[1];
849
850 firstRoundKey = _mm_loadu_si128( (__m128i *) &pExpandedKey->RoundKey[0] );
851 lastRoundKey = _mm_loadu_si128( (__m128i *) pExpandedKey->lastEncRoundKey );
852
853 while( cbDataMain > 0 )
854 {
855 // At loop entry, tweakBuffer[0-7] are tweakValues for the next 8 blocks
856 c0 = _mm_xor_si128( tweakBuffer[0].m128i, firstRoundKey );
857 c1 = _mm_xor_si128( tweakBuffer[1].m128i, firstRoundKey );
858 c2 = _mm_xor_si128( tweakBuffer[2].m128i, firstRoundKey );
859 c3 = _mm_xor_si128( tweakBuffer[3].m128i, firstRoundKey );
860 c4 = _mm_xor_si128( tweakBuffer[4].m128i, firstRoundKey );
861 c5 = _mm_xor_si128( tweakBuffer[5].m128i, firstRoundKey );
862 c6 = _mm_xor_si128( tweakBuffer[6].m128i, firstRoundKey );
863 c7 = _mm_xor_si128( tweakBuffer[7].m128i, firstRoundKey );
864
865 c0 = _mm_xor_si128( c0, _mm_loadu_si128( ( __m128i * ) (pbSrc + 0) ) );
866 c1 = _mm_xor_si128( c1, _mm_loadu_si128( ( __m128i * ) (pbSrc + 16) ) );
867 c2 = _mm_xor_si128( c2, _mm_loadu_si128( ( __m128i * ) (pbSrc + 32) ) );
868 c3 = _mm_xor_si128( c3, _mm_loadu_si128( ( __m128i * ) (pbSrc + 48) ) );
869 c4 = _mm_xor_si128( c4, _mm_loadu_si128( ( __m128i * ) (pbSrc + 64) ) );
870 c5 = _mm_xor_si128( c5, _mm_loadu_si128( ( __m128i * ) (pbSrc + 80) ) );
871 c6 = _mm_xor_si128( c6, _mm_loadu_si128( ( __m128i * ) (pbSrc + 96) ) );
872 c7 = _mm_xor_si128( c7, _mm_loadu_si128( ( __m128i * ) (pbSrc + 112) ) );
873
874 keyPtr = &pExpandedKey->RoundKey[1];
875
876 // Do 8 full rounds (AES-128|AES-192|AES-256) with stitched XTS (performed in scalar registers)
877 for( aesEncryptXtsLoop = 0; aesEncryptXtsLoop < 8; aesEncryptXtsLoop++ )
878 {
879 roundkey = _mm_loadu_si128( (__m128i *) keyPtr );
880 keyPtr ++;
881 c0 = _mm_aesenc_si128( c0, roundkey );
882 c1 = _mm_aesenc_si128( c1, roundkey );
883 c2 = _mm_aesenc_si128( c2, roundkey );
884 c3 = _mm_aesenc_si128( c3, roundkey );
885 c4 = _mm_aesenc_si128( c4, roundkey );
886 c5 = _mm_aesenc_si128( c5, roundkey );
887 c6 = _mm_aesenc_si128( c6, roundkey );
888 c7 = _mm_aesenc_si128( c7, roundkey );
889
890 // Prepare tweakBuffer[8-15] with tweak^lastRoundKey
891 tweakBuffer[ 8+aesEncryptXtsLoop ].m128i = _mm_xor_si128( tweakBuffer[ aesEncryptXtsLoop ].m128i, lastRoundKey );
892 // Prepare tweakBuffer[0-7] with tweaks for next 8 blocks
893 XTS_MUL_ALPHA_Scalar( lastTweakLow, lastTweakHigh );
894 tweakBuffer[ aesEncryptXtsLoop ].ull[0] = lastTweakLow;
895 tweakBuffer[ aesEncryptXtsLoop ].ull[1] = lastTweakHigh;
896 }
897
898 do
899 {
900 roundkey = _mm_loadu_si128( (__m128i *) keyPtr );
901 keyPtr ++;
902 c0 = _mm_aesenc_si128( c0, roundkey );
903 c1 = _mm_aesenc_si128( c1, roundkey );
904 c2 = _mm_aesenc_si128( c2, roundkey );
905 c3 = _mm_aesenc_si128( c3, roundkey );
906 c4 = _mm_aesenc_si128( c4, roundkey );
907 c5 = _mm_aesenc_si128( c5, roundkey );
908 c6 = _mm_aesenc_si128( c6, roundkey );
909 c7 = _mm_aesenc_si128( c7, roundkey );
910 } while( keyPtr < keyLimit );
911
912 _mm_storeu_si128( (__m128i *) (pbDst + 0), _mm_aesenclast_si128( c0, tweakBuffer[ 8].m128i ) );
913 _mm_storeu_si128( (__m128i *) (pbDst + 16), _mm_aesenclast_si128( c1, tweakBuffer[ 9].m128i ) );
914 _mm_storeu_si128( (__m128i *) (pbDst + 32), _mm_aesenclast_si128( c2, tweakBuffer[10].m128i ) );
915 _mm_storeu_si128( (__m128i *) (pbDst + 48), _mm_aesenclast_si128( c3, tweakBuffer[11].m128i ) );
916 _mm_storeu_si128( (__m128i *) (pbDst + 64), _mm_aesenclast_si128( c4, tweakBuffer[12].m128i ) );
917 _mm_storeu_si128( (__m128i *) (pbDst + 80), _mm_aesenclast_si128( c5, tweakBuffer[13].m128i ) );
918 _mm_storeu_si128( (__m128i *) (pbDst + 96), _mm_aesenclast_si128( c6, tweakBuffer[14].m128i ) );
919 _mm_storeu_si128( (__m128i *) (pbDst + 112), _mm_aesenclast_si128( c7, tweakBuffer[15].m128i ) );
920
921 pbSrc += 8 * SYMCRYPT_AES_BLOCK_SIZE;
922 pbDst += 8 * SYMCRYPT_AES_BLOCK_SIZE;
923 cbDataMain -= 8 * SYMCRYPT_AES_BLOCK_SIZE;
924 }
925
926 if( cbDataTail == 0 )
927 {
928 return; // <-- expected case; early return here
929 }
930
931 // Rare case, with data unit length not being multiple of 128 bytes, handle the tail one block at a time
932 t0 = tweakBuffer[0].m128i;
933
934 while( cbDataTail >= 2*SYMCRYPT_AES_BLOCK_SIZE )
935 {
936 c0 = _mm_xor_si128( _mm_loadu_si128( ( __m128i * ) pbSrc ), t0 );
937 pbSrc += SYMCRYPT_AES_BLOCK_SIZE;
938 AES_ENCRYPT_1( pExpandedKey, c0 );
939 _mm_storeu_si128( (__m128i *) pbDst, _mm_xor_si128( c0, t0 ) );
940 pbDst += SYMCRYPT_AES_BLOCK_SIZE;
941 XTS_MUL_ALPHA( t0, t0 );
942 cbDataTail -= SYMCRYPT_AES_BLOCK_SIZE;
943 }
944
945 if( cbDataTail > SYMCRYPT_AES_BLOCK_SIZE )
946 {
947 // Ciphertext stealing encryption
948 //
949 // +--------------+
950 // | |
951 // | V
952 // +-----------------+ | +-----+-----------+
953 // | P_m-1 | | | P_m |++++CP+++++|
954 // +-----------------+ | +-----+-----------+
955 // | | |
956 // enc_m-1 | enc_m
957 // | | |
958 // V | V
959 // +-----+-----------+ | +-----------------+
960 // | C_m |++++CP+++++|--+ | C_m-1 |
961 // +-----+-----------+ +-----------------+
962 // | /
963 // +---------------- / --+
964 // / |
965 // | V
966 // +-----------------+ | +-----+
967 // | C_m-1 |<-+ | C_m |
968 // +-----------------+ +-----+
969
970 // Encrypt penultimate plaintext block into tweakBuffer[0]
971 c0 = _mm_xor_si128( _mm_loadu_si128( (__m128i *) pbSrc ), t0 );
972 AES_ENCRYPT_1( pExpandedKey, c0 );
973 tweakBuffer[0].m128i = _mm_xor_si128( c0, t0 );
974
975 cbDataTail -= SYMCRYPT_AES_BLOCK_SIZE;
976
977 // Copy tweakBuffer[0] to tweakBuffer[1]
978 tweakBuffer[1].m128i = tweakBuffer[0].m128i;
979 // Copy final plaintext bytes to prefix of tweakBuffer[0] - we must read before writing to support in-place encryption
980 memcpy( &tweakBuffer[0].ul[0], pbSrc + SYMCRYPT_AES_BLOCK_SIZE, cbDataTail );
981 // Copy prefix of tweakBuffer[1] to the right place in the destination buffer
982 memcpy( pbDst + SYMCRYPT_AES_BLOCK_SIZE, &tweakBuffer[1].ul[0], cbDataTail );
983
984 // Do final tweak update
985 XTS_MUL_ALPHA( t0, t0 );
986
987 // Load updated tweakBuffer[0] into c0
988 c0 = tweakBuffer[0].m128i;
989 } else {
990 // Just load final plaintext block into c0
991 c0 = _mm_loadu_si128( (__m128i*) pbSrc );
992 }
993
994 // Final full block encryption
995 c0 = _mm_xor_si128( c0, t0 );
996 AES_ENCRYPT_1( pExpandedKey, c0 );
997 _mm_storeu_si128( (__m128i *) pbDst, _mm_xor_si128( c0, t0 ) );
998}
999
1000VOID
1001SYMCRYPT_CALL
1002SymCryptXtsAesDecryptDataUnitXmm(
1003 _In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey,
1004 _In_reads_( SYMCRYPT_AES_BLOCK_SIZE ) PBYTE pbTweakBlock,
1005 _Out_writes_( SYMCRYPT_AES_BLOCK_SIZE*16 ) PBYTE pbScratch,
1006 _In_reads_( cbData ) PCBYTE pbSrc,
1007 _Out_writes_( cbData ) PBYTE pbDst,
1008 SIZE_T cbData )
1009{
1010 __m128i t0;
1011 __m128i c0, c1, c2, c3, c4, c5, c6, c7;
1012 __m128i roundkey, firstRoundKey, lastRoundKey;
1013 __m128i XTS_ALPHA_MASK = _mm_set_epi32( 1, 1, 1, 0x87 );
1014 SYMCRYPT_GF128_ELEMENT* tweakBuffer = (SYMCRYPT_GF128_ELEMENT*) pbScratch;
1015
1016 const BYTE (*keyPtr)[4][4];
1017 const BYTE (*keyLimit)[4][4] = pExpandedKey->lastDecRoundKey;
1018 UINT64 lastTweakLow, lastTweakHigh;
1019 int aesDecryptXtsLoop;
1020
1021 SIZE_T cbDataMain; // number of bytes to handle in the main loop
1022 SIZE_T cbDataTail; // number of bytes to handle in the tail loop
1023
1024 SYMCRYPT_ASSERT(cbData >= SYMCRYPT_AES_BLOCK_SIZE);
1025
1026 // To simplify logic and unusual size processing, we handle all
1027 // data not a multiple of 8 blocks in the tail loop
1028 cbDataTail = cbData & ((8*SYMCRYPT_AES_BLOCK_SIZE)-1);
1029 // Additionally, so that ciphertext stealing logic does not rely on
1030 // reading back from the destination buffer, when we have a non-zero
1031 // tail, we ensure that we handle at least 1 whole block in the tail
1032 //
1033 // Note that our caller has ensured we have at least 1 whole block
1034 // to process, this is checked in debug build
1035 // This means that cbDataTail is in [1,15] at this point iff there are
1036 // at least 8 whole blocks to process; so the below does not cause
1037 // cbDataTail or cbDataMain to exceed cbData
1038 cbDataTail += ((cbDataTail > 0) && (cbDataTail < SYMCRYPT_AES_BLOCK_SIZE)) ? (8*SYMCRYPT_AES_BLOCK_SIZE) : 0;
1039 cbDataMain = cbData - cbDataTail;
1040
1041 SYMCRYPT_ASSERT(cbDataMain <= cbData);
1042 SYMCRYPT_ASSERT(cbDataTail <= cbData);
1043 SYMCRYPT_ASSERT((cbDataMain & ((8*SYMCRYPT_AES_BLOCK_SIZE)-1)) == 0);
1044
1045 c0 = _mm_loadu_si128( (__m128i *) pbTweakBlock );
1046 XTS_MUL_ALPHA( c0, c1 );
1047 XTS_MUL_ALPHA( c1, c2 );
1048 XTS_MUL_ALPHA( c2, c3 );
1049
1050 XTS_MUL_ALPHA4( c0, c4 );
1051 XTS_MUL_ALPHA ( c4, c5 );
1052 XTS_MUL_ALPHA ( c5, c6 );
1053 XTS_MUL_ALPHA ( c6, c7 );
1054
1055 tweakBuffer[0].m128i = c0;
1056 tweakBuffer[1].m128i = c1;
1057 tweakBuffer[2].m128i = c2;
1058 tweakBuffer[3].m128i = c3;
1059 tweakBuffer[4].m128i = c4;
1060 tweakBuffer[5].m128i = c5;
1061 tweakBuffer[6].m128i = c6;
1062 tweakBuffer[7].m128i = c7;
1063 lastTweakLow = tweakBuffer[7].ull[0];
1064 lastTweakHigh = tweakBuffer[7].ull[1];
1065
1066 firstRoundKey = _mm_loadu_si128( (__m128i *) pExpandedKey->lastEncRoundKey );
1067 lastRoundKey = _mm_loadu_si128( (__m128i *) pExpandedKey->lastDecRoundKey );
1068
1069 while( cbDataMain > 0 )
1070 {
1071 // At loop entry, tweakBuffer[0-7] are tweakValues for the next 8 blocks
1072 c0 = _mm_xor_si128( tweakBuffer[0].m128i, firstRoundKey );
1073 c1 = _mm_xor_si128( tweakBuffer[1].m128i, firstRoundKey );
1074 c2 = _mm_xor_si128( tweakBuffer[2].m128i, firstRoundKey );
1075 c3 = _mm_xor_si128( tweakBuffer[3].m128i, firstRoundKey );
1076 c4 = _mm_xor_si128( tweakBuffer[4].m128i, firstRoundKey );
1077 c5 = _mm_xor_si128( tweakBuffer[5].m128i, firstRoundKey );
1078 c6 = _mm_xor_si128( tweakBuffer[6].m128i, firstRoundKey );
1079 c7 = _mm_xor_si128( tweakBuffer[7].m128i, firstRoundKey );
1080
1081 c0 = _mm_xor_si128( c0, _mm_loadu_si128( ( __m128i * ) (pbSrc + 0) ) );
1082 c1 = _mm_xor_si128( c1, _mm_loadu_si128( ( __m128i * ) (pbSrc + 16) ) );
1083 c2 = _mm_xor_si128( c2, _mm_loadu_si128( ( __m128i * ) (pbSrc + 32) ) );
1084 c3 = _mm_xor_si128( c3, _mm_loadu_si128( ( __m128i * ) (pbSrc + 48) ) );
1085 c4 = _mm_xor_si128( c4, _mm_loadu_si128( ( __m128i * ) (pbSrc + 64) ) );
1086 c5 = _mm_xor_si128( c5, _mm_loadu_si128( ( __m128i * ) (pbSrc + 80) ) );
1087 c6 = _mm_xor_si128( c6, _mm_loadu_si128( ( __m128i * ) (pbSrc + 96) ) );
1088 c7 = _mm_xor_si128( c7, _mm_loadu_si128( ( __m128i * ) (pbSrc + 112) ) );
1089
1090 keyPtr = pExpandedKey->lastEncRoundKey + 1;
1091
1092 // Do 8 full rounds (AES-128|AES-192|AES-256) with stitched XTS (performed in scalar registers)
1093 for( aesDecryptXtsLoop = 0; aesDecryptXtsLoop < 8; aesDecryptXtsLoop++ )
1094 {
1095 roundkey = _mm_loadu_si128( (__m128i *) keyPtr );
1096 keyPtr ++;
1097 c0 = _mm_aesdec_si128( c0, roundkey );
1098 c1 = _mm_aesdec_si128( c1, roundkey );
1099 c2 = _mm_aesdec_si128( c2, roundkey );
1100 c3 = _mm_aesdec_si128( c3, roundkey );
1101 c4 = _mm_aesdec_si128( c4, roundkey );
1102 c5 = _mm_aesdec_si128( c5, roundkey );
1103 c6 = _mm_aesdec_si128( c6, roundkey );
1104 c7 = _mm_aesdec_si128( c7, roundkey );
1105
1106 // Prepare tweakBuffer[8-15] with tweak^lastRoundKey
1107 tweakBuffer[ 8+aesDecryptXtsLoop ].m128i = _mm_xor_si128( tweakBuffer[ aesDecryptXtsLoop ].m128i, lastRoundKey );
1108 // Prepare tweakBuffer[0-7] with tweaks for next 8 blocks
1109 XTS_MUL_ALPHA_Scalar( lastTweakLow, lastTweakHigh );
1110 tweakBuffer[ aesDecryptXtsLoop ].ull[0] = lastTweakLow;
1111 tweakBuffer[ aesDecryptXtsLoop ].ull[1] = lastTweakHigh;
1112 }
1113
1114 do
1115 {
1116 roundkey = _mm_loadu_si128( (__m128i *) keyPtr );
1117 keyPtr ++;
1118 c0 = _mm_aesdec_si128( c0, roundkey );
1119 c1 = _mm_aesdec_si128( c1, roundkey );
1120 c2 = _mm_aesdec_si128( c2, roundkey );
1121 c3 = _mm_aesdec_si128( c3, roundkey );
1122 c4 = _mm_aesdec_si128( c4, roundkey );
1123 c5 = _mm_aesdec_si128( c5, roundkey );
1124 c6 = _mm_aesdec_si128( c6, roundkey );
1125 c7 = _mm_aesdec_si128( c7, roundkey );
1126 } while( keyPtr < keyLimit );
1127
1128 _mm_storeu_si128( (__m128i *) (pbDst + 0), _mm_aesdeclast_si128( c0, tweakBuffer[ 8].m128i ) );
1129 _mm_storeu_si128( (__m128i *) (pbDst + 16), _mm_aesdeclast_si128( c1, tweakBuffer[ 9].m128i ) );
1130 _mm_storeu_si128( (__m128i *) (pbDst + 32), _mm_aesdeclast_si128( c2, tweakBuffer[10].m128i ) );
1131 _mm_storeu_si128( (__m128i *) (pbDst + 48), _mm_aesdeclast_si128( c3, tweakBuffer[11].m128i ) );
1132 _mm_storeu_si128( (__m128i *) (pbDst + 64), _mm_aesdeclast_si128( c4, tweakBuffer[12].m128i ) );
1133 _mm_storeu_si128( (__m128i *) (pbDst + 80), _mm_aesdeclast_si128( c5, tweakBuffer[13].m128i ) );
1134 _mm_storeu_si128( (__m128i *) (pbDst + 96), _mm_aesdeclast_si128( c6, tweakBuffer[14].m128i ) );
1135 _mm_storeu_si128( (__m128i *) (pbDst + 112), _mm_aesdeclast_si128( c7, tweakBuffer[15].m128i ) );
1136
1137 pbSrc += 8 * SYMCRYPT_AES_BLOCK_SIZE;
1138 pbDst += 8 * SYMCRYPT_AES_BLOCK_SIZE;
1139 cbDataMain -= 8 * SYMCRYPT_AES_BLOCK_SIZE;
1140 }
1141
1142 if( cbDataTail == 0 )
1143 {
1144 return; // <-- expected case; early return here
1145 }
1146
1147 // Rare case, with data unit length not being multiple of 128 bytes, handle the tail one block at a time
1148 t0 = tweakBuffer[0].m128i;
1149
1150 while( cbDataTail >= 2*SYMCRYPT_AES_BLOCK_SIZE )
1151 {
1152 c0 = _mm_xor_si128( _mm_loadu_si128( ( __m128i * ) pbSrc ), t0 );
1153 pbSrc += SYMCRYPT_AES_BLOCK_SIZE;
1154 AES_DECRYPT_1( pExpandedKey, c0 );
1155 _mm_storeu_si128( (__m128i *) pbDst, _mm_xor_si128( c0, t0 ) );
1156 pbDst += SYMCRYPT_AES_BLOCK_SIZE;
1157 c7 = t0;
1158 XTS_MUL_ALPHA( t0, t0 );
1159 cbDataTail -= SYMCRYPT_AES_BLOCK_SIZE;
1160 }
1161
1162 if( cbDataTail > SYMCRYPT_AES_BLOCK_SIZE )
1163 {
1164 // Ciphertext stealing decryption
1165 //
1166 // +--------------+
1167 // | |
1168 // | V
1169 // +-----------------+ | +-----+-----------+
1170 // | C_m-1 | | | C_m |++++CP+++++|
1171 // +-----------------+ | +-----+-----------+
1172 // | | |
1173 // dec_m | dec_m-1
1174 // | | |
1175 // V | V
1176 // +-----+-----------+ | +-----------------+
1177 // | P_m |++++CP+++++|--+ | P_m-1 |
1178 // +-----+-----------+ +-----------------+
1179 // | /
1180 // +---------------- / --+
1181 // / |
1182 // | V
1183 // +-----------------+ | +-----+
1184 // | P_m-1 |<-+ | P_m |
1185 // +-----------------+ +-----+
1186
1187 // Do final tweak update into c1
1188 // Penultimate tweak is in t0, ready for final decryption
1189 XTS_MUL_ALPHA( t0, c1 );
1190
1191 // Decrypt penultimate ciphertext block into tweakBuffer[0]
1192 c0 = _mm_xor_si128( _mm_loadu_si128( (__m128i *) pbSrc ), c1 );
1193 AES_DECRYPT_1( pExpandedKey, c0 );
1194 tweakBuffer[0].m128i = _mm_xor_si128( c0, c1 );
1195
1196 cbDataTail -= SYMCRYPT_AES_BLOCK_SIZE;
1197
1198 // Copy tweakBuffer[0] to tweakBuffer[1]
1199 tweakBuffer[1].m128i = tweakBuffer[0].m128i;
1200 // Copy final ciphertext bytes to prefix of tweakBuffer[0] - we must read before writing to support in-place decryption
1201 memcpy( &tweakBuffer[0].ul[0], pbSrc + SYMCRYPT_AES_BLOCK_SIZE, cbDataTail );
1202 // Copy prefix of tweakBuffer[1] to the right place in the destination buffer
1203 memcpy( pbDst + SYMCRYPT_AES_BLOCK_SIZE, &tweakBuffer[1].ul[0], cbDataTail );
1204
1205 // Load updated tweakBuffer[0] into c0
1206 c0 = tweakBuffer[0].m128i;
1207 } else {
1208 // Just load final ciphertext block into c0
1209 c0 = _mm_loadu_si128( (__m128i*) pbSrc );
1210 }
1211
1212 // Final full block decryption
1213 c0 = _mm_xor_si128( c0, t0 );
1214 AES_DECRYPT_1( pExpandedKey, c0 );
1215 _mm_storeu_si128( (__m128i *) pbDst, _mm_xor_si128( c0, t0 ) );
1216}
1217
1218#define AES_FULLROUND_4_GHASH_1( roundkey, keyPtr, c0, c1, c2, c3, r0, t0, t1, gHashPointer, byteReverseOrder, gHashExpandedKeyTable, todo, resl, resm, resh ) \
1219{ \
1220 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
1221 keyPtr ++; \
1222 c0 = _mm_aesenc_si128( c0, roundkey ); \
1223 c1 = _mm_aesenc_si128( c1, roundkey ); \
1224 c2 = _mm_aesenc_si128( c2, roundkey ); \
1225 c3 = _mm_aesenc_si128( c3, roundkey ); \
1226\
1227 r0 = _mm_loadu_si128( (__m128i *) gHashPointer ); \
1228 r0 = _mm_shuffle_epi8( r0, byteReverseOrder ); \
1229 gHashPointer += 16; \
1230\
1231 t1 = _mm_loadu_si128( (__m128i *) &GHASH_H_POWER(gHashExpandedKeyTable, todo) ); \
1232 t0 = _mm_clmulepi64_si128( r0, t1, 0x00 ); \
1233 t1 = _mm_clmulepi64_si128( r0, t1, 0x11 ); \
1234\
1235 resl = _mm_xor_si128( resl, t0 ); \
1236 resh = _mm_xor_si128( resh, t1 ); \
1237\
1238 t0 = _mm_srli_si128( r0, 8 ); \
1239 r0 = _mm_xor_si128( r0, t0 ); \
1240 t1 = _mm_loadu_si128( (__m128i *) &GHASH_Hx_POWER(gHashExpandedKeyTable, todo) ); \
1241 t1 = _mm_clmulepi64_si128( r0, t1, 0x00 ); \
1242\
1243 resm = _mm_xor_si128( resm, t1 ); \
1244 todo --; \
1245};
1246
1247#define AES_GCM_ENCRYPT_4( pExpandedKey, c0, c1, c2, c3, gHashPointer, ghashRounds, byteReverseOrder, gHashExpandedKeyTable, todo, resl, resm, resh ) \
1248{ \
1249 const BYTE (*keyPtr)[4][4]; \
1250 const BYTE (*keyLimit)[4][4]; \
1251 __m128i roundkey; \
1252 __m128i t0, t1; \
1253 __m128i r0; \
1254 SIZE_T aesEncryptGhashLoop; \
1255\
1256 keyPtr = &pExpandedKey->RoundKey[0]; \
1257 keyLimit = pExpandedKey->lastEncRoundKey; \
1258\
1259 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
1260 keyPtr ++; \
1261 c0 = _mm_xor_si128( c0, roundkey ); \
1262 c1 = _mm_xor_si128( c1, roundkey ); \
1263 c2 = _mm_xor_si128( c2, roundkey ); \
1264 c3 = _mm_xor_si128( c3, roundkey ); \
1265\
1266 /* Do ghashRounds full rounds (AES-128|AES-192|AES-256) with stitched GHASH */ \
1267 for( aesEncryptGhashLoop = 0; aesEncryptGhashLoop < ghashRounds; aesEncryptGhashLoop++ ) \
1268 { \
1269 AES_FULLROUND_4_GHASH_1( roundkey, keyPtr, c0, c1, c2, c3, r0, t0, t1, gHashPointer, byteReverseOrder, gHashExpandedKeyTable, todo, resl, resm, resh ); \
1270 } \
1271\
1272 do \
1273 { \
1274 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
1275 keyPtr ++; \
1276 c0 = _mm_aesenc_si128( c0, roundkey ); \
1277 c1 = _mm_aesenc_si128( c1, roundkey ); \
1278 c2 = _mm_aesenc_si128( c2, roundkey ); \
1279 c3 = _mm_aesenc_si128( c3, roundkey ); \
1280 } while( keyPtr < keyLimit ); \
1281\
1282 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
1283\
1284 c0 = _mm_aesenclast_si128( c0, roundkey ); \
1285 c1 = _mm_aesenclast_si128( c1, roundkey ); \
1286 c2 = _mm_aesenclast_si128( c2, roundkey ); \
1287 c3 = _mm_aesenclast_si128( c3, roundkey ); \
1288};
1289
1290#define AES_FULLROUND_8_GHASH_1( roundkey, keyPtr, c0, c1, c2, c3, c4, c5, c6, c7, r0, t0, t1, gHashPointer, byteReverseOrder, gHashExpandedKeyTable, todo, resl, resm, resh ) \
1291{ \
1292 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
1293 keyPtr ++; \
1294 c0 = _mm_aesenc_si128( c0, roundkey ); \
1295 c1 = _mm_aesenc_si128( c1, roundkey ); \
1296 c2 = _mm_aesenc_si128( c2, roundkey ); \
1297 c3 = _mm_aesenc_si128( c3, roundkey ); \
1298 c4 = _mm_aesenc_si128( c4, roundkey ); \
1299 c5 = _mm_aesenc_si128( c5, roundkey ); \
1300 c6 = _mm_aesenc_si128( c6, roundkey ); \
1301 c7 = _mm_aesenc_si128( c7, roundkey ); \
1302\
1303 r0 = _mm_loadu_si128( (__m128i *) gHashPointer ); \
1304 r0 = _mm_shuffle_epi8( r0, byteReverseOrder ); \
1305 gHashPointer += 16; \
1306\
1307 t1 = _mm_loadu_si128( (__m128i *) &GHASH_H_POWER(gHashExpandedKeyTable, todo) ); \
1308 t0 = _mm_clmulepi64_si128( r0, t1, 0x00 ); \
1309 t1 = _mm_clmulepi64_si128( r0, t1, 0x11 ); \
1310\
1311 resl = _mm_xor_si128( resl, t0 ); \
1312 resh = _mm_xor_si128( resh, t1 ); \
1313\
1314 t0 = _mm_srli_si128( r0, 8 ); \
1315 r0 = _mm_xor_si128( r0, t0 ); \
1316 t1 = _mm_loadu_si128( (__m128i *) &GHASH_Hx_POWER(gHashExpandedKeyTable, todo) ); \
1317 t1 = _mm_clmulepi64_si128( r0, t1, 0x00 ); \
1318\
1319 resm = _mm_xor_si128( resm, t1 ); \
1320 todo --; \
1321};
1322
1323#define AES_GCM_ENCRYPT_8( pExpandedKey, c0, c1, c2, c3, c4, c5, c6, c7, gHashPointer, ghashRounds, byteReverseOrder, gHashExpandedKeyTable, todo, resl, resm, resh ) \
1324{ \
1325 const BYTE (*keyPtr)[4][4]; \
1326 const BYTE (*keyLimit)[4][4]; \
1327 __m128i roundkey; \
1328 __m128i t0, t1; \
1329 __m128i r0; \
1330 SIZE_T aesEncryptGhashLoop; \
1331\
1332 keyPtr = &pExpandedKey->RoundKey[0]; \
1333 keyLimit = pExpandedKey->lastEncRoundKey; \
1334\
1335 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
1336 keyPtr ++; \
1337 c0 = _mm_xor_si128( c0, roundkey ); \
1338 c1 = _mm_xor_si128( c1, roundkey ); \
1339 c2 = _mm_xor_si128( c2, roundkey ); \
1340 c3 = _mm_xor_si128( c3, roundkey ); \
1341 c4 = _mm_xor_si128( c4, roundkey ); \
1342 c5 = _mm_xor_si128( c5, roundkey ); \
1343 c6 = _mm_xor_si128( c6, roundkey ); \
1344 c7 = _mm_xor_si128( c7, roundkey ); \
1345\
1346 /* Do ghashRounds full rounds (AES-128|AES-192|AES-256) with stitched GHASH */ \
1347 for( aesEncryptGhashLoop = 0; aesEncryptGhashLoop < ghashRounds; aesEncryptGhashLoop++ ) \
1348 { \
1349 AES_FULLROUND_8_GHASH_1( roundkey, keyPtr, c0, c1, c2, c3, c4, c5, c6, c7, r0, t0, t1, gHashPointer, byteReverseOrder, gHashExpandedKeyTable, todo, resl, resm, resh ); \
1350 } \
1351\
1352 do \
1353 { \
1354 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
1355 keyPtr ++; \
1356 c0 = _mm_aesenc_si128( c0, roundkey ); \
1357 c1 = _mm_aesenc_si128( c1, roundkey ); \
1358 c2 = _mm_aesenc_si128( c2, roundkey ); \
1359 c3 = _mm_aesenc_si128( c3, roundkey ); \
1360 c4 = _mm_aesenc_si128( c4, roundkey ); \
1361 c5 = _mm_aesenc_si128( c5, roundkey ); \
1362 c6 = _mm_aesenc_si128( c6, roundkey ); \
1363 c7 = _mm_aesenc_si128( c7, roundkey ); \
1364 } while( keyPtr < keyLimit ); \
1365\
1366 roundkey = _mm_loadu_si128( (__m128i *) keyPtr ); \
1367\
1368 c0 = _mm_aesenclast_si128( c0, roundkey ); \
1369 c1 = _mm_aesenclast_si128( c1, roundkey ); \
1370 c2 = _mm_aesenclast_si128( c2, roundkey ); \
1371 c3 = _mm_aesenclast_si128( c3, roundkey ); \
1372 c4 = _mm_aesenclast_si128( c4, roundkey ); \
1373 c5 = _mm_aesenclast_si128( c5, roundkey ); \
1374 c6 = _mm_aesenclast_si128( c6, roundkey ); \
1375 c7 = _mm_aesenclast_si128( c7, roundkey ); \
1376};
1377
1378// This call is functionally identical to:
1379// SymCryptAesCtrMsb64Xmm( pExpandedKey,
1380// pbChainingValue,
1381// pbSrc,
1382// pbDst,
1383// cbData );
1384// SymCryptGHashAppendDataPclmulqdq( expandedKeyTable,
1385// pState,
1386// pbDst,
1387// cbData );
1388VOID
1389SYMCRYPT_CALL
1390SymCryptAesGcmEncryptStitchedXmm(
1391 _In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey,
1392 _In_reads_( SYMCRYPT_AES_BLOCK_SIZE ) PBYTE pbChainingValue,
1393 _In_reads_( SYMCRYPT_GF128_FIELD_SIZE ) PCSYMCRYPT_GF128_ELEMENT expandedKeyTable,
1394 _Inout_ PSYMCRYPT_GF128_ELEMENT pState,
1395 _In_reads_( cbData ) PCBYTE pbSrc,
1396 _Out_writes_( cbData ) PBYTE pbDst,
1397 SIZE_T cbData )
1398{
1399 __m128i chain = _mm_loadu_si128( (__m128i *) pbChainingValue );
1400
1401 __m128i BYTE_REVERSE_ORDER = _mm_set_epi8(
1402 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 );
1403 __m128i vMultiplicationConstant = _mm_set_epi32( 0, 0, 0xc2000000, 0 );
1404
1405 __m128i chainIncrement1 = _mm_set_epi32( 0, 0, 0, 1 );
1406 __m128i chainIncrement2 = _mm_set_epi32( 0, 0, 0, 2 );
1407 __m128i chainIncrement8 = _mm_set_epi32( 0, 0, 0, 8 );
1408
1409 __m128i c0, c1, c2, c3, c4, c5, c6, c7;
1410 __m128i r0, r1;
1411
1412 __m128i state;
1413 __m128i a0, a1, a2;
1414 SIZE_T nBlocks = cbData / SYMCRYPT_GF128_BLOCK_SIZE;
1415 SIZE_T todo;
1416 PCBYTE pbGhashSrc = pbDst;
1417
1418 SYMCRYPT_ASSERT( (cbData & SYMCRYPT_GCM_BLOCK_MOD_MASK) == 0 ); // cbData is multiple of block size
1419
1420 chain = _mm_shuffle_epi8( chain, BYTE_REVERSE_ORDER );
1421 state = _mm_loadu_si128( (__m128i *) pState );
1422
1423 todo = SYMCRYPT_MIN( nBlocks, SYMCRYPT_GHASH_PCLMULQDQ_HPOWERS );
1424 CLMUL_3( state, GHASH_H_POWER(expandedKeyTable, todo), GHASH_Hx_POWER(expandedKeyTable, todo), a0, a1, a2 );
1425
1426 // Do 8 blocks of CTR either for tail (if total blocks <8) or for encryption of first 8 blocks
1427 c0 = chain;
1428 c1 = _mm_add_epi32( chain, chainIncrement1 );
1429 c2 = _mm_add_epi32( chain, chainIncrement2 );
1430 c3 = _mm_add_epi32( c1, chainIncrement2 );
1431 c4 = _mm_add_epi32( c2, chainIncrement2 );
1432 c5 = _mm_add_epi32( c3, chainIncrement2 );
1433 c6 = _mm_add_epi32( c4, chainIncrement2 );
1434 c7 = _mm_add_epi32( c5, chainIncrement2 );
1435
1436 c0 = _mm_shuffle_epi8( c0, BYTE_REVERSE_ORDER );
1437 c1 = _mm_shuffle_epi8( c1, BYTE_REVERSE_ORDER );
1438 c2 = _mm_shuffle_epi8( c2, BYTE_REVERSE_ORDER );
1439 c3 = _mm_shuffle_epi8( c3, BYTE_REVERSE_ORDER );
1440 c4 = _mm_shuffle_epi8( c4, BYTE_REVERSE_ORDER );
1441 c5 = _mm_shuffle_epi8( c5, BYTE_REVERSE_ORDER );
1442 c6 = _mm_shuffle_epi8( c6, BYTE_REVERSE_ORDER );
1443 c7 = _mm_shuffle_epi8( c7, BYTE_REVERSE_ORDER );
1444
1445 AES_ENCRYPT_8( pExpandedKey, c0, c1, c2, c3, c4, c5, c6, c7 );
1446
1447 if( nBlocks >= 8 )
1448 {
1449 // Encrypt first 8 blocks - update chain
1450 chain = _mm_add_epi32( chain, chainIncrement8 );
1451
1452 _mm_storeu_si128( (__m128i *) (pbDst + 0), _mm_xor_si128( c0, _mm_loadu_si128( ( __m128i * ) (pbSrc + 0) ) ) );
1453 _mm_storeu_si128( (__m128i *) (pbDst + 16), _mm_xor_si128( c1, _mm_loadu_si128( ( __m128i * ) (pbSrc + 16) ) ) );
1454 _mm_storeu_si128( (__m128i *) (pbDst + 32), _mm_xor_si128( c2, _mm_loadu_si128( ( __m128i * ) (pbSrc + 32) ) ) );
1455 _mm_storeu_si128( (__m128i *) (pbDst + 48), _mm_xor_si128( c3, _mm_loadu_si128( ( __m128i * ) (pbSrc + 48) ) ) );
1456 _mm_storeu_si128( (__m128i *) (pbDst + 64), _mm_xor_si128( c4, _mm_loadu_si128( ( __m128i * ) (pbSrc + 64) ) ) );
1457 _mm_storeu_si128( (__m128i *) (pbDst + 80), _mm_xor_si128( c5, _mm_loadu_si128( ( __m128i * ) (pbSrc + 80) ) ) );
1458 _mm_storeu_si128( (__m128i *) (pbDst + 96), _mm_xor_si128( c6, _mm_loadu_si128( ( __m128i * ) (pbSrc + 96) ) ) );
1459 _mm_storeu_si128( (__m128i *) (pbDst +112), _mm_xor_si128( c7, _mm_loadu_si128( ( __m128i * ) (pbSrc +112) ) ) );
1460
1461 pbDst += 8 * SYMCRYPT_AES_BLOCK_SIZE;
1462 pbSrc += 8 * SYMCRYPT_AES_BLOCK_SIZE;
1463
1464 while( nBlocks >= 16 )
1465 {
1466 // In this loop we always have 8 blocks to encrypt and we have already encrypted the previous 8 blocks ready for GHASH
1467 c0 = chain;
1468 c1 = _mm_add_epi32( chain, chainIncrement1 );
1469 c2 = _mm_add_epi32( chain, chainIncrement2 );
1470 c3 = _mm_add_epi32( c1, chainIncrement2 );
1471 c4 = _mm_add_epi32( c2, chainIncrement2 );
1472 c5 = _mm_add_epi32( c3, chainIncrement2 );
1473 c6 = _mm_add_epi32( c4, chainIncrement2 );
1474 c7 = _mm_add_epi32( c5, chainIncrement2 );
1475 chain = _mm_add_epi32( c6, chainIncrement2 );
1476
1477 c0 = _mm_shuffle_epi8( c0, BYTE_REVERSE_ORDER );
1478 c1 = _mm_shuffle_epi8( c1, BYTE_REVERSE_ORDER );
1479 c2 = _mm_shuffle_epi8( c2, BYTE_REVERSE_ORDER );
1480 c3 = _mm_shuffle_epi8( c3, BYTE_REVERSE_ORDER );
1481 c4 = _mm_shuffle_epi8( c4, BYTE_REVERSE_ORDER );
1482 c5 = _mm_shuffle_epi8( c5, BYTE_REVERSE_ORDER );
1483 c6 = _mm_shuffle_epi8( c6, BYTE_REVERSE_ORDER );
1484 c7 = _mm_shuffle_epi8( c7, BYTE_REVERSE_ORDER );
1485
1486 AES_GCM_ENCRYPT_8( pExpandedKey, c0, c1, c2, c3, c4, c5, c6, c7, pbGhashSrc, 8, BYTE_REVERSE_ORDER, expandedKeyTable, todo, a0, a1, a2 );
1487
1488 _mm_storeu_si128( (__m128i *) (pbDst + 0), _mm_xor_si128( c0, _mm_loadu_si128( ( __m128i * ) (pbSrc + 0) ) ) );
1489 _mm_storeu_si128( (__m128i *) (pbDst + 16), _mm_xor_si128( c1, _mm_loadu_si128( ( __m128i * ) (pbSrc + 16) ) ) );
1490 _mm_storeu_si128( (__m128i *) (pbDst + 32), _mm_xor_si128( c2, _mm_loadu_si128( ( __m128i * ) (pbSrc + 32) ) ) );
1491 _mm_storeu_si128( (__m128i *) (pbDst + 48), _mm_xor_si128( c3, _mm_loadu_si128( ( __m128i * ) (pbSrc + 48) ) ) );
1492 _mm_storeu_si128( (__m128i *) (pbDst + 64), _mm_xor_si128( c4, _mm_loadu_si128( ( __m128i * ) (pbSrc + 64) ) ) );
1493 _mm_storeu_si128( (__m128i *) (pbDst + 80), _mm_xor_si128( c5, _mm_loadu_si128( ( __m128i * ) (pbSrc + 80) ) ) );
1494 _mm_storeu_si128( (__m128i *) (pbDst + 96), _mm_xor_si128( c6, _mm_loadu_si128( ( __m128i * ) (pbSrc + 96) ) ) );
1495 _mm_storeu_si128( (__m128i *) (pbDst +112), _mm_xor_si128( c7, _mm_loadu_si128( ( __m128i * ) (pbSrc +112) ) ) );
1496
1497 pbDst += 8 * SYMCRYPT_AES_BLOCK_SIZE;
1498 pbSrc += 8 * SYMCRYPT_AES_BLOCK_SIZE;
1499 nBlocks -= 8;
1500
1501 if( todo == 0 )
1502 {
1503 CLMUL_3_POST( a0, a1, a2 );
1504 MODREDUCE( vMultiplicationConstant, a0, a1, a2, state );
1505
1506 todo = SYMCRYPT_MIN( nBlocks, SYMCRYPT_GHASH_PCLMULQDQ_HPOWERS );
1507 CLMUL_3( state, GHASH_H_POWER(expandedKeyTable, todo), GHASH_Hx_POWER(expandedKeyTable, todo), a0, a1, a2 );
1508 }
1509 }
1510
1511 // We now have at least 8 blocks of encrypted data to GHASH and at most 7 blocks left to encrypt
1512 // Do 8 blocks of GHASH in parallel with generating 0, 4, or 8 AES-CTR blocks for tail encryption
1513 nBlocks -= 8;
1514 if (nBlocks > 0)
1515 {
1516 c0 = chain;
1517 c1 = _mm_add_epi32( chain, chainIncrement1 );
1518 c2 = _mm_add_epi32( chain, chainIncrement2 );
1519 c3 = _mm_add_epi32( c1, chainIncrement2 );
1520 c4 = _mm_add_epi32( c2, chainIncrement2 );
1521
1522 c0 = _mm_shuffle_epi8( c0, BYTE_REVERSE_ORDER );
1523 c1 = _mm_shuffle_epi8( c1, BYTE_REVERSE_ORDER );
1524 c2 = _mm_shuffle_epi8( c2, BYTE_REVERSE_ORDER );
1525 c3 = _mm_shuffle_epi8( c3, BYTE_REVERSE_ORDER );
1526
1527 if (nBlocks > 4)
1528 {
1529 // Do 8 rounds of AES-CTR for tail in parallel with 8 rounds of GHASH
1530 c5 = _mm_add_epi32( c4, chainIncrement1 );
1531 c6 = _mm_add_epi32( c4, chainIncrement2 );
1532
1533 c4 = _mm_shuffle_epi8( c4, BYTE_REVERSE_ORDER );
1534 c5 = _mm_shuffle_epi8( c5, BYTE_REVERSE_ORDER );
1535 c6 = _mm_shuffle_epi8( c6, BYTE_REVERSE_ORDER );
1536
1537 AES_GCM_ENCRYPT_8( pExpandedKey, c0, c1, c2, c3, c4, c5, c6, c7, pbGhashSrc, 8, BYTE_REVERSE_ORDER, expandedKeyTable, todo, a0, a1, a2 );
1538 }
1539 else
1540 {
1541 // Do 4 rounds of AES-CTR for tail in parallel with 8 rounds of GHASH
1542 AES_GCM_ENCRYPT_4( pExpandedKey, c0, c1, c2, c3, pbGhashSrc, 8, BYTE_REVERSE_ORDER, expandedKeyTable, todo, a0, a1, a2 );
1543 }
1544
1545 if( todo == 0)
1546 {
1547 CLMUL_3_POST( a0, a1, a2 );
1548 MODREDUCE( vMultiplicationConstant, a0, a1, a2, state );
1549
1550 todo = SYMCRYPT_MIN( nBlocks, SYMCRYPT_GHASH_PCLMULQDQ_HPOWERS );
1551 CLMUL_3( state, GHASH_H_POWER(expandedKeyTable, todo), GHASH_Hx_POWER(expandedKeyTable, todo), a0, a1, a2 );
1552 }
1553 }
1554 else
1555 {
1556 // Just do the final 8 rounds of GHASH
1557 for( todo=8; todo>0; todo-- )
1558 {
1559 r0 = _mm_shuffle_epi8( _mm_loadu_si128( (__m128i *) (pbGhashSrc + 0) ), BYTE_REVERSE_ORDER );
1560 pbGhashSrc += SYMCRYPT_AES_BLOCK_SIZE;
1561
1562 CLMUL_ACC_3( r0, GHASH_H_POWER(expandedKeyTable, todo), GHASH_Hx_POWER(expandedKeyTable, todo), a0, a1, a2 );
1563 }
1564
1565 CLMUL_3_POST( a0, a1, a2 );
1566 MODREDUCE( vMultiplicationConstant, a0, a1, a2, state );
1567 }
1568 }
1569
1570 if( nBlocks > 0 )
1571 {
1572 // Encrypt 1-7 blocks with pre-generated AES-CTR blocks and GHASH the results
1573 while( nBlocks >= 2 )
1574 {
1575 chain = _mm_add_epi32( chain, chainIncrement2 );
1576
1577 r0 = _mm_xor_si128( c0, _mm_loadu_si128( ( __m128i * ) (pbSrc + 0) ) );
1578 r1 = _mm_xor_si128( c1, _mm_loadu_si128( ( __m128i * ) (pbSrc + 16) ) );
1579
1580 _mm_storeu_si128( (__m128i *) (pbDst + 0), r0 );
1581 _mm_storeu_si128( (__m128i *) (pbDst + 16), r1 );
1582
1583 r0 = _mm_shuffle_epi8( r0, BYTE_REVERSE_ORDER );
1584 r1 = _mm_shuffle_epi8( r1, BYTE_REVERSE_ORDER );
1585
1586 CLMUL_ACC_3( r0, GHASH_H_POWER(expandedKeyTable, todo - 0), GHASH_Hx_POWER(expandedKeyTable, todo - 0), a0, a1, a2 );
1587 CLMUL_ACC_3( r1, GHASH_H_POWER(expandedKeyTable, todo - 1), GHASH_Hx_POWER(expandedKeyTable, todo - 1), a0, a1, a2 );
1588
1589 pbDst += 2*SYMCRYPT_AES_BLOCK_SIZE;
1590 pbSrc += 2*SYMCRYPT_AES_BLOCK_SIZE;
1591 todo -= 2;
1592 nBlocks -= 2;
1593 c0 = c2;
1594 c1 = c3;
1595 c2 = c4;
1596 c3 = c5;
1597 c4 = c6;
1598 }
1599
1600 if( nBlocks > 0 )
1601 {
1602 chain = _mm_add_epi32( chain, chainIncrement1 );
1603
1604 r0 = _mm_xor_si128( c0, _mm_loadu_si128( ( __m128i * ) (pbSrc + 0) ) );
1605
1606 _mm_storeu_si128( (__m128i *) (pbDst + 0), r0 );
1607
1608 r0 = _mm_shuffle_epi8( r0, BYTE_REVERSE_ORDER );
1609
1610 CLMUL_ACC_3( r0, GHASH_H_POWER(expandedKeyTable, 1), GHASH_Hx_POWER(expandedKeyTable, 1), a0, a1, a2 );
1611 }
1612
1613 CLMUL_3_POST( a0, a1, a2 );
1614 MODREDUCE( vMultiplicationConstant, a0, a1, a2, state );
1615 }
1616
1617 chain = _mm_shuffle_epi8( chain, BYTE_REVERSE_ORDER );
1618 _mm_storeu_si128( (__m128i *) pbChainingValue, chain );
1619 _mm_storeu_si128( (__m128i *) pState, state );
1620}
1621
1622#pragma warning(push)
1623#pragma warning( disable:4701 )
1624#pragma runtime_checks( "u", off )
1625// This call is functionally identical to:
1626// SymCryptGHashAppendDataPclmulqdq( expandedKeyTable,
1627// pState,
1628// pbSrc,
1629// cbData );
1630// SymCryptAesCtrMsb64Xmm( pExpandedKey,
1631// pbChainingValue,
1632// pbSrc,
1633// pbDst,
1634// cbData );
1635VOID
1636SYMCRYPT_CALL
1637SymCryptAesGcmDecryptStitchedXmm(
1638 _In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey,
1639 _In_reads_( SYMCRYPT_AES_BLOCK_SIZE ) PBYTE pbChainingValue,
1640 _In_reads_( SYMCRYPT_GF128_FIELD_SIZE ) PCSYMCRYPT_GF128_ELEMENT expandedKeyTable,
1641 _Inout_ PSYMCRYPT_GF128_ELEMENT pState,
1642 _In_reads_( cbData ) PCBYTE pbSrc,
1643 _Out_writes_( cbData ) PBYTE pbDst,
1644 SIZE_T cbData )
1645{
1646 __m128i chain = _mm_loadu_si128( (__m128i *) pbChainingValue );
1647
1648 __m128i BYTE_REVERSE_ORDER = _mm_set_epi8(
1649 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 );
1650 __m128i vMultiplicationConstant = _mm_set_epi32( 0, 0, 0xc2000000, 0 );
1651
1652 __m128i chainIncrement1 = _mm_set_epi32( 0, 0, 0, 1 );
1653 __m128i chainIncrement2 = _mm_set_epi32( 0, 0, 0, 2 );
1654
1655 __m128i c0, c1, c2, c3, c4, c5, c6, c7;
1656
1657 __m128i state;
1658 __m128i a0, a1, a2;
1659 SIZE_T nBlocks = cbData / SYMCRYPT_GF128_BLOCK_SIZE;
1660 SIZE_T todo = 0;
1661 PCBYTE pbGhashSrc = pbSrc;
1662
1663 SYMCRYPT_ASSERT( (cbData & SYMCRYPT_GCM_BLOCK_MOD_MASK) == 0 ); // cbData is multiple of block size
1664
1665 chain = _mm_shuffle_epi8( chain, BYTE_REVERSE_ORDER );
1666 state = _mm_loadu_si128( (__m128i *) pState );
1667
1668 todo = SYMCRYPT_MIN( nBlocks, SYMCRYPT_GHASH_PCLMULQDQ_HPOWERS );
1669 CLMUL_3( state, GHASH_H_POWER(expandedKeyTable, todo), GHASH_Hx_POWER(expandedKeyTable, todo), a0, a1, a2 );
1670
1671 while( nBlocks >= 8 )
1672 {
1673 // In this loop we always have 8 blocks to decrypt and GHASH
1674 c0 = chain;
1675 c1 = _mm_add_epi32( chain, chainIncrement1 );
1676 c2 = _mm_add_epi32( chain, chainIncrement2 );
1677 c3 = _mm_add_epi32( c1, chainIncrement2 );
1678 c4 = _mm_add_epi32( c2, chainIncrement2 );
1679 c5 = _mm_add_epi32( c3, chainIncrement2 );
1680 c6 = _mm_add_epi32( c4, chainIncrement2 );
1681 c7 = _mm_add_epi32( c5, chainIncrement2 );
1682 chain = _mm_add_epi32( c6, chainIncrement2 );
1683
1684 c0 = _mm_shuffle_epi8( c0, BYTE_REVERSE_ORDER );
1685 c1 = _mm_shuffle_epi8( c1, BYTE_REVERSE_ORDER );
1686 c2 = _mm_shuffle_epi8( c2, BYTE_REVERSE_ORDER );
1687 c3 = _mm_shuffle_epi8( c3, BYTE_REVERSE_ORDER );
1688 c4 = _mm_shuffle_epi8( c4, BYTE_REVERSE_ORDER );
1689 c5 = _mm_shuffle_epi8( c5, BYTE_REVERSE_ORDER );
1690 c6 = _mm_shuffle_epi8( c6, BYTE_REVERSE_ORDER );
1691 c7 = _mm_shuffle_epi8( c7, BYTE_REVERSE_ORDER );
1692
1693 AES_GCM_ENCRYPT_8( pExpandedKey, c0, c1, c2, c3, c4, c5, c6, c7, pbGhashSrc, 8, BYTE_REVERSE_ORDER, expandedKeyTable, todo, a0, a1, a2 );
1694
1695 _mm_storeu_si128( (__m128i *) (pbDst + 0), _mm_xor_si128( c0, _mm_loadu_si128( ( __m128i * ) (pbSrc + 0) ) ) );
1696 _mm_storeu_si128( (__m128i *) (pbDst + 16), _mm_xor_si128( c1, _mm_loadu_si128( ( __m128i * ) (pbSrc + 16) ) ) );
1697 _mm_storeu_si128( (__m128i *) (pbDst + 32), _mm_xor_si128( c2, _mm_loadu_si128( ( __m128i * ) (pbSrc + 32) ) ) );
1698 _mm_storeu_si128( (__m128i *) (pbDst + 48), _mm_xor_si128( c3, _mm_loadu_si128( ( __m128i * ) (pbSrc + 48) ) ) );
1699 _mm_storeu_si128( (__m128i *) (pbDst + 64), _mm_xor_si128( c4, _mm_loadu_si128( ( __m128i * ) (pbSrc + 64) ) ) );
1700 _mm_storeu_si128( (__m128i *) (pbDst + 80), _mm_xor_si128( c5, _mm_loadu_si128( ( __m128i * ) (pbSrc + 80) ) ) );
1701 _mm_storeu_si128( (__m128i *) (pbDst + 96), _mm_xor_si128( c6, _mm_loadu_si128( ( __m128i * ) (pbSrc + 96) ) ) );
1702 _mm_storeu_si128( (__m128i *) (pbDst +112), _mm_xor_si128( c7, _mm_loadu_si128( ( __m128i * ) (pbSrc +112) ) ) );
1703
1704 pbDst += 8 * SYMCRYPT_AES_BLOCK_SIZE;
1705 pbSrc += 8 * SYMCRYPT_AES_BLOCK_SIZE;
1706 nBlocks -= 8;
1707
1708 if ( todo == 0 )
1709 {
1710 CLMUL_3_POST( a0, a1, a2 );
1711 MODREDUCE( vMultiplicationConstant, a0, a1, a2, state );
1712
1713 if ( nBlocks > 0 )
1714 {
1715 todo = SYMCRYPT_MIN( nBlocks, SYMCRYPT_GHASH_PCLMULQDQ_HPOWERS );
1716 CLMUL_3( state, GHASH_H_POWER(expandedKeyTable, todo), GHASH_Hx_POWER(expandedKeyTable, todo), a0, a1, a2 );
1717 }
1718 }
1719 }
1720
1721 if( nBlocks > 0 )
1722 {
1723 // We have 1-7 blocks to GHASH and decrypt
1724 // Do the exact number of GHASH blocks we need in parallel with generating either 4 or 8 blocks of AES-CTR
1725 c0 = chain;
1726 c1 = _mm_add_epi32( chain, chainIncrement1 );
1727 c2 = _mm_add_epi32( chain, chainIncrement2 );
1728 c3 = _mm_add_epi32( c1, chainIncrement2 );
1729 c4 = _mm_add_epi32( c2, chainIncrement2 );
1730
1731 c0 = _mm_shuffle_epi8( c0, BYTE_REVERSE_ORDER );
1732 c1 = _mm_shuffle_epi8( c1, BYTE_REVERSE_ORDER );
1733 c2 = _mm_shuffle_epi8( c2, BYTE_REVERSE_ORDER );
1734 c3 = _mm_shuffle_epi8( c3, BYTE_REVERSE_ORDER );
1735
1736 if( nBlocks > 4 )
1737 {
1738 c5 = _mm_add_epi32( c4, chainIncrement1 );
1739 c6 = _mm_add_epi32( c4, chainIncrement2 );
1740
1741 c4 = _mm_shuffle_epi8( c4, BYTE_REVERSE_ORDER );
1742 c5 = _mm_shuffle_epi8( c5, BYTE_REVERSE_ORDER );
1743 c6 = _mm_shuffle_epi8( c6, BYTE_REVERSE_ORDER );
1744
1745 AES_GCM_ENCRYPT_8( pExpandedKey, c0, c1, c2, c3, c4, c5, c6, c7, pbGhashSrc, nBlocks, BYTE_REVERSE_ORDER, expandedKeyTable, todo, a0, a1, a2 );
1746 } else {
1747 AES_GCM_ENCRYPT_4( pExpandedKey, c0, c1, c2, c3, pbGhashSrc, nBlocks, BYTE_REVERSE_ORDER, expandedKeyTable, todo, a0, a1, a2 );
1748 }
1749
1750 CLMUL_3_POST( a0, a1, a2 );
1751 MODREDUCE( vMultiplicationConstant, a0, a1, a2, state );
1752
1753 // Decrypt 1-7 blocks with pre-generated AES-CTR blocks
1754 while( nBlocks >= 2 )
1755 {
1756 chain = _mm_add_epi32( chain, chainIncrement2 );
1757
1758 _mm_storeu_si128( (__m128i *) (pbDst + 0), _mm_xor_si128( c0, _mm_loadu_si128( ( __m128i * ) (pbSrc + 0) ) ) );
1759 _mm_storeu_si128( (__m128i *) (pbDst + 16), _mm_xor_si128( c1, _mm_loadu_si128( ( __m128i * ) (pbSrc + 16) ) ) );
1760
1761 pbDst += 2*SYMCRYPT_AES_BLOCK_SIZE;
1762 pbSrc += 2*SYMCRYPT_AES_BLOCK_SIZE;
1763 nBlocks -= 2;
1764 c0 = c2;
1765 c1 = c3;
1766 c2 = c4;
1767 c3 = c5;
1768 c4 = c6;
1769 }
1770
1771 if( nBlocks > 0 )
1772 {
1773 chain = _mm_add_epi32( chain, chainIncrement1 );
1774
1775 _mm_storeu_si128( (__m128i *) (pbDst + 0), _mm_xor_si128( c0, _mm_loadu_si128( ( __m128i * ) (pbSrc + 0) ) ) );
1776 }
1777 }
1778
1779 chain = _mm_shuffle_epi8( chain, BYTE_REVERSE_ORDER );
1780 _mm_storeu_si128( (__m128i *) pbChainingValue, chain );
1781 _mm_storeu_si128((__m128i *)pState, state );
1782}
1783#pragma runtime_checks( "u", restore )
1784#pragma warning(pop)
1785
1786#ifdef __clang__
1787#pragma clang attribute pop
1788#else
1789#pragma GCC pop_options
1790#endif
1791
1792#endif // CPU_X86 | CPU_AMD64
UINT64
COMPILER_DEPENDENT_UINT64 UINT64
Definition: actypes.h:131
state
static int state
Definition: maze.c:121
_mm_storeu_si128
void _mm_storeu_si128(__m128i_u *p, __m128i b)
Definition: emmintrin.h:1684
_mm_set_epi8
__m128i _mm_set_epi8(char b15, char b14, char b13, char b12, char b11, char b10, char b9, char b8, char b7, char b6, char b5, char b4, char b3, char b2, char b1, char b0)
Definition: emmintrin.h:1610
_mm_set_epi32
__m128i _mm_set_epi32(int i3, int i2, int i1, int i0)
Definition: emmintrin.h:1598
_mm_xor_si128
__m128i _mm_xor_si128(__m128i a, __m128i b)
Definition: emmintrin.h:1345
_mm_castsi128_ps
__m128 _mm_castsi128_ps(__m128i a)
Definition: emmintrin.h:1948
_mm_add_epi32
__m128i _mm_add_epi32(__m128i a, __m128i b)
Definition: emmintrin.h:1137
_mm_loadu_si128
__m128i _mm_loadu_si128(__m128i_u const *p)
Definition: emmintrin.h:1561
_mm_set1_epi32
__m128i _mm_set1_epi32(int i)
Definition: emmintrin.h:1631
ghash_definitions.h
x
GLint GLint GLint GLint GLint x
Definition: gl.h:1548
c
const GLubyte * c
Definition: glext.h:8905
d
#define d
Definition: ke_i.h:81
c
#define c
Definition: ke_i.h:80
memcpy
#define memcpy(s1, s2, n)
Definition: mkisofs.h:878
todo
BOOL todo
Definition: filedlg.c:313
a1
static const struct update_accum a1
Definition: msg.c:534
a2
static const struct update_accum a2
Definition: msg.c:542
r1
static DNS_RECORDW r1
Definition: record.c:37
_In_reads_
#define _In_reads_(s)
Definition: no_sal2.h:168
_Inout_updates_
#define _Inout_updates_(s)
Definition: no_sal2.h:182
_Inout_
#define _Inout_
Definition: no_sal2.h:162
_Out_writes_
#define _Out_writes_(s)
Definition: no_sal2.h:176
_In_
#define _In_
Definition: no_sal2.h:158
PBYTE
BYTE * PBYTE
Definition: pedump.c:66
SymCryptAesCbcDecryptXmm
VOID SYMCRYPT_CALL SymCryptAesCbcDecryptXmm(_In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, _Inout_updates_(SYMCRYPT_AES_BLOCK_SIZE) PBYTE pbChainingValue, _In_reads_(cbData) PCBYTE pbSrc, _Out_writes_(cbData) PBYTE pbDst, SIZE_T cbData)
SymCryptAesCbcMacXmm
VOID SYMCRYPT_CALL SymCryptAesCbcMacXmm(_In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, _Inout_updates_(SYMCRYPT_AES_BLOCK_SIZE) PBYTE pbChainingValue, _In_reads_(cbData) PCBYTE pbData, SIZE_T cbData)
SymCryptXtsAesEncryptDataUnitXmm
VOID SYMCRYPT_CALL SymCryptXtsAesEncryptDataUnitXmm(_In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, _In_reads_(SYMCRYPT_AES_BLOCK_SIZE) PBYTE pbTweakBlock, _Out_writes_(SYMCRYPT_AES_BLOCK_SIZE *16) PBYTE pbScratch, _In_reads_(cbData) PCBYTE pbSrc, _Out_writes_(cbData) PBYTE pbDst, SIZE_T cbData)
SymCryptXtsAesDecryptDataUnitXmm
VOID SYMCRYPT_CALL SymCryptXtsAesDecryptDataUnitXmm(_In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, _In_reads_(SYMCRYPT_AES_BLOCK_SIZE) PBYTE pbTweakBlock, _Out_writes_(SYMCRYPT_AES_BLOCK_SIZE *16) PBYTE pbScratch, _In_reads_(cbData) PCBYTE pbSrc, _Out_writes_(cbData) PBYTE pbDst, SIZE_T cbData)
SymCryptAesCbcEncryptXmm
VOID SYMCRYPT_CALL SymCryptAesCbcEncryptXmm(_In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, _Inout_updates_(SYMCRYPT_AES_BLOCK_SIZE) PBYTE pbChainingValue, _In_reads_(cbData) PCBYTE pbSrc, _Out_writes_(cbData) PBYTE pbDst, SIZE_T cbData)
SymCryptAesGcmEncryptStitchedXmm
VOID SYMCRYPT_CALL SymCryptAesGcmEncryptStitchedXmm(_In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, _In_reads_(SYMCRYPT_AES_BLOCK_SIZE) PBYTE pbChainingValue, _In_reads_(SYMCRYPT_GF128_FIELD_SIZE) PCSYMCRYPT_GF128_ELEMENT expandedKeyTable, _Inout_ PSYMCRYPT_GF128_ELEMENT pState, _In_reads_(cbData) PCBYTE pbSrc, _Out_writes_(cbData) PBYTE pbDst, SIZE_T cbData)
SymCryptAesEcbEncryptXmm
VOID SYMCRYPT_CALL SymCryptAesEcbEncryptXmm(_In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, _In_reads_(cbData) PCBYTE pbSrc, _Out_writes_(cbData) PBYTE pbDst, SIZE_T cbData)
SymCryptAes4SboxXmm
VOID SYMCRYPT_CALL SymCryptAes4SboxXmm(_In_reads_(4) PCBYTE pIn, _Out_writes_(4) PBYTE pOut)
SymCryptAesGcmDecryptStitchedXmm
VOID SYMCRYPT_CALL SymCryptAesGcmDecryptStitchedXmm(_In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, _In_reads_(SYMCRYPT_AES_BLOCK_SIZE) PBYTE pbChainingValue, _In_reads_(SYMCRYPT_GF128_FIELD_SIZE) PCSYMCRYPT_GF128_ELEMENT expandedKeyTable, _Inout_ PSYMCRYPT_GF128_ELEMENT pState, _In_reads_(cbData) PCBYTE pbSrc, _Out_writes_(cbData) PBYTE pbDst, SIZE_T cbData)
SymCryptAesEncryptXmm
VOID SYMCRYPT_CALL SymCryptAesEncryptXmm(_In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, _In_reads_(SYMCRYPT_AES_BLOCK_SIZE) PCBYTE pbSrc, _Out_writes_(SYMCRYPT_AES_BLOCK_SIZE) PBYTE pbDst)
SymCryptAesCreateDecryptionRoundKeyXmm
VOID SYMCRYPT_CALL SymCryptAesCreateDecryptionRoundKeyXmm(_In_reads_(16) PCBYTE pEncryptionRoundKey, _Out_writes_(16) PBYTE pDecryptionRoundKey)
SymCryptAesDecryptXmm
VOID SYMCRYPT_CALL SymCryptAesDecryptXmm(_In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, _In_reads_(SYMCRYPT_AES_BLOCK_SIZE) PCBYTE pbSrc, _Out_writes_(SYMCRYPT_AES_BLOCK_SIZE) PBYTE pbDst)
SYMCRYPT_ASSERT
#define SYMCRYPT_ASSERT(_x)
Definition: symcrypt.h:10807
SYMCRYPT_AES_BLOCK_SIZE
#define SYMCRYPT_AES_BLOCK_SIZE
Definition: symcrypt.h:4255
pbChainingValue
PBYTE pbChainingValue
Definition: symcrypt_internal.h:1990
pbSrc
PCBYTE pbSrc
Definition: symcrypt_internal.h:1988
SYMCRYPT_CALL
#define SYMCRYPT_CALL
Definition: symcrypt_internal.h:167
PCSYMCRYPT_GF128_ELEMENT
const SYMCRYPT_GF128_ELEMENT * PCSYMCRYPT_GF128_ELEMENT
Definition: symcrypt_internal.h:1941
SYMCRYPT_GCM_BLOCK_MOD_MASK
#define SYMCRYPT_GCM_BLOCK_MOD_MASK
Definition: symcrypt_internal.h:1859
PCSYMCRYPT_AES_EXPANDED_KEY
const SYMCRYPT_AES_EXPANDED_KEY * PCSYMCRYPT_AES_EXPANDED_KEY
Definition: symcrypt_internal.h:1656
SYMCRYPT_GF128_FIELD_SIZE
#define SYMCRYPT_GF128_FIELD_SIZE
Definition: symcrypt_internal.h:1851
SYMCRYPT_GF128_BLOCK_SIZE
#define SYMCRYPT_GF128_BLOCK_SIZE
Definition: symcrypt_internal.h:1852
SYMCRYPT_MIN
#define SYMCRYPT_MIN(_a, _b)
Definition: symcrypt_internal.h:312
PSYMCRYPT_GF128_ELEMENT
* PSYMCRYPT_GF128_ELEMENT
Definition: symcrypt_internal.h:1940
cbData
PCBYTE PBYTE SIZE_T cbData
Definition: symcrypt_internal.h:1989
pbDst
PCBYTE PBYTE pbDst
Definition: symcrypt_internal.h:1988
SYMCRYPT_GF128_ELEMENT
SYMCRYPT_GF128_ELEMENT
Definition: symcrypt_internal.h:1940
pState
PSYMCRYPT_COMMON_HASH_STATE pState
Definition: symcrypt_internal.h:1278
PCBYTE
const BYTE * PCBYTE
Definition: symcrypt_internal.h:248
pbData
PCBYTE pbData
Definition: symcrypt_internal.h:1246
pExpandedKey
PCVOID pExpandedKey
Definition: symcrypt_internal.h:1826
chain
struct sock * chain
Definition: tcpcore.h:1
SIZE_T
ULONG_PTR SIZE_T
Definition: typedefs.h:80
_mm_store_ss
void _mm_store_ss(float *p, __m128 a)
Definition: xmmintrin.h:1052
xtsaes_definitions.h
XTS_MUL_ALPHA
#define XTS_MUL_ALPHA(_in, _res)
Definition: xtsaes_definitions.h:41
XTS_MUL_ALPHA_Scalar
#define XTS_MUL_ALPHA_Scalar(_inout_low_u64, _inout_high_u64)
Definition: xtsaes_definitions.h:170
XTS_MUL_ALPHA4
#define XTS_MUL_ALPHA4(_in, _res)
Definition: xtsaes_definitions.h:92
BYTE
unsigned char BYTE
Definition: xxhash.c:193