ReactOS 0.4.17-dev-1005-g171e1de
aes-ymm.c
Go to the documentation of this file.
1//
2// aes-ymm.c code for AES implementation
3//
4// Copyright (c) Microsoft Corporation. Licensed under the MIT license.
5//
6// All YMM code for AES operations
7// Requires compiler support for aesni, pclmulqdq, avx2, vaes and vpclmulqdq
8//
9
10#include "precomp.h"
11
12#if SYMCRYPT_CPU_X86 | SYMCRYPT_CPU_AMD64
13
14#ifdef __clang__
15#pragma clang attribute push (__attribute__((target("avx2,pclmul,vaes,vpclmulqdq"))), apply_to=function)
16#else
17#pragma GCC push_options
18#pragma GCC target("avx2,pclmul,vaes,vpclmulqdq")
19#endif
20
21#include "xtsaes_definitions.h"
22#include "ghash_definitions.h"
23
24#define AES_ENCRYPT_YMM_2048( pExpandedKey, c0, c1, c2, c3, c4, c5, c6, c7 ) \
25{ \
26 const BYTE (*keyPtr)[4][4]; \
27 const BYTE (*keyLimit)[4][4]; \
28 __m256i roundkeys; \
29\
30 keyPtr = pExpandedKey->RoundKey; \
31 keyLimit = pExpandedKey->lastEncRoundKey; \
32\
33 /* _mm256_broadcastsi128_si256 requires AVX2 */ \
34 roundkeys = _mm256_broadcastsi128_si256( *( (const __m128i *) keyPtr ) ); \
35 keyPtr ++; \
36\
37 /* _mm256_xor_si256 requires AVX2 */ \
38 c0 = _mm256_xor_si256( c0, roundkeys ); \
39 c1 = _mm256_xor_si256( c1, roundkeys ); \
40 c2 = _mm256_xor_si256( c2, roundkeys ); \
41 c3 = _mm256_xor_si256( c3, roundkeys ); \
42 c4 = _mm256_xor_si256( c4, roundkeys ); \
43 c5 = _mm256_xor_si256( c5, roundkeys ); \
44 c6 = _mm256_xor_si256( c6, roundkeys ); \
45 c7 = _mm256_xor_si256( c7, roundkeys ); \
46\
47 do \
48 { \
49 roundkeys = _mm256_broadcastsi128_si256( *( (const __m128i *) keyPtr ) ); \
50 keyPtr ++; \
51 c0 = _mm256_aesenc_epi128( c0, roundkeys ); \
52 c1 = _mm256_aesenc_epi128( c1, roundkeys ); \
53 c2 = _mm256_aesenc_epi128( c2, roundkeys ); \
54 c3 = _mm256_aesenc_epi128( c3, roundkeys ); \
55 c4 = _mm256_aesenc_epi128( c4, roundkeys ); \
56 c5 = _mm256_aesenc_epi128( c5, roundkeys ); \
57 c6 = _mm256_aesenc_epi128( c6, roundkeys ); \
58 c7 = _mm256_aesenc_epi128( c7, roundkeys ); \
59 } while( keyPtr < keyLimit ); \
60\
61 roundkeys = _mm256_broadcastsi128_si256( *( (const __m128i *) keyPtr ) ); \
62\
63 c0 = _mm256_aesenclast_epi128( c0, roundkeys ); \
64 c1 = _mm256_aesenclast_epi128( c1, roundkeys ); \
65 c2 = _mm256_aesenclast_epi128( c2, roundkeys ); \
66 c3 = _mm256_aesenclast_epi128( c3, roundkeys ); \
67 c4 = _mm256_aesenclast_epi128( c4, roundkeys ); \
68 c5 = _mm256_aesenclast_epi128( c5, roundkeys ); \
69 c6 = _mm256_aesenclast_epi128( c6, roundkeys ); \
70 c7 = _mm256_aesenclast_epi128( c7, roundkeys ); \
71};
72
73#define AES_DECRYPT_YMM_2048( pExpandedKey, c0, c1, c2, c3, c4, c5, c6, c7 ) \
74{ \
75 const BYTE (*keyPtr)[4][4]; \
76 const BYTE (*keyLimit)[4][4]; \
77 __m256i roundkeys; \
78\
79 keyPtr = pExpandedKey->lastEncRoundKey; \
80 keyLimit = pExpandedKey->lastDecRoundKey; \
81\
82 /* _mm256_broadcastsi128_si256 requires AVX2 */ \
83 roundkeys = _mm256_broadcastsi128_si256( *( (const __m128i *) keyPtr ) ); \
84 keyPtr ++; \
85\
86 /* _mm256_xor_si256 requires AVX2 */ \
87 c0 = _mm256_xor_si256( c0, roundkeys ); \
88 c1 = _mm256_xor_si256( c1, roundkeys ); \
89 c2 = _mm256_xor_si256( c2, roundkeys ); \
90 c3 = _mm256_xor_si256( c3, roundkeys ); \
91 c4 = _mm256_xor_si256( c4, roundkeys ); \
92 c5 = _mm256_xor_si256( c5, roundkeys ); \
93 c6 = _mm256_xor_si256( c6, roundkeys ); \
94 c7 = _mm256_xor_si256( c7, roundkeys ); \
95\
96 do \
97 { \
98 roundkeys = _mm256_broadcastsi128_si256( *( (const __m128i *) keyPtr ) ); \
99 keyPtr ++; \
100 c0 = _mm256_aesdec_epi128( c0, roundkeys ); \
101 c1 = _mm256_aesdec_epi128( c1, roundkeys ); \
102 c2 = _mm256_aesdec_epi128( c2, roundkeys ); \
103 c3 = _mm256_aesdec_epi128( c3, roundkeys ); \
104 c4 = _mm256_aesdec_epi128( c4, roundkeys ); \
105 c5 = _mm256_aesdec_epi128( c5, roundkeys ); \
106 c6 = _mm256_aesdec_epi128( c6, roundkeys ); \
107 c7 = _mm256_aesdec_epi128( c7, roundkeys ); \
108 } while( keyPtr < keyLimit ); \
109\
110 roundkeys = _mm256_broadcastsi128_si256( *( (const __m128i *) keyPtr ) ); \
111\
112 c0 = _mm256_aesdeclast_epi128( c0, roundkeys ); \
113 c1 = _mm256_aesdeclast_epi128( c1, roundkeys ); \
114 c2 = _mm256_aesdeclast_epi128( c2, roundkeys ); \
115 c3 = _mm256_aesdeclast_epi128( c3, roundkeys ); \
116 c4 = _mm256_aesdeclast_epi128( c4, roundkeys ); \
117 c5 = _mm256_aesdeclast_epi128( c5, roundkeys ); \
118 c6 = _mm256_aesdeclast_epi128( c6, roundkeys ); \
119 c7 = _mm256_aesdeclast_epi128( c7, roundkeys ); \
120};
121
122VOID
123SYMCRYPT_CALL
124SymCryptXtsAesEncryptDataUnitYmm_2048(
125 _In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey,
126 _Inout_updates_( SYMCRYPT_AES_BLOCK_SIZE ) PBYTE pbTweakBlock,
127 _Out_writes_( SYMCRYPT_AES_BLOCK_SIZE*16 ) PBYTE pbScratch,
128 _In_reads_( cbData ) PCBYTE pbSrc,
129 _Out_writes_( cbData ) PBYTE pbDst,
130 SIZE_T cbData )
131{
132 __m128i t0, t1, t2, t3, t4, t5, t6, t7;
133 __m256i c0, c1, c2, c3, c4, c5, c6, c7;
134 __m128i XTS_ALPHA_MASK;
135 __m256i XTS_ALPHA_MULTIPLIER_Ymm;
136
137 // Load tweaks into big T
138 __m256i T0, T1, T2, T3, T4, T5, T6, T7;
139
140 SIZE_T cbDataMain; // number of bytes to handle in the main loop
141 SIZE_T cbDataTail; // number of bytes to handle in the tail loop
142
143 // To simplify logic and unusual size processing, we handle all
144 // data not a multiple of 16 blocks in the tail loop
145 cbDataTail = cbData & ((16*SYMCRYPT_AES_BLOCK_SIZE)-1);
146 // Additionally, so that ciphertext stealing logic does not rely on
147 // reading back from the destination buffer, when we have a non-zero
148 // tail, we ensure that we handle at least 1 whole block in the tail
149 cbDataTail += ((cbDataTail > 0) && (cbDataTail < SYMCRYPT_AES_BLOCK_SIZE)) ? (16*SYMCRYPT_AES_BLOCK_SIZE) : 0;
150 cbDataMain = cbData - cbDataTail;
151
152 SYMCRYPT_ASSERT(cbDataMain <= cbData);
153 SYMCRYPT_ASSERT(cbDataTail <= cbData);
154 SYMCRYPT_ASSERT((cbDataMain & ((16*SYMCRYPT_AES_BLOCK_SIZE)-1)) == 0);
155
156 if( cbDataMain == 0 )
157 {
158 SymCryptXtsAesEncryptDataUnitXmm( pExpandedKey, pbTweakBlock, pbScratch, pbSrc, pbDst, cbDataTail );
159 return;
160 }
161
162 t0 = _mm_loadu_si128( (__m128i *) pbTweakBlock );
163 XTS_ALPHA_MASK = _mm_set_epi32( 1, 1, 1, 0x87 );
164 XTS_ALPHA_MULTIPLIER_Ymm = _mm256_set_epi64x( 0, 0x87, 0, 0x87 );
165
166 // Do not stall.
167 XTS_MUL_ALPHA4( t0, t4 );
168 XTS_MUL_ALPHA ( t0, t1 );
169 XTS_MUL_ALPHA ( t4, t5 );
170 XTS_MUL_ALPHA ( t1, t2 );
171 XTS_MUL_ALPHA ( t5, t6 );
172 XTS_MUL_ALPHA ( t2, t3 );
173 XTS_MUL_ALPHA ( t6, t7 );
174
175 T0 = _mm256_insertf128_si256( _mm256_castsi128_si256( t0 ), t1, 1 ); // AVX
176 T1 = _mm256_insertf128_si256( _mm256_castsi128_si256( t2 ), t3, 1 );
177 T2 = _mm256_insertf128_si256( _mm256_castsi128_si256( t4 ), t5, 1 );
178 T3 = _mm256_insertf128_si256( _mm256_castsi128_si256( t6 ), t7, 1 );
179 XTS_MUL_ALPHA8_YMM(T0, T4);
180 XTS_MUL_ALPHA8_YMM(T1, T5);
181 XTS_MUL_ALPHA8_YMM(T2, T6);
182 XTS_MUL_ALPHA8_YMM(T3, T7);
183
184 for(;;)
185 {
186 c0 = _mm256_xor_si256( T0, _mm256_loadu_si256( ( __m256i * ) ( pbSrc + 0 ) ) );
187 c1 = _mm256_xor_si256( T1, _mm256_loadu_si256( ( __m256i * ) ( pbSrc + 2*SYMCRYPT_AES_BLOCK_SIZE ) ) );
188 c2 = _mm256_xor_si256( T2, _mm256_loadu_si256( ( __m256i * ) ( pbSrc + 4*SYMCRYPT_AES_BLOCK_SIZE ) ) );
189 c3 = _mm256_xor_si256( T3, _mm256_loadu_si256( ( __m256i * ) ( pbSrc + 6*SYMCRYPT_AES_BLOCK_SIZE ) ) );
190 c4 = _mm256_xor_si256( T4, _mm256_loadu_si256( ( __m256i * ) ( pbSrc + 8*SYMCRYPT_AES_BLOCK_SIZE ) ) );
191 c5 = _mm256_xor_si256( T5, _mm256_loadu_si256( ( __m256i * ) ( pbSrc + 10*SYMCRYPT_AES_BLOCK_SIZE ) ) );
192 c6 = _mm256_xor_si256( T6, _mm256_loadu_si256( ( __m256i * ) ( pbSrc + 12*SYMCRYPT_AES_BLOCK_SIZE ) ) );
193 c7 = _mm256_xor_si256( T7, _mm256_loadu_si256( ( __m256i * ) ( pbSrc + 14*SYMCRYPT_AES_BLOCK_SIZE ) ) );
194
195 pbSrc += 16 * SYMCRYPT_AES_BLOCK_SIZE;
196
197 AES_ENCRYPT_YMM_2048( pExpandedKey, c0, c1, c2, c3, c4, c5, c6, c7 );
198
199 _mm256_storeu_si256( ( __m256i * ) ( pbDst + 0 ), _mm256_xor_si256( c0, T0 ) );
200 _mm256_storeu_si256( ( __m256i * ) ( pbDst + 2*SYMCRYPT_AES_BLOCK_SIZE ), _mm256_xor_si256( c1, T1 ) );
201 _mm256_storeu_si256( ( __m256i * ) ( pbDst + 4*SYMCRYPT_AES_BLOCK_SIZE ), _mm256_xor_si256( c2, T2 ) );
202 _mm256_storeu_si256( ( __m256i * ) ( pbDst + 6*SYMCRYPT_AES_BLOCK_SIZE ), _mm256_xor_si256( c3, T3 ) );
203 _mm256_storeu_si256( ( __m256i * ) ( pbDst + 8*SYMCRYPT_AES_BLOCK_SIZE ), _mm256_xor_si256( c4, T4 ) );
204 _mm256_storeu_si256( ( __m256i * ) ( pbDst + 10*SYMCRYPT_AES_BLOCK_SIZE ), _mm256_xor_si256( c5, T5 ) );
205 _mm256_storeu_si256( ( __m256i * ) ( pbDst + 12*SYMCRYPT_AES_BLOCK_SIZE ), _mm256_xor_si256( c6, T6 ) );
206 _mm256_storeu_si256( ( __m256i * ) ( pbDst + 14*SYMCRYPT_AES_BLOCK_SIZE ), _mm256_xor_si256( c7, T7 ) );
207
208 pbDst += 16 * SYMCRYPT_AES_BLOCK_SIZE;
209
210 cbDataMain -= 16 * SYMCRYPT_AES_BLOCK_SIZE;
211 if( cbDataMain < 16 * SYMCRYPT_AES_BLOCK_SIZE )
212 {
213 break;
214 }
215
216 XTS_MUL_ALPHA16_YMM(T0, T0);
217 XTS_MUL_ALPHA16_YMM(T1, T1);
218 XTS_MUL_ALPHA16_YMM(T2, T2);
219 XTS_MUL_ALPHA16_YMM(T3, T3);
220 XTS_MUL_ALPHA16_YMM(T4, T4);
221 XTS_MUL_ALPHA16_YMM(T5, T5);
222 XTS_MUL_ALPHA16_YMM(T6, T6);
223 XTS_MUL_ALPHA16_YMM(T7, T7);
224 }
225
226 // We won't do another 16-block set so we don't update the tweak blocks
227
228 if( cbDataTail > 0 )
229 {
230 //
231 // This is a rare case: the data unit length is not a multiple of 256 bytes.
232 // We do this in the Xmm implementation.
233 // Fix up the tweak block first
234 //
235 t7 = _mm256_extracti128_si256 ( T7, 1 /* Highest 128 bits */ ); // AVX2
236 _mm256_zeroupper();
237 XTS_MUL_ALPHA( t7, t0 );
238 _mm_storeu_si128( (__m128i *) pbTweakBlock, t0 );
239
240 SymCryptXtsAesEncryptDataUnitXmm( pExpandedKey, pbTweakBlock, pbScratch, pbSrc, pbDst, cbDataTail );
241 }
242 else {
243 _mm256_zeroupper();
244 }
245}
246
247VOID
248SYMCRYPT_CALL
249SymCryptXtsAesDecryptDataUnitYmm_2048(
250 _In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey,
251 _Inout_updates_( SYMCRYPT_AES_BLOCK_SIZE ) PBYTE pbTweakBlock,
252 _Out_writes_( SYMCRYPT_AES_BLOCK_SIZE*16 ) PBYTE pbScratch,
253 _In_reads_( cbData ) PCBYTE pbSrc,
254 _Out_writes_( cbData ) PBYTE pbDst,
255 SIZE_T cbData )
256{
257 __m128i t0, t1, t2, t3, t4, t5, t6, t7;
258 __m256i c0, c1, c2, c3, c4, c5, c6, c7;
259 __m128i XTS_ALPHA_MASK;
260 __m256i XTS_ALPHA_MULTIPLIER_Ymm;
261
262 // Load tweaks into big T
263 __m256i T0, T1, T2, T3, T4, T5, T6, T7;
264
265 SIZE_T cbDataMain; // number of bytes to handle in the main loop
266 SIZE_T cbDataTail; // number of bytes to handle in the tail loop
267
268 // To simplify logic and unusual size processing, we handle all
269 // data not a multiple of 16 blocks in the tail loop
270 cbDataTail = cbData & ((16*SYMCRYPT_AES_BLOCK_SIZE)-1);
271 // Additionally, so that ciphertext stealing logic does not rely on
272 // reading back from the destination buffer, when we have a non-zero
273 // tail, we ensure that we handle at least 1 whole block in the tail
274 cbDataTail += ((cbDataTail > 0) && (cbDataTail < SYMCRYPT_AES_BLOCK_SIZE)) ? (16*SYMCRYPT_AES_BLOCK_SIZE) : 0;
275 cbDataMain = cbData - cbDataTail;
276
277 SYMCRYPT_ASSERT(cbDataMain <= cbData);
278 SYMCRYPT_ASSERT(cbDataTail <= cbData);
279 SYMCRYPT_ASSERT((cbDataMain & ((16*SYMCRYPT_AES_BLOCK_SIZE)-1)) == 0);
280
281 if( cbDataMain == 0 )
282 {
283 SymCryptXtsAesDecryptDataUnitXmm( pExpandedKey, pbTweakBlock, pbScratch, pbSrc, pbDst, cbDataTail );
284 return;
285 }
286
287 t0 = _mm_loadu_si128( (__m128i *) pbTweakBlock );
288 XTS_ALPHA_MASK = _mm_set_epi32( 1, 1, 1, 0x87 );
289 XTS_ALPHA_MULTIPLIER_Ymm = _mm256_set_epi64x( 0, 0x87, 0, 0x87 );
290
291 // Do not stall.
292 XTS_MUL_ALPHA4( t0, t4 );
293 XTS_MUL_ALPHA ( t0, t1 );
294 XTS_MUL_ALPHA ( t4, t5 );
295 XTS_MUL_ALPHA ( t1, t2 );
296 XTS_MUL_ALPHA ( t5, t6 );
297 XTS_MUL_ALPHA ( t2, t3 );
298 XTS_MUL_ALPHA ( t6, t7 );
299
300 T0 = _mm256_insertf128_si256( _mm256_castsi128_si256( t0 ), t1, 1); // AVX
301 T1 = _mm256_insertf128_si256( _mm256_castsi128_si256( t2 ), t3, 1);
302 T2 = _mm256_insertf128_si256( _mm256_castsi128_si256( t4 ), t5, 1);
303 T3 = _mm256_insertf128_si256( _mm256_castsi128_si256( t6 ), t7, 1);
304 XTS_MUL_ALPHA8_YMM(T0, T4);
305 XTS_MUL_ALPHA8_YMM(T1, T5);
306 XTS_MUL_ALPHA8_YMM(T2, T6);
307 XTS_MUL_ALPHA8_YMM(T3, T7);
308
309 for(;;)
310 {
311 c0 = _mm256_xor_si256( T0, _mm256_loadu_si256( ( __m256i * ) ( pbSrc + 0 ) ) );
312 c1 = _mm256_xor_si256( T1, _mm256_loadu_si256( ( __m256i * ) ( pbSrc + 2*SYMCRYPT_AES_BLOCK_SIZE ) ) );
313 c2 = _mm256_xor_si256( T2, _mm256_loadu_si256( ( __m256i * ) ( pbSrc + 4*SYMCRYPT_AES_BLOCK_SIZE ) ) );
314 c3 = _mm256_xor_si256( T3, _mm256_loadu_si256( ( __m256i * ) ( pbSrc + 6*SYMCRYPT_AES_BLOCK_SIZE ) ) );
315 c4 = _mm256_xor_si256( T4, _mm256_loadu_si256( ( __m256i * ) ( pbSrc + 8*SYMCRYPT_AES_BLOCK_SIZE ) ) );
316 c5 = _mm256_xor_si256( T5, _mm256_loadu_si256( ( __m256i * ) ( pbSrc + 10*SYMCRYPT_AES_BLOCK_SIZE ) ) );
317 c6 = _mm256_xor_si256( T6, _mm256_loadu_si256( ( __m256i * ) ( pbSrc + 12*SYMCRYPT_AES_BLOCK_SIZE ) ) );
318 c7 = _mm256_xor_si256( T7, _mm256_loadu_si256( ( __m256i * ) ( pbSrc + 14*SYMCRYPT_AES_BLOCK_SIZE ) ) );
319
320 pbSrc += 16 * SYMCRYPT_AES_BLOCK_SIZE;
321
322 AES_DECRYPT_YMM_2048( pExpandedKey, c0, c1, c2, c3, c4, c5, c6, c7 );
323
324 _mm256_storeu_si256( ( __m256i * ) ( pbDst + 0 ), _mm256_xor_si256( c0, T0 ) );
325 _mm256_storeu_si256( ( __m256i * ) ( pbDst + 2*SYMCRYPT_AES_BLOCK_SIZE ), _mm256_xor_si256( c1, T1 ) );
326 _mm256_storeu_si256( ( __m256i * ) ( pbDst + 4*SYMCRYPT_AES_BLOCK_SIZE ), _mm256_xor_si256( c2, T2 ) );
327 _mm256_storeu_si256( ( __m256i * ) ( pbDst + 6*SYMCRYPT_AES_BLOCK_SIZE ), _mm256_xor_si256( c3, T3 ) );
328 _mm256_storeu_si256( ( __m256i * ) ( pbDst + 8*SYMCRYPT_AES_BLOCK_SIZE ), _mm256_xor_si256( c4, T4 ) );
329 _mm256_storeu_si256( ( __m256i * ) ( pbDst + 10*SYMCRYPT_AES_BLOCK_SIZE ), _mm256_xor_si256( c5, T5 ) );
330 _mm256_storeu_si256( ( __m256i * ) ( pbDst + 12*SYMCRYPT_AES_BLOCK_SIZE ), _mm256_xor_si256( c6, T6 ) );
331 _mm256_storeu_si256( ( __m256i * ) ( pbDst + 14*SYMCRYPT_AES_BLOCK_SIZE ), _mm256_xor_si256( c7, T7 ) );
332
333 pbDst += 16 * SYMCRYPT_AES_BLOCK_SIZE;
334
335 cbDataMain -= 16 * SYMCRYPT_AES_BLOCK_SIZE;
336 if( cbDataMain < 16 * SYMCRYPT_AES_BLOCK_SIZE )
337 {
338 break;
339 }
340
341 XTS_MUL_ALPHA16_YMM(T0, T0);
342 XTS_MUL_ALPHA16_YMM(T1, T1);
343 XTS_MUL_ALPHA16_YMM(T2, T2);
344 XTS_MUL_ALPHA16_YMM(T3, T3);
345 XTS_MUL_ALPHA16_YMM(T4, T4);
346 XTS_MUL_ALPHA16_YMM(T5, T5);
347 XTS_MUL_ALPHA16_YMM(T6, T6);
348 XTS_MUL_ALPHA16_YMM(T7, T7);
349 }
350
351 // We won't do another 16-block set so we don't update the tweak blocks
352
353 if( cbDataTail > 0 )
354 {
355 //
356 // This is a rare case: the data unit length is not a multiple of 256 bytes.
357 // We do this in the Xmm implementation.
358 // Fix up the tweak block first
359 //
360 t7 = _mm256_extracti128_si256 ( T7, 1 /* Highest 128 bits */ ); // AVX2
361 _mm256_zeroupper();
362 XTS_MUL_ALPHA( t7, t0 );
363 _mm_storeu_si128( (__m128i *) pbTweakBlock, t0 );
364
365 SymCryptXtsAesDecryptDataUnitXmm( pExpandedKey, pbTweakBlock, pbScratch, pbSrc, pbDst, cbDataTail );
366 }
367 else {
368 _mm256_zeroupper();
369 }
370}
371
372#define AES_FULLROUND_16_GHASH_2_Ymm( roundkeys, keyPtr, c0, c1, c2, c3, c4, c5, c6, c7, r0, t0, t1, gHashPointer, byteReverseOrder, gHashExpandedKeyTable, todo, resl, resm, resh ) \
373{ \
374 roundkeys = _mm256_broadcastsi128_si256( *( (const __m128i *) keyPtr ) ); \
375 keyPtr ++; \
376 c0 = _mm256_aesenc_epi128( c0, roundkeys ); \
377 c1 = _mm256_aesenc_epi128( c1, roundkeys ); \
378 c2 = _mm256_aesenc_epi128( c2, roundkeys ); \
379 c3 = _mm256_aesenc_epi128( c3, roundkeys ); \
380 c4 = _mm256_aesenc_epi128( c4, roundkeys ); \
381 c5 = _mm256_aesenc_epi128( c5, roundkeys ); \
382 c6 = _mm256_aesenc_epi128( c6, roundkeys ); \
383 c7 = _mm256_aesenc_epi128( c7, roundkeys ); \
384\
385 r0 = _mm256_loadu_si256( (__m256i *) gHashPointer ); \
386 r0 = _mm256_shuffle_epi8( r0, byteReverseOrder ); \
387 gHashPointer += 32; \
388\
389 t1 = _mm256_loadu_si256( (__m256i *) &GHASH_H_POWER(gHashExpandedKeyTable, todo) ); \
390 t0 = _mm256_clmulepi64_epi128( r0, t1, 0x00 ); \
391 t1 = _mm256_clmulepi64_epi128( r0, t1, 0x11 ); \
392\
393 resl = _mm256_xor_si256( resl, t0 ); \
394 resh = _mm256_xor_si256( resh, t1 ); \
395\
396 t0 = _mm256_srli_si256( r0, 8 ); \
397 r0 = _mm256_xor_si256( r0, t0 ); \
398 t1 = _mm256_loadu_si256( (__m256i *) &GHASH_Hx_POWER(gHashExpandedKeyTable, todo) ); \
399 t1 = _mm256_clmulepi64_epi128( r0, t1, 0x00 ); \
400\
401 resm = _mm256_xor_si256( resm, t1 ); \
402 todo -= 2; \
403};
404
405#define AES_GCM_ENCRYPT_16_Ymm( pExpandedKey, c0, c1, c2, c3, c4, c5, c6, c7, gHashPointer, byteReverseOrder, gHashExpandedKeyTable, todo, resl, resm, resh ) \
406{ \
407 const BYTE (*keyPtr)[4][4]; \
408 const BYTE (*keyLimit)[4][4]; \
409 __m256i roundkeys; \
410 __m256i t0, t1; \
411 __m256i r0; \
412 int aesEncryptGhashLoop; \
413\
414 keyPtr = pExpandedKey->RoundKey; \
415 keyLimit = pExpandedKey->lastEncRoundKey; \
416\
417 /* _mm256_broadcastsi128_si256 requires AVX2 */ \
418 roundkeys = _mm256_broadcastsi128_si256( *( (const __m128i *) keyPtr ) ); \
419 keyPtr ++; \
420\
421 /* _mm256_xor_si256 requires AVX2 */ \
422 c0 = _mm256_xor_si256( c0, roundkeys ); \
423 c1 = _mm256_xor_si256( c1, roundkeys ); \
424 c2 = _mm256_xor_si256( c2, roundkeys ); \
425 c3 = _mm256_xor_si256( c3, roundkeys ); \
426 c4 = _mm256_xor_si256( c4, roundkeys ); \
427 c5 = _mm256_xor_si256( c5, roundkeys ); \
428 c6 = _mm256_xor_si256( c6, roundkeys ); \
429 c7 = _mm256_xor_si256( c7, roundkeys ); \
430\
431 /* Do 8(x2) full rounds (AES-128|AES-192|AES-256) with stitched GHASH */ \
432 for( aesEncryptGhashLoop = 0; aesEncryptGhashLoop < 4; aesEncryptGhashLoop++ ) \
433 { \
434 AES_FULLROUND_16_GHASH_2_Ymm( roundkeys, keyPtr, c0, c1, c2, c3, c4, c5, c6, c7, r0, t0, t1, gHashPointer, byteReverseOrder, gHashExpandedKeyTable, todo, resl, resm, resh ); \
435 AES_FULLROUND_16_GHASH_2_Ymm( roundkeys, keyPtr, c0, c1, c2, c3, c4, c5, c6, c7, r0, t0, t1, gHashPointer, byteReverseOrder, gHashExpandedKeyTable, todo, resl, resm, resh ); \
436 } \
437\
438 do \
439 { \
440 roundkeys = _mm256_broadcastsi128_si256( *( (const __m128i *) keyPtr ) ); \
441 keyPtr ++; \
442 c0 = _mm256_aesenc_epi128( c0, roundkeys ); \
443 c1 = _mm256_aesenc_epi128( c1, roundkeys ); \
444 c2 = _mm256_aesenc_epi128( c2, roundkeys ); \
445 c3 = _mm256_aesenc_epi128( c3, roundkeys ); \
446 c4 = _mm256_aesenc_epi128( c4, roundkeys ); \
447 c5 = _mm256_aesenc_epi128( c5, roundkeys ); \
448 c6 = _mm256_aesenc_epi128( c6, roundkeys ); \
449 c7 = _mm256_aesenc_epi128( c7, roundkeys ); \
450 } while( keyPtr < keyLimit ); \
451\
452 roundkeys = _mm256_broadcastsi128_si256( *( (const __m128i *) keyPtr ) ); \
453\
454 c0 = _mm256_aesenclast_epi128( c0, roundkeys ); \
455 c1 = _mm256_aesenclast_epi128( c1, roundkeys ); \
456 c2 = _mm256_aesenclast_epi128( c2, roundkeys ); \
457 c3 = _mm256_aesenclast_epi128( c3, roundkeys ); \
458 c4 = _mm256_aesenclast_epi128( c4, roundkeys ); \
459 c5 = _mm256_aesenclast_epi128( c5, roundkeys ); \
460 c6 = _mm256_aesenclast_epi128( c6, roundkeys ); \
461 c7 = _mm256_aesenclast_epi128( c7, roundkeys ); \
462};
463
464VOID
465SYMCRYPT_CALL
466SymCryptAesGcmEncryptStitchedYmm_2048(
467 _In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey,
468 _In_reads_( SYMCRYPT_AES_BLOCK_SIZE ) PBYTE pbChainingValue,
469 _In_reads_( SYMCRYPT_GF128_FIELD_SIZE ) PCSYMCRYPT_GF128_ELEMENT expandedKeyTable,
470 _Inout_ PSYMCRYPT_GF128_ELEMENT pState,
471 _In_reads_( cbData ) PCBYTE pbSrc,
472 _Out_writes_( cbData ) PBYTE pbDst,
473 SIZE_T cbData )
474{
475 __m128i chain = _mm_loadu_si128( (__m128i *) pbChainingValue );
476
477 __m128i BYTE_REVERSE_ORDER_xmm = _mm_set_epi8(
478 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 );
479 __m256i BYTE_REVERSE_ORDER = _mm256_set_epi64x( 0x0001020304050607, 0x08090a0b0c0d0e0f, 0x0001020304050607, 0x08090a0b0c0d0e0f );
480 __m128i vMultiplicationConstant = _mm_set_epi32( 0, 0, 0xc2000000, 0 );
481
482 __m256i chainIncrementUpper1 = _mm256_set_epi64x( 0, 1, 0, 0 );
483 __m256i chainIncrement2 = _mm256_set_epi64x( 0, 2, 0, 2 );
484 __m256i chainIncrement4 = _mm256_set_epi64x( 0, 4, 0, 4 );
485 __m256i chainIncrement16 = _mm256_set_epi64x( 0, 16, 0, 16 );
486
487 __m256i ctr0, ctr1, ctr2, ctr3, ctr4, ctr5, ctr6, ctr7;
488 __m256i c0, c1, c2, c3, c4, c5, c6, c7;
489 __m256i r0, r1, r2, r3, r4, r5, r6, r7;
490 __m256i Hi, Hix;
491
492 __m128i state;
493 __m128i a0_xmm, a1_xmm, a2_xmm;
494 __m256i a0, a1, a2;
495 SIZE_T nBlocks = cbData / SYMCRYPT_GF128_BLOCK_SIZE;
496 SIZE_T todo;
497 PCBYTE pbGhashSrc = pbDst;
498
499 SYMCRYPT_ASSERT( (cbData & SYMCRYPT_GCM_BLOCK_MOD_MASK) == 0 ); // cbData is multiple of block size
500 SYMCRYPT_ASSERT( nBlocks >= GCM_YMM_MINBLOCKS );
501
502 todo = SYMCRYPT_MIN( nBlocks, SYMCRYPT_GHASH_PCLMULQDQ_HPOWERS ) & ~(GCM_YMM_MINBLOCKS-1);
503 chain = _mm_shuffle_epi8( chain, BYTE_REVERSE_ORDER_xmm );
504
505 state = _mm_loadu_si128( (__m128i *) pState );
506 ctr0 = _mm256_insertf128_si256( _mm256_castsi128_si256( chain ), chain, 1); // AVX
507 ctr0 = _mm256_add_epi32( ctr0, chainIncrementUpper1 );
508 ctr1 = _mm256_add_epi32( ctr0, chainIncrement2 );
509 ctr2 = _mm256_add_epi32( ctr0, chainIncrement4 );
510 ctr3 = _mm256_add_epi32( ctr1, chainIncrement4 );
511 ctr4 = _mm256_add_epi32( ctr2, chainIncrement4 );
512 ctr5 = _mm256_add_epi32( ctr3, chainIncrement4 );
513 ctr6 = _mm256_add_epi32( ctr4, chainIncrement4 );
514 ctr7 = _mm256_add_epi32( ctr5, chainIncrement4 );
515
516 CLMUL_3( state, GHASH_H_POWER(expandedKeyTable, todo), GHASH_Hx_POWER(expandedKeyTable, todo), a0_xmm, a1_xmm, a2_xmm );
517 a0 = a1 = a2 = _mm256_setzero_si256();
518
519 c0 = _mm256_shuffle_epi8( ctr0, BYTE_REVERSE_ORDER );
520 c1 = _mm256_shuffle_epi8( ctr1, BYTE_REVERSE_ORDER );
521 c2 = _mm256_shuffle_epi8( ctr2, BYTE_REVERSE_ORDER );
522 c3 = _mm256_shuffle_epi8( ctr3, BYTE_REVERSE_ORDER );
523 c4 = _mm256_shuffle_epi8( ctr4, BYTE_REVERSE_ORDER );
524 c5 = _mm256_shuffle_epi8( ctr5, BYTE_REVERSE_ORDER );
525 c6 = _mm256_shuffle_epi8( ctr6, BYTE_REVERSE_ORDER );
526 c7 = _mm256_shuffle_epi8( ctr7, BYTE_REVERSE_ORDER );
527
528 ctr0 = _mm256_add_epi32( ctr0, chainIncrement16 );
529 ctr1 = _mm256_add_epi32( ctr1, chainIncrement16 );
530 ctr2 = _mm256_add_epi32( ctr2, chainIncrement16 );
531 ctr3 = _mm256_add_epi32( ctr3, chainIncrement16 );
532 ctr4 = _mm256_add_epi32( ctr4, chainIncrement16 );
533 ctr5 = _mm256_add_epi32( ctr5, chainIncrement16 );
534 ctr6 = _mm256_add_epi32( ctr6, chainIncrement16 );
535 ctr7 = _mm256_add_epi32( ctr7, chainIncrement16 );
536
537 AES_ENCRYPT_YMM_2048( pExpandedKey, c0, c1, c2, c3, c4, c5, c6, c7 );
538
539 _mm256_storeu_si256( (__m256i *) (pbDst + 0), _mm256_xor_si256( c0, _mm256_loadu_si256( ( __m256i * ) (pbSrc + 0) ) ) );
540 _mm256_storeu_si256( (__m256i *) (pbDst + 32), _mm256_xor_si256( c1, _mm256_loadu_si256( ( __m256i * ) (pbSrc + 32) ) ) );
541 _mm256_storeu_si256( (__m256i *) (pbDst + 64), _mm256_xor_si256( c2, _mm256_loadu_si256( ( __m256i * ) (pbSrc + 64) ) ) );
542 _mm256_storeu_si256( (__m256i *) (pbDst + 96), _mm256_xor_si256( c3, _mm256_loadu_si256( ( __m256i * ) (pbSrc + 96) ) ) );
543 _mm256_storeu_si256( (__m256i *) (pbDst +128), _mm256_xor_si256( c4, _mm256_loadu_si256( ( __m256i * ) (pbSrc +128) ) ) );
544 _mm256_storeu_si256( (__m256i *) (pbDst +160), _mm256_xor_si256( c5, _mm256_loadu_si256( ( __m256i * ) (pbSrc +160) ) ) );
545 _mm256_storeu_si256( (__m256i *) (pbDst +192), _mm256_xor_si256( c6, _mm256_loadu_si256( ( __m256i * ) (pbSrc +192) ) ) );
546 _mm256_storeu_si256( (__m256i *) (pbDst +224), _mm256_xor_si256( c7, _mm256_loadu_si256( ( __m256i * ) (pbSrc +224) ) ) );
547
548 pbDst += 16 * SYMCRYPT_AES_BLOCK_SIZE;
549 pbSrc += 16 * SYMCRYPT_AES_BLOCK_SIZE;
550
551 while( nBlocks >= 2*GCM_YMM_MINBLOCKS )
552 {
553 c0 = _mm256_shuffle_epi8( ctr0, BYTE_REVERSE_ORDER );
554 c1 = _mm256_shuffle_epi8( ctr1, BYTE_REVERSE_ORDER );
555 c2 = _mm256_shuffle_epi8( ctr2, BYTE_REVERSE_ORDER );
556 c3 = _mm256_shuffle_epi8( ctr3, BYTE_REVERSE_ORDER );
557 c4 = _mm256_shuffle_epi8( ctr4, BYTE_REVERSE_ORDER );
558 c5 = _mm256_shuffle_epi8( ctr5, BYTE_REVERSE_ORDER );
559 c6 = _mm256_shuffle_epi8( ctr6, BYTE_REVERSE_ORDER );
560 c7 = _mm256_shuffle_epi8( ctr7, BYTE_REVERSE_ORDER );
561
562 ctr0 = _mm256_add_epi32( ctr0, chainIncrement16 );
563 ctr1 = _mm256_add_epi32( ctr1, chainIncrement16 );
564 ctr2 = _mm256_add_epi32( ctr2, chainIncrement16 );
565 ctr3 = _mm256_add_epi32( ctr3, chainIncrement16 );
566 ctr4 = _mm256_add_epi32( ctr4, chainIncrement16 );
567 ctr5 = _mm256_add_epi32( ctr5, chainIncrement16 );
568 ctr6 = _mm256_add_epi32( ctr6, chainIncrement16 );
569 ctr7 = _mm256_add_epi32( ctr7, chainIncrement16 );
570
571 AES_GCM_ENCRYPT_16_Ymm( pExpandedKey, c0, c1, c2, c3, c4, c5, c6, c7, pbGhashSrc, BYTE_REVERSE_ORDER, expandedKeyTable, todo, a0, a1, a2 );
572
573 _mm256_storeu_si256( (__m256i *) (pbDst + 0), _mm256_xor_si256( c0, _mm256_loadu_si256( ( __m256i * ) (pbSrc + 0) ) ) );
574 _mm256_storeu_si256( (__m256i *) (pbDst + 32), _mm256_xor_si256( c1, _mm256_loadu_si256( ( __m256i * ) (pbSrc + 32) ) ) );
575 _mm256_storeu_si256( (__m256i *) (pbDst + 64), _mm256_xor_si256( c2, _mm256_loadu_si256( ( __m256i * ) (pbSrc + 64) ) ) );
576 _mm256_storeu_si256( (__m256i *) (pbDst + 96), _mm256_xor_si256( c3, _mm256_loadu_si256( ( __m256i * ) (pbSrc + 96) ) ) );
577 _mm256_storeu_si256( (__m256i *) (pbDst +128), _mm256_xor_si256( c4, _mm256_loadu_si256( ( __m256i * ) (pbSrc +128) ) ) );
578 _mm256_storeu_si256( (__m256i *) (pbDst +160), _mm256_xor_si256( c5, _mm256_loadu_si256( ( __m256i * ) (pbSrc +160) ) ) );
579 _mm256_storeu_si256( (__m256i *) (pbDst +192), _mm256_xor_si256( c6, _mm256_loadu_si256( ( __m256i * ) (pbSrc +192) ) ) );
580 _mm256_storeu_si256( (__m256i *) (pbDst +224), _mm256_xor_si256( c7, _mm256_loadu_si256( ( __m256i * ) (pbSrc +224) ) ) );
581
582 pbDst += 16 * SYMCRYPT_AES_BLOCK_SIZE;
583 pbSrc += 16 * SYMCRYPT_AES_BLOCK_SIZE;
584 nBlocks -= 16;
585
586 if ( todo == 0 )
587 {
588 a0_xmm = _mm_xor_si128( a0_xmm, _mm256_extracti128_si256 ( a0, 0 /* Lowest 128 bits */ ));
589 a1_xmm = _mm_xor_si128( a1_xmm, _mm256_extracti128_si256 ( a1, 0 /* Lowest 128 bits */ ));
590 a2_xmm = _mm_xor_si128( a2_xmm, _mm256_extracti128_si256 ( a2, 0 /* Lowest 128 bits */ ));
591
592 a0_xmm = _mm_xor_si128( a0_xmm, _mm256_extracti128_si256 ( a0, 1 /* Highest 128 bits */ ));
593 a1_xmm = _mm_xor_si128( a1_xmm, _mm256_extracti128_si256 ( a1, 1 /* Highest 128 bits */ ));
594 a2_xmm = _mm_xor_si128( a2_xmm, _mm256_extracti128_si256 ( a2, 1 /* Highest 128 bits */ ));
595 CLMUL_3_POST( a0_xmm, a1_xmm, a2_xmm );
596 MODREDUCE( vMultiplicationConstant, a0_xmm, a1_xmm, a2_xmm, state );
597
598 todo = SYMCRYPT_MIN( nBlocks, SYMCRYPT_GHASH_PCLMULQDQ_HPOWERS ) & ~(GCM_YMM_MINBLOCKS-1);
599 CLMUL_3( state, GHASH_H_POWER(expandedKeyTable, todo), GHASH_Hx_POWER(expandedKeyTable, todo), a0_xmm, a1_xmm, a2_xmm );
600 a0 = a1 = a2 = _mm256_setzero_si256();
601 }
602 }
603
604 r0 = _mm256_shuffle_epi8( _mm256_loadu_si256( (__m256i *) (pbGhashSrc + 0) ), BYTE_REVERSE_ORDER );
605 r1 = _mm256_shuffle_epi8( _mm256_loadu_si256( (__m256i *) (pbGhashSrc + 32) ), BYTE_REVERSE_ORDER );
606 r2 = _mm256_shuffle_epi8( _mm256_loadu_si256( (__m256i *) (pbGhashSrc + 64) ), BYTE_REVERSE_ORDER );
607 r3 = _mm256_shuffle_epi8( _mm256_loadu_si256( (__m256i *) (pbGhashSrc + 96) ), BYTE_REVERSE_ORDER );
608 r4 = _mm256_shuffle_epi8( _mm256_loadu_si256( (__m256i *) (pbGhashSrc +128) ), BYTE_REVERSE_ORDER );
609 r5 = _mm256_shuffle_epi8( _mm256_loadu_si256( (__m256i *) (pbGhashSrc +160) ), BYTE_REVERSE_ORDER );
610 r6 = _mm256_shuffle_epi8( _mm256_loadu_si256( (__m256i *) (pbGhashSrc +192) ), BYTE_REVERSE_ORDER );
611 r7 = _mm256_shuffle_epi8( _mm256_loadu_si256( (__m256i *) (pbGhashSrc +224) ), BYTE_REVERSE_ORDER );
612
613 Hi = _mm256_loadu_si256( (__m256i *) &GHASH_H_POWER(expandedKeyTable, todo - 0) );
614 Hix = _mm256_loadu_si256( (__m256i *) &GHASH_Hx_POWER(expandedKeyTable, todo - 0) );
615 CLMUL_ACC_3_Ymm( r0, Hi, Hix, a0, a1, a2 );
616 Hi = _mm256_loadu_si256( (__m256i *) &GHASH_H_POWER(expandedKeyTable, todo - 2) );
617 Hix = _mm256_loadu_si256( (__m256i *) &GHASH_Hx_POWER(expandedKeyTable, todo - 2) );
618 CLMUL_ACC_3_Ymm( r1, Hi, Hix, a0, a1, a2 );
619 Hi = _mm256_loadu_si256( (__m256i *) &GHASH_H_POWER(expandedKeyTable, todo - 4) );
620 Hix = _mm256_loadu_si256( (__m256i *) &GHASH_Hx_POWER(expandedKeyTable, todo - 4) );
621 CLMUL_ACC_3_Ymm( r2, Hi, Hix, a0, a1, a2 );
622 Hi = _mm256_loadu_si256( (__m256i *) &GHASH_H_POWER(expandedKeyTable, todo - 6) );
623 Hix = _mm256_loadu_si256( (__m256i *) &GHASH_Hx_POWER(expandedKeyTable, todo - 6) );
624 CLMUL_ACC_3_Ymm( r3, Hi, Hix, a0, a1, a2 );
625 Hi = _mm256_loadu_si256( (__m256i *) &GHASH_H_POWER(expandedKeyTable, todo - 8) );
626 Hix = _mm256_loadu_si256( (__m256i *) &GHASH_Hx_POWER(expandedKeyTable, todo - 8) );
627 CLMUL_ACC_3_Ymm( r4, Hi, Hix, a0, a1, a2 );
628 Hi = _mm256_loadu_si256( (__m256i *) &GHASH_H_POWER(expandedKeyTable, todo -10) );
629 Hix = _mm256_loadu_si256( (__m256i *) &GHASH_Hx_POWER(expandedKeyTable, todo -10) );
630 CLMUL_ACC_3_Ymm( r5, Hi, Hix, a0, a1, a2 );
631 Hi = _mm256_loadu_si256( (__m256i *) &GHASH_H_POWER(expandedKeyTable, todo -12) );
632 Hix = _mm256_loadu_si256( (__m256i *) &GHASH_Hx_POWER(expandedKeyTable, todo -12) );
633 CLMUL_ACC_3_Ymm( r6, Hi, Hix, a0, a1, a2 );
634 Hi = _mm256_loadu_si256( (__m256i *) &GHASH_H_POWER(expandedKeyTable, todo -14) );
635 Hix = _mm256_loadu_si256( (__m256i *) &GHASH_Hx_POWER(expandedKeyTable, todo -14) );
636 CLMUL_ACC_3_Ymm( r7, Hi, Hix, a0, a1, a2 );
637
638 a0_xmm = _mm_xor_si128( a0_xmm, _mm256_extracti128_si256 ( a0, 0 /* Lowest 128 bits */ ));
639 a1_xmm = _mm_xor_si128( a1_xmm, _mm256_extracti128_si256 ( a1, 0 /* Lowest 128 bits */ ));
640 a2_xmm = _mm_xor_si128( a2_xmm, _mm256_extracti128_si256 ( a2, 0 /* Lowest 128 bits */ ));
641
642 a0_xmm = _mm_xor_si128( a0_xmm, _mm256_extracti128_si256 ( a0, 1 /* Highest 128 bits */ ));
643 a1_xmm = _mm_xor_si128( a1_xmm, _mm256_extracti128_si256 ( a1, 1 /* Highest 128 bits */ ));
644 a2_xmm = _mm_xor_si128( a2_xmm, _mm256_extracti128_si256 ( a2, 1 /* Highest 128 bits */ ));
645 CLMUL_3_POST( a0_xmm, a1_xmm, a2_xmm );
646 MODREDUCE( vMultiplicationConstant, a0_xmm, a1_xmm, a2_xmm, state );
647
648 chain = _mm256_extracti128_si256 ( ctr0, 0 /* Lowest 128 bits */ );
649 _mm256_zeroupper();
650
651 chain = _mm_shuffle_epi8( chain, BYTE_REVERSE_ORDER_xmm );
652 _mm_storeu_si128((__m128i *) pbChainingValue, chain );
653 _mm_storeu_si128((__m128i *) pState, state );
654
655 cbData &= ( GCM_YMM_MINBLOCKS*SYMCRYPT_AES_BLOCK_SIZE ) - 1;
656 SYMCRYPT_ASSERT( cbData == (nBlocks-16)*SYMCRYPT_AES_BLOCK_SIZE );
657 if ( cbData >= SYMCRYPT_AES_BLOCK_SIZE )
658 {
659 SymCryptAesGcmEncryptStitchedXmm( pExpandedKey, pbChainingValue, expandedKeyTable, pState, pbSrc, pbDst, cbData);
660 }
661}
662
663VOID
664SYMCRYPT_CALL
665SymCryptAesGcmDecryptStitchedYmm_2048(
666 _In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey,
667 _In_reads_( SYMCRYPT_AES_BLOCK_SIZE ) PBYTE pbChainingValue,
668 _In_reads_( SYMCRYPT_GF128_FIELD_SIZE ) PCSYMCRYPT_GF128_ELEMENT expandedKeyTable,
669 _Inout_ PSYMCRYPT_GF128_ELEMENT pState,
670 _In_reads_( cbData ) PCBYTE pbSrc,
671 _Out_writes_( cbData ) PBYTE pbDst,
672 SIZE_T cbData )
673{
674 __m128i chain = _mm_loadu_si128( (__m128i *) pbChainingValue );
675
676 __m128i BYTE_REVERSE_ORDER_xmm = _mm_set_epi8(
677 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 );
678 __m256i BYTE_REVERSE_ORDER = _mm256_set_epi64x( 0x0001020304050607, 0x08090a0b0c0d0e0f, 0x0001020304050607, 0x08090a0b0c0d0e0f );
679 __m128i vMultiplicationConstant = _mm_set_epi32( 0, 0, 0xc2000000, 0 );
680
681 __m256i chainIncrementUpper1 = _mm256_set_epi64x( 0, 1, 0, 0 );
682 __m256i chainIncrement2 = _mm256_set_epi64x( 0, 2, 0, 2 );
683 __m256i chainIncrement4 = _mm256_set_epi64x( 0, 4, 0, 4 );
684 __m256i chainIncrement16 = _mm256_set_epi64x( 0, 16, 0, 16 );
685
686 __m256i ctr0, ctr1, ctr2, ctr3, ctr4, ctr5, ctr6, ctr7;
687 __m256i c0, c1, c2, c3, c4, c5, c6, c7;
688
689 __m128i state;
690 __m128i a0_xmm, a1_xmm, a2_xmm;
691 __m256i a0, a1, a2;
692 SIZE_T nBlocks = cbData / SYMCRYPT_GF128_BLOCK_SIZE;
693 SIZE_T todo;
694 PCBYTE pbGhashSrc = pbSrc;
695
696 SYMCRYPT_ASSERT( (cbData & SYMCRYPT_GCM_BLOCK_MOD_MASK) == 0 ); // cbData is multiple of block size
697 SYMCRYPT_ASSERT( nBlocks >= GCM_YMM_MINBLOCKS );
698
699 todo = SYMCRYPT_MIN( nBlocks, SYMCRYPT_GHASH_PCLMULQDQ_HPOWERS ) & ~(GCM_YMM_MINBLOCKS-1);
700 chain = _mm_shuffle_epi8( chain, BYTE_REVERSE_ORDER_xmm );
701
702 state = _mm_loadu_si128( (__m128i *) pState );
703 ctr0 = _mm256_insertf128_si256( _mm256_castsi128_si256( chain ), chain, 1); // AVX
704 ctr0 = _mm256_add_epi32( ctr0, chainIncrementUpper1 );
705 ctr1 = _mm256_add_epi32( ctr0, chainIncrement2 );
706 ctr2 = _mm256_add_epi32( ctr0, chainIncrement4 );
707 ctr3 = _mm256_add_epi32( ctr1, chainIncrement4 );
708 ctr4 = _mm256_add_epi32( ctr2, chainIncrement4 );
709 ctr5 = _mm256_add_epi32( ctr3, chainIncrement4 );
710 ctr6 = _mm256_add_epi32( ctr4, chainIncrement4 );
711 ctr7 = _mm256_add_epi32( ctr5, chainIncrement4 );
712
713 CLMUL_3( state, GHASH_H_POWER(expandedKeyTable, todo), GHASH_Hx_POWER(expandedKeyTable, todo), a0_xmm, a1_xmm, a2_xmm );
714 a0 = a1 = a2 = _mm256_setzero_si256();
715
716 while( nBlocks >= GCM_YMM_MINBLOCKS )
717 {
718 c0 = _mm256_shuffle_epi8( ctr0, BYTE_REVERSE_ORDER );
719 c1 = _mm256_shuffle_epi8( ctr1, BYTE_REVERSE_ORDER );
720 c2 = _mm256_shuffle_epi8( ctr2, BYTE_REVERSE_ORDER );
721 c3 = _mm256_shuffle_epi8( ctr3, BYTE_REVERSE_ORDER );
722 c4 = _mm256_shuffle_epi8( ctr4, BYTE_REVERSE_ORDER );
723 c5 = _mm256_shuffle_epi8( ctr5, BYTE_REVERSE_ORDER );
724 c6 = _mm256_shuffle_epi8( ctr6, BYTE_REVERSE_ORDER );
725 c7 = _mm256_shuffle_epi8( ctr7, BYTE_REVERSE_ORDER );
726
727 ctr0 = _mm256_add_epi32( ctr0, chainIncrement16 );
728 ctr1 = _mm256_add_epi32( ctr1, chainIncrement16 );
729 ctr2 = _mm256_add_epi32( ctr2, chainIncrement16 );
730 ctr3 = _mm256_add_epi32( ctr3, chainIncrement16 );
731 ctr4 = _mm256_add_epi32( ctr4, chainIncrement16 );
732 ctr5 = _mm256_add_epi32( ctr5, chainIncrement16 );
733 ctr6 = _mm256_add_epi32( ctr6, chainIncrement16 );
734 ctr7 = _mm256_add_epi32( ctr7, chainIncrement16 );
735
736 AES_GCM_ENCRYPT_16_Ymm( pExpandedKey, c0, c1, c2, c3, c4, c5, c6, c7, pbGhashSrc, BYTE_REVERSE_ORDER, expandedKeyTable, todo, a0, a1, a2 );
737
738 _mm256_storeu_si256( (__m256i *) (pbDst + 0), _mm256_xor_si256( c0, _mm256_loadu_si256( ( __m256i * ) (pbSrc + 0) ) ) );
739 _mm256_storeu_si256( (__m256i *) (pbDst + 32), _mm256_xor_si256( c1, _mm256_loadu_si256( ( __m256i * ) (pbSrc + 32) ) ) );
740 _mm256_storeu_si256( (__m256i *) (pbDst + 64), _mm256_xor_si256( c2, _mm256_loadu_si256( ( __m256i * ) (pbSrc + 64) ) ) );
741 _mm256_storeu_si256( (__m256i *) (pbDst + 96), _mm256_xor_si256( c3, _mm256_loadu_si256( ( __m256i * ) (pbSrc + 96) ) ) );
742 _mm256_storeu_si256( (__m256i *) (pbDst +128), _mm256_xor_si256( c4, _mm256_loadu_si256( ( __m256i * ) (pbSrc +128) ) ) );
743 _mm256_storeu_si256( (__m256i *) (pbDst +160), _mm256_xor_si256( c5, _mm256_loadu_si256( ( __m256i * ) (pbSrc +160) ) ) );
744 _mm256_storeu_si256( (__m256i *) (pbDst +192), _mm256_xor_si256( c6, _mm256_loadu_si256( ( __m256i * ) (pbSrc +192) ) ) );
745 _mm256_storeu_si256( (__m256i *) (pbDst +224), _mm256_xor_si256( c7, _mm256_loadu_si256( ( __m256i * ) (pbSrc +224) ) ) );
746
747 pbDst += 16 * SYMCRYPT_AES_BLOCK_SIZE;
748 pbSrc += 16 * SYMCRYPT_AES_BLOCK_SIZE;
749 nBlocks -= 16;
750
751 if ( todo == 0 )
752 {
753 a0_xmm = _mm_xor_si128( a0_xmm, _mm256_extracti128_si256 ( a0, 0 /* Lowest 128 bits */ ));
754 a1_xmm = _mm_xor_si128( a1_xmm, _mm256_extracti128_si256 ( a1, 0 /* Lowest 128 bits */ ));
755 a2_xmm = _mm_xor_si128( a2_xmm, _mm256_extracti128_si256 ( a2, 0 /* Lowest 128 bits */ ));
756
757 a0_xmm = _mm_xor_si128( a0_xmm, _mm256_extracti128_si256 ( a0, 1 /* Highest 128 bits */ ));
758 a1_xmm = _mm_xor_si128( a1_xmm, _mm256_extracti128_si256 ( a1, 1 /* Highest 128 bits */ ));
759 a2_xmm = _mm_xor_si128( a2_xmm, _mm256_extracti128_si256 ( a2, 1 /* Highest 128 bits */ ));
760 CLMUL_3_POST( a0_xmm, a1_xmm, a2_xmm );
761 MODREDUCE( vMultiplicationConstant, a0_xmm, a1_xmm, a2_xmm, state );
762
763 if ( nBlocks > 0 )
764 {
765 todo = SYMCRYPT_MIN( nBlocks, SYMCRYPT_GHASH_PCLMULQDQ_HPOWERS ) & ~(GCM_YMM_MINBLOCKS-1);
766 CLMUL_3( state, GHASH_H_POWER(expandedKeyTable, todo), GHASH_Hx_POWER(expandedKeyTable, todo), a0_xmm, a1_xmm, a2_xmm );
767 a0 = a1 = a2 = _mm256_setzero_si256();
768 }
769 }
770 }
771
772 chain = _mm256_extracti128_si256 ( ctr0, 0 /* Lowest 128 bits */ );
773 _mm256_zeroupper();
774
775 chain = _mm_shuffle_epi8( chain, BYTE_REVERSE_ORDER_xmm );
776 _mm_storeu_si128((__m128i *) pbChainingValue, chain );
777 _mm_storeu_si128((__m128i *) pState, state );
778
779 cbData &= ( GCM_YMM_MINBLOCKS*SYMCRYPT_AES_BLOCK_SIZE ) - 1;
780 SYMCRYPT_ASSERT( cbData == nBlocks*SYMCRYPT_AES_BLOCK_SIZE );
781 if ( cbData >= SYMCRYPT_AES_BLOCK_SIZE )
782 {
783 SymCryptAesGcmDecryptStitchedXmm( pExpandedKey, pbChainingValue, expandedKeyTable, pState, pbSrc, pbDst, cbData);
784 }
785}
786
787#ifdef __clang__
788#pragma clang attribute pop
789#else
790#pragma GCC pop_options
791#endif
792
793#endif // CPU_X86 | CPU_AMD64
state
static int state
Definition: maze.c:121
_mm_storeu_si128
void _mm_storeu_si128(__m128i_u *p, __m128i b)
Definition: emmintrin.h:1684
_mm_set_epi8
__m128i _mm_set_epi8(char b15, char b14, char b13, char b12, char b11, char b10, char b9, char b8, char b7, char b6, char b5, char b4, char b3, char b2, char b1, char b0)
Definition: emmintrin.h:1610
_mm_set_epi32
__m128i _mm_set_epi32(int i3, int i2, int i1, int i0)
Definition: emmintrin.h:1598
_mm_xor_si128
__m128i _mm_xor_si128(__m128i a, __m128i b)
Definition: emmintrin.h:1345
_mm_loadu_si128
__m128i _mm_loadu_si128(__m128i_u const *p)
Definition: emmintrin.h:1561
ghash_definitions.h
_mm256_setzero_si256
__m256i __cdecl _mm256_setzero_si256(void)
Definition: immintrin.h:155
_mm256_zeroupper
void __cdecl _mm256_zeroupper(void)
Definition: immintrin.h:160
todo
BOOL todo
Definition: filedlg.c:313
a1
static const struct update_accum a1
Definition: msg.c:534
a2
static const struct update_accum a2
Definition: msg.c:542
r3
static DNS_RECORDW r3
Definition: record.c:39
r1
static DNS_RECORDW r1
Definition: record.c:37
r2
static DNS_RECORDW r2
Definition: record.c:38
_In_reads_
#define _In_reads_(s)
Definition: no_sal2.h:168
_Inout_updates_
#define _Inout_updates_(s)
Definition: no_sal2.h:182
_Inout_
#define _Inout_
Definition: no_sal2.h:162
_Out_writes_
#define _Out_writes_(s)
Definition: no_sal2.h:176
_In_
#define _In_
Definition: no_sal2.h:158
PBYTE
BYTE * PBYTE
Definition: pedump.c:66
GCM_YMM_MINBLOCKS
#define GCM_YMM_MINBLOCKS
Definition: sc_lib.h:1611
SymCryptXtsAesEncryptDataUnitXmm
VOID SYMCRYPT_CALL SymCryptXtsAesEncryptDataUnitXmm(_In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, _In_reads_(SYMCRYPT_AES_BLOCK_SIZE) PBYTE pbTweakBlock, _Out_writes_(SYMCRYPT_AES_BLOCK_SIZE *16) PBYTE pbScratch, _In_reads_(cbData) PCBYTE pbSrc, _Out_writes_(cbData) PBYTE pbDst, SIZE_T cbData)
SymCryptXtsAesDecryptDataUnitXmm
VOID SYMCRYPT_CALL SymCryptXtsAesDecryptDataUnitXmm(_In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, _In_reads_(SYMCRYPT_AES_BLOCK_SIZE) PBYTE pbTweakBlock, _Out_writes_(SYMCRYPT_AES_BLOCK_SIZE *16) PBYTE pbScratch, _In_reads_(cbData) PCBYTE pbSrc, _Out_writes_(cbData) PBYTE pbDst, SIZE_T cbData)
SymCryptAesGcmDecryptStitchedYmm_2048
VOID SYMCRYPT_CALL SymCryptAesGcmDecryptStitchedYmm_2048(_In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, _In_reads_(SYMCRYPT_AES_BLOCK_SIZE) PBYTE pbChainingValue, _In_reads_(SYMCRYPT_GF128_FIELD_SIZE) PCSYMCRYPT_GF128_ELEMENT expandedKeyTable, _Inout_ PSYMCRYPT_GF128_ELEMENT pState, _In_reads_(cbData) PCBYTE pbSrc, _Out_writes_(cbData) PBYTE pbDst, SIZE_T cbData)
SymCryptAesGcmEncryptStitchedXmm
VOID SYMCRYPT_CALL SymCryptAesGcmEncryptStitchedXmm(_In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, _In_reads_(SYMCRYPT_AES_BLOCK_SIZE) PBYTE pbChainingValue, _In_reads_(SYMCRYPT_GF128_FIELD_SIZE) PCSYMCRYPT_GF128_ELEMENT expandedKeyTable, _Inout_ PSYMCRYPT_GF128_ELEMENT pState, _In_reads_(cbData) PCBYTE pbSrc, _Out_writes_(cbData) PBYTE pbDst, SIZE_T cbData)
SymCryptAesGcmDecryptStitchedXmm
VOID SYMCRYPT_CALL SymCryptAesGcmDecryptStitchedXmm(_In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, _In_reads_(SYMCRYPT_AES_BLOCK_SIZE) PBYTE pbChainingValue, _In_reads_(SYMCRYPT_GF128_FIELD_SIZE) PCSYMCRYPT_GF128_ELEMENT expandedKeyTable, _Inout_ PSYMCRYPT_GF128_ELEMENT pState, _In_reads_(cbData) PCBYTE pbSrc, _Out_writes_(cbData) PBYTE pbDst, SIZE_T cbData)
SymCryptAesGcmEncryptStitchedYmm_2048
VOID SYMCRYPT_CALL SymCryptAesGcmEncryptStitchedYmm_2048(_In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, _In_reads_(SYMCRYPT_AES_BLOCK_SIZE) PBYTE pbChainingValue, _In_reads_(SYMCRYPT_GF128_FIELD_SIZE) PCSYMCRYPT_GF128_ELEMENT expandedKeyTable, _Inout_ PSYMCRYPT_GF128_ELEMENT pState, _In_reads_(cbData) PCBYTE pbSrc, _Out_writes_(cbData) PBYTE pbDst, SIZE_T cbData)
SymCryptXtsAesDecryptDataUnitYmm_2048
VOID SYMCRYPT_CALL SymCryptXtsAesDecryptDataUnitYmm_2048(_In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, _Inout_updates_(SYMCRYPT_AES_BLOCK_SIZE) PBYTE pbTweakBlock, _Out_writes_(SYMCRYPT_AES_BLOCK_SIZE *16) PBYTE pbScratch, _In_reads_(cbData) PCBYTE pbSrc, _Out_writes_(cbData) PBYTE pbDst, SIZE_T cbData)
SymCryptXtsAesEncryptDataUnitYmm_2048
VOID SYMCRYPT_CALL SymCryptXtsAesEncryptDataUnitYmm_2048(_In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, _Inout_updates_(SYMCRYPT_AES_BLOCK_SIZE) PBYTE pbTweakBlock, _Out_writes_(SYMCRYPT_AES_BLOCK_SIZE *16) PBYTE pbScratch, _In_reads_(cbData) PCBYTE pbSrc, _Out_writes_(cbData) PBYTE pbDst, SIZE_T cbData)
SYMCRYPT_ASSERT
#define SYMCRYPT_ASSERT(_x)
Definition: symcrypt.h:10807
SYMCRYPT_AES_BLOCK_SIZE
#define SYMCRYPT_AES_BLOCK_SIZE
Definition: symcrypt.h:4255
pbChainingValue
PBYTE pbChainingValue
Definition: symcrypt_internal.h:1990
pbSrc
PCBYTE pbSrc
Definition: symcrypt_internal.h:1988
SYMCRYPT_CALL
#define SYMCRYPT_CALL
Definition: symcrypt_internal.h:167
PCSYMCRYPT_GF128_ELEMENT
const SYMCRYPT_GF128_ELEMENT * PCSYMCRYPT_GF128_ELEMENT
Definition: symcrypt_internal.h:1941
SYMCRYPT_GCM_BLOCK_MOD_MASK
#define SYMCRYPT_GCM_BLOCK_MOD_MASK
Definition: symcrypt_internal.h:1859
PCSYMCRYPT_AES_EXPANDED_KEY
const SYMCRYPT_AES_EXPANDED_KEY * PCSYMCRYPT_AES_EXPANDED_KEY
Definition: symcrypt_internal.h:1656
SYMCRYPT_GF128_FIELD_SIZE
#define SYMCRYPT_GF128_FIELD_SIZE
Definition: symcrypt_internal.h:1851
SYMCRYPT_GF128_BLOCK_SIZE
#define SYMCRYPT_GF128_BLOCK_SIZE
Definition: symcrypt_internal.h:1852
SYMCRYPT_MIN
#define SYMCRYPT_MIN(_a, _b)
Definition: symcrypt_internal.h:312
PSYMCRYPT_GF128_ELEMENT
* PSYMCRYPT_GF128_ELEMENT
Definition: symcrypt_internal.h:1940
cbData
PCBYTE PBYTE SIZE_T cbData
Definition: symcrypt_internal.h:1989
pbDst
PCBYTE PBYTE pbDst
Definition: symcrypt_internal.h:1988
pState
PSYMCRYPT_COMMON_HASH_STATE pState
Definition: symcrypt_internal.h:1278
PCBYTE
const BYTE * PCBYTE
Definition: symcrypt_internal.h:248
pExpandedKey
PCVOID pExpandedKey
Definition: symcrypt_internal.h:1826
chain
struct sock * chain
Definition: tcpcore.h:1
SIZE_T
ULONG_PTR SIZE_T
Definition: typedefs.h:80
xtsaes_definitions.h
XTS_MUL_ALPHA8_YMM
#define XTS_MUL_ALPHA8_YMM(_in, _res)
Definition: xtsaes_definitions.h:126
XTS_MUL_ALPHA
#define XTS_MUL_ALPHA(_in, _res)
Definition: xtsaes_definitions.h:41
XTS_MUL_ALPHA16_YMM
#define XTS_MUL_ALPHA16_YMM(_in, _res)
Definition: xtsaes_definitions.h:136
XTS_MUL_ALPHA4
#define XTS_MUL_ALPHA4(_in, _res)
Definition: xtsaes_definitions.h:92