ghash.c File Reference

#include "precomp.h"
#include "ghash_definitions.h"

Include dependency graph for ghash.c:

Go to the source code of this file.

Functions
VOID SYMCRYPT_CALL	SymCryptGHashExpandKeyC (_Out_writes_(SYMCRYPT_GF128_FIELD_SIZE) PSYMCRYPT_GF128_ELEMENT expandedKey, _In_reads_(SYMCRYPT_GF128_BLOCK_SIZE) PCBYTE pH)
VOID SYMCRYPT_CALL	SymCryptGHashAppendDataC (_In_reads_(SYMCRYPT_GF128_FIELD_SIZE) PCSYMCRYPT_GF128_ELEMENT expandedKeyTable, _Inout_ PSYMCRYPT_GF128_ELEMENT pState, _In_reads_(cbData) PCBYTE pbData, SIZE_T cbData)
VOID SYMCRYPT_CALL	SymCryptGHashResult (_In_ PCSYMCRYPT_GF128_ELEMENT pState, _Out_writes_(SYMCRYPT_GF128_BLOCK_SIZE) PBYTE pbResult)
VOID SYMCRYPT_CALL	SymCryptGHashExpandKeyXmm (_Out_writes_(SYMCRYPT_GF128_FIELD_SIZE) PSYMCRYPT_GF128_ELEMENT expandedKey, _In_reads_(SYMCRYPT_GF128_BLOCK_SIZE) PCBYTE pH)
VOID SYMCRYPT_CALL	SymCryptGHashExpandKey (_Out_ PSYMCRYPT_GHASH_EXPANDED_KEY expandedKey, _In_reads_(SYMCRYPT_GF128_BLOCK_SIZE) PCBYTE pH)
VOID SYMCRYPT_CALL	SymCryptGHashAppendData (_In_ PCSYMCRYPT_GHASH_EXPANDED_KEY expandedKey, _Inout_ PSYMCRYPT_GF128_ELEMENT pState, _In_reads_(cbData) PCBYTE pbData, SIZE_T cbData)

Function Documentation

SymCryptGHashAppendData()

VOID SYMCRYPT_CALL SymCryptGHashAppendData	(	_In_ PCSYMCRYPT_GHASH_EXPANDED_KEY	expandedKey,
		_Inout_ PSYMCRYPT_GF128_ELEMENT	pState,
		_In_reads_(cbData) PCBYTE	pbData,
		SIZE_T	cbData
	)

Definition at line 884 of file ghash.c.

{
#if SYMCRYPT_CPU_X86
    PCSYMCRYPT_GF128_ELEMENT pExpandedKeyTable;
    SYMCRYPT_EXTENDED_SAVE_DATA  SaveData;
 
    pExpandedKeyTable = (PSYMCRYPT_GF128_ELEMENT)&expandedKey->tableSpace[expandedKey->tableOffset];
 
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_PCLMULQDQ_CODE ) )
    {
        if( SymCryptSaveXmm( &SaveData ) != SYMCRYPT_NO_ERROR )
        {
            SymCryptFatal( 'pclm' );
        }
        SymCryptGHashAppendDataPclmulqdq( pExpandedKeyTable, pState, pbData, cbData );
        SymCryptRestoreXmm( &SaveData );
    } else if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_SSE2 ) && SymCryptSaveXmm( &SaveData ) == SYMCRYPT_NO_ERROR )
    {
        SymCryptGHashAppendDataXmm( pExpandedKeyTable, pState, pbData, cbData );
        SymCryptRestoreXmm( &SaveData );
    } else {
        SymCryptGHashAppendDataC( pExpandedKeyTable, pState, pbData, cbData );
    }
 
#elif SYMCRYPT_CPU_AMD64
    PCSYMCRYPT_GF128_ELEMENT pExpandedKeyTable;
 
    pExpandedKeyTable = &expandedKey->table[0];
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_PCLMULQDQ_CODE ) )
    {
        SymCryptGHashAppendDataPclmulqdq( pExpandedKeyTable, pState, pbData, cbData );
    } else if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_SSE2 ) )
    {
        SymCryptGHashAppendDataXmm( pExpandedKeyTable, pState, pbData, cbData );
    } else {
        SymCryptGHashAppendDataC( pExpandedKeyTable, pState, pbData, cbData );
    }
#elif SYMCRYPT_CPU_ARM
    PCSYMCRYPT_GF128_ELEMENT pExpandedKeyTable;
 
    pExpandedKeyTable = &expandedKey->table[0];
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON ) )
    {
        SymCryptGHashAppendDataNeon( pExpandedKeyTable, pState, pbData, cbData );
    } else {
        SymCryptGHashAppendDataC( pExpandedKeyTable, pState, pbData, cbData );
    }
#elif SYMCRYPT_CPU_ARM64
    PCSYMCRYPT_GF128_ELEMENT pExpandedKeyTable;
 
    pExpandedKeyTable = &expandedKey->table[0];
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON_PMULL ) )
    {
        SymCryptGHashAppendDataPmull( pExpandedKeyTable, pState, pbData, cbData );
    } else if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON ) )
    {
        SymCryptGHashAppendDataNeon( pExpandedKeyTable, pState, pbData, cbData );
    } else {
        SymCryptGHashAppendDataC( pExpandedKeyTable, pState, pbData, cbData );
    }
#else
    SymCryptGHashAppendDataC( &expandedKey->table[0], pState, pbData, cbData );
#endif
}

Referenced by SymCryptAesGcmDecryptPartOnePass(), SymCryptAesGcmEncryptPartOnePass(), SymCryptGcmAddMacData(), SymCryptGcmComputeTag(), SymCryptGcmDecrypt(), SymCryptGcmEncrypt(), SymCryptGcmPadMacData(), and SymCryptGcmSetNonce().

SymCryptGHashAppendDataC()

VOID SYMCRYPT_CALL SymCryptGHashAppendDataC	(	_In_reads_(SYMCRYPT_GF128_FIELD_SIZE) PCSYMCRYPT_GF128_ELEMENT	expandedKeyTable,
		_Inout_ PSYMCRYPT_GF128_ELEMENT	pState,
		_In_reads_(cbData) PCBYTE	pbData,
		SIZE_T	cbData
	)

Definition at line 61 of file ghash.c.

{
    UINT64 R0, R1;
    UINT64 mask;
    SYMCRYPT_ALIGN UINT32 state32[4];
    UINT32 t;
    int i,j;
    while( cbData >= SYMCRYPT_GF128_BLOCK_SIZE )
    {
        R0 = R1 = 0;
 
        //
        // We have two nested loops so that we can do most of our operations
        // on 32-bit words. 64-bit rotates/shifts can be really slow on a 32-bit CPU.
        // On AMD64 we use the XMM version which is much faster.
        //
        state32[0] = (UINT32)pState->ull[0];
        state32[1] = (UINT32)(pState->ull[0] >> 32);
        state32[2] = (UINT32)pState->ull[1];
        state32[3] = (UINT32)(pState->ull[1] >> 32);
        for( i=0; i<4; i++ )
        {
            t = SYMCRYPT_LOAD_MSBFIRST32( &pbData[4*i] ) ^ state32[3-i];
            for( j=31; j>=0; j-- )
            {
                mask = (UINT64)( -(INT64)(t & 1 ));
                R0 ^= expandedKeyTable[32*i+j].ull[0] & mask;
                R1 ^= expandedKeyTable[32*i+j].ull[1] & mask;
                t >>= 1;
            }
        }
        pState->ull[0] = R0;
        pState->ull[1] = R1;
        pbData += SYMCRYPT_GF128_BLOCK_SIZE;
        cbData -= SYMCRYPT_GF128_BLOCK_SIZE;
    }
 
    SymCryptWipeKnownSize( state32, sizeof( state32 ) );
}

Referenced by SymCryptGHashAppendData().

SymCryptGHashExpandKey()

VOID SYMCRYPT_CALL SymCryptGHashExpandKey	(	_Out_ PSYMCRYPT_GHASH_EXPANDED_KEY	expandedKey,
		_In_reads_(SYMCRYPT_GF128_BLOCK_SIZE) PCBYTE	pH
	)

Definition at line 816 of file ghash.c.

{
#if  SYMCRYPT_CPU_X86
    PSYMCRYPT_GF128_ELEMENT pExpandedKeyTable;
    SYMCRYPT_EXTENDED_SAVE_DATA  SaveData;
 
    //
    // Initialize offset into table space for 16-alignment.
    //
    expandedKey->tableOffset = (0 -((UINT_PTR) &expandedKey->tableSpace[0])) % sizeof(SYMCRYPT_GF128_ELEMENT);
 
    pExpandedKeyTable = (PSYMCRYPT_GF128_ELEMENT)&expandedKey->tableSpace[expandedKey->tableOffset];
 
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_PCLMULQDQ_CODE ) )
    {
        //
        // We can only use the PCLMULQDQ data representation if the SaveXmm never fails.
        // This is one of the CPU features required.
        // We check anyway...
        //
        if( SymCryptSaveXmm( &SaveData ) != SYMCRYPT_NO_ERROR )
        {
            SymCryptFatal( 'pclm' );
        }
        SymCryptGHashExpandKeyPclmulqdq( pExpandedKeyTable, pH );
        SymCryptRestoreXmm( &SaveData );
    } else if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_SSE2 ) && SymCryptSaveXmm( &SaveData ) == SYMCRYPT_NO_ERROR )
    {
        SymCryptGHashExpandKeyXmm( pExpandedKeyTable, pH );
        SymCryptRestoreXmm( &SaveData );
    } else {
        SymCryptGHashExpandKeyC( pExpandedKeyTable, pH );
    }
 
#elif SYMCRYPT_CPU_AMD64
    PSYMCRYPT_GF128_ELEMENT pExpandedKeyTable;
    pExpandedKeyTable = &expandedKey->table[0];
 
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_PCLMULQDQ_CODE ) )
    {
        SymCryptGHashExpandKeyPclmulqdq( pExpandedKeyTable, pH );
    } else if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_SSE2 ) )
    {
        SymCryptGHashExpandKeyXmm( pExpandedKeyTable, pH );
    } else {
        SymCryptGHashExpandKeyC( pExpandedKeyTable, pH );
    }
 
#elif SYMCRYPT_CPU_ARM64
    PSYMCRYPT_GF128_ELEMENT pExpandedKeyTable;
    pExpandedKeyTable = &expandedKey->table[0];
 
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON_PMULL ) )
    {
        SymCryptGHashExpandKeyPmull( pExpandedKeyTable, pH );
    } else {
        SymCryptGHashExpandKeyC( pExpandedKeyTable, pH );
    }
 
#else
    SymCryptGHashExpandKeyC( &expandedKey->table[0], pH );      // Default expansion (does not need alignment)
#endif
}

Referenced by SymCryptGcmExpandKey().

SymCryptGHashExpandKeyC()

VOID SYMCRYPT_CALL SymCryptGHashExpandKeyC	(	_Out_writes_(SYMCRYPT_GF128_FIELD_SIZE) PSYMCRYPT_GF128_ELEMENT	expandedKey,
		_In_reads_(SYMCRYPT_GF128_BLOCK_SIZE) PCBYTE	pH
	)

Definition at line 27 of file ghash.c.

{
    UINT64 H0, H1, t;
    UINT32 i;
 
    //
    // (H1, H0) form a 128-bit integer, H1 is the upper part, H0 the lower part.
    // Convert pH[] to (H1, H0) using MSByte first convention.
    //
    H1 = SYMCRYPT_LOAD_MSBFIRST64( &pH[0] );
    H0 = SYMCRYPT_LOAD_MSBFIRST64( &pH[8] );
 
    for( i=0; i<SYMCRYPT_GF128_FIELD_SIZE; i++ )
    {
        expandedKey[i].ull[0] = H0;
        expandedKey[i].ull[1] = H1;
        //
        // Multiply (H1,H0) by x in the GF(2^128) field using the field encoding from SP800-38D
        //
        t =  UINT64_NEG(H0 & 1) & ((UINT64)GF128_FIELD_R_BYTE << (8 * ( sizeof( UINT64 ) - 1 )) ) ;
        H0 = (H0 >> 1) | (H1 << 63);
        H1 = (H1 >> 1) ^ t;
    }
}

Referenced by SymCryptGHashExpandKey(), and SymCryptGHashExpandKeyXmm().

SymCryptGHashExpandKeyXmm()

VOID SYMCRYPT_CALL SymCryptGHashExpandKeyXmm	(	_Out_writes_(SYMCRYPT_GF128_FIELD_SIZE) PSYMCRYPT_GF128_ELEMENT	expandedKey,
		_In_reads_(SYMCRYPT_GF128_BLOCK_SIZE) PCBYTE	pH
	)

Definition at line 122 of file ghash.c.

{
    //
    // We use the same layout for XMM code as we did for C code, so we can use the same key
    // expansion code.
    // Improvement: we can add an expansion routine that uses the XMM registers for speed.
    //
 
    SymCryptGHashExpandKeyC( expandedKey, pH );
}

Referenced by SymCryptGHashExpandKey().

SymCryptGHashResult()

VOID SYMCRYPT_CALL SymCryptGHashResult	(	_In_ PCSYMCRYPT_GF128_ELEMENT	pState,
		_Out_writes_(SYMCRYPT_GF128_BLOCK_SIZE) PBYTE	pbResult
	)

Definition at line 108 of file ghash.c.

{
    SYMCRYPT_STORE_MSBFIRST64( pbResult    , pState->ull[1] );
    SYMCRYPT_STORE_MSBFIRST64( pbResult + 8, pState->ull[0] );
}

Referenced by SymCryptGcmSetNonce().