fse.h File Reference

#include <stddef.h>

Include dependency graph for fse.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Macros
#define	FSE_H
#define	FSE_PUBLIC_API
#define	FSE_VERSION_MAJOR   0
#define	FSE_VERSION_MINOR   9
#define	FSE_VERSION_RELEASE   0
#define	FSE_LIB_VERSION   FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE
#define	FSE_QUOTE(str)   #str
#define	FSE_EXPAND_AND_QUOTE(str)   FSE_QUOTE(str)
#define	FSE_VERSION_STRING   FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)
#define	FSE_VERSION_NUMBER   (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)

Typedefs
typedef unsigned	FSE_CTable
typedef unsigned	FSE_DTable

Functions
FSE_PUBLIC_API unsigned	FSE_versionNumber (void)
FSE_PUBLIC_API size_t	FSE_compress (void *dst, size_t dstCapacity, const void *src, size_t srcSize)
FSE_PUBLIC_API size_t	FSE_decompress (void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize)
FSE_PUBLIC_API size_t	FSE_compressBound (size_t size)
FSE_PUBLIC_API unsigned	FSE_isError (size_t code)
FSE_PUBLIC_API const char *	FSE_getErrorName (size_t code)
FSE_PUBLIC_API size_t	FSE_compress2 (void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog)
FSE_PUBLIC_API unsigned	FSE_optimalTableLog (unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
FSE_PUBLIC_API size_t	FSE_normalizeCount (short *normalizedCounter, unsigned tableLog, const unsigned *count, size_t srcSize, unsigned maxSymbolValue)
FSE_PUBLIC_API size_t	FSE_NCountWriteBound (unsigned maxSymbolValue, unsigned tableLog)
FSE_PUBLIC_API size_t	FSE_writeNCount (void *buffer, size_t bufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
FSE_PUBLIC_API FSE_CTable *	FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog)
FSE_PUBLIC_API void	FSE_freeCTable (FSE_CTable *ct)
FSE_PUBLIC_API size_t	FSE_buildCTable (FSE_CTable *ct, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
FSE_PUBLIC_API size_t	FSE_compress_usingCTable (void *dst, size_t dstCapacity, const void *src, size_t srcSize, const FSE_CTable *ct)
FSE_PUBLIC_API size_t	FSE_readNCount (short *normalizedCounter, unsigned *maxSymbolValuePtr, unsigned *tableLogPtr, const void *rBuffer, size_t rBuffSize)
FSE_PUBLIC_API FSE_DTable *	FSE_createDTable (unsigned tableLog)
FSE_PUBLIC_API void	FSE_freeDTable (FSE_DTable *dt)
FSE_PUBLIC_API size_t	FSE_buildDTable (FSE_DTable *dt, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
FSE_PUBLIC_API size_t	FSE_decompress_usingDTable (void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt)

Macro Definition Documentation

FSE_EXPAND_AND_QUOTE

#define FSE_EXPAND_AND_QUOTE

(

str

)

FSE_QUOTE(str)

Definition at line 49 of file fse.h.

FSE_H

#define FSE_H

Definition at line 20 of file fse.h.

FSE_LIB_VERSION

#define FSE_LIB_VERSION   FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE

Definition at line 47 of file fse.h.

FSE_PUBLIC_API

#define FSE_PUBLIC_API

Definition at line 39 of file fse.h.

FSE_QUOTE

#define FSE_QUOTE

(

str

)

#str

Definition at line 48 of file fse.h.

FSE_VERSION_MAJOR

#define FSE_VERSION_MAJOR   0

Definition at line 43 of file fse.h.

FSE_VERSION_MINOR

#define FSE_VERSION_MINOR   9

Definition at line 44 of file fse.h.

FSE_VERSION_NUMBER

#define FSE_VERSION_NUMBER   (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)

Definition at line 52 of file fse.h.

FSE_VERSION_RELEASE

#define FSE_VERSION_RELEASE   0

Definition at line 45 of file fse.h.

FSE_VERSION_STRING

#define FSE_VERSION_STRING   FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)

Definition at line 50 of file fse.h.

Typedef Documentation

FSE_CTable

typedef unsigned FSE_CTable

Constructor and Destructor of FSE_CTable. Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue'

Definition at line 160 of file fse.h.

FSE_DTable

typedef unsigned FSE_DTable

Constructor and Destructor of FSE_DTable. Note that its size depends on 'tableLog'

Definition at line 233 of file fse.h.

Function Documentation

FSE_buildCTable()

FSE_PUBLIC_API size_t FSE_buildCTable	(	FSE_CTable *	ct,
		const short *	normalizedCounter,
		unsigned	maxSymbolValue,
		unsigned	tableLog
	)

FSE_buildCTable(): Builds ct, which must be already allocated, using FSE_createCTable().

Returns

: 0, or an errorCode, which can be tested using FSE_isError()

Definition at line 175 of file fse_compress.c.

{
    FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE];   /* memset() is not necessary, even if static analyzer complain about it */
    return FSE_buildCTable_wksp(ct, normalizedCounter, maxSymbolValue, tableLog, tableSymbol, sizeof(tableSymbol));
}

FSE_buildDTable()

FSE_PUBLIC_API size_t FSE_buildDTable	(	FSE_DTable *	dt,
		const short *	normalizedCounter,
		unsigned	maxSymbolValue,
		unsigned	tableLog
	)

FSE_buildDTable(): Builds 'dt', which must be already allocated, using FSE_createDTable(). return : 0, or an errorCode, which can be tested using FSE_isError()

Definition at line 73 of file fse_decompress.c.

{
    void* const tdPtr = dt+1;   /* because *dt is unsigned, 32-bits aligned on 32-bits */
    FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr);
    U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1];
 
    U32 const maxSV1 = maxSymbolValue + 1;
    U32 const tableSize = 1 << tableLog;
    U32 highThreshold = tableSize-1;
 
    /* Sanity Checks */
    if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge);
    if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
 
    /* Init, lay down lowprob symbols */
    {   FSE_DTableHeader DTableH;
        DTableH.tableLog = (U16)tableLog;
        DTableH.fastMode = 1;
        {   S16 const largeLimit= (S16)(1 << (tableLog-1));
            U32 s;
            for (s=0; s<maxSV1; s++) {
                if (normalizedCounter[s]==-1) {
                    tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
                    symbolNext[s] = 1;
                } else {
                    if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
                    symbolNext[s] = normalizedCounter[s];
        }   }   }
        memcpy(dt, &DTableH, sizeof(DTableH));
    }
 
    /* Spread symbols */
    {   U32 const tableMask = tableSize-1;
        U32 const step = FSE_TABLESTEP(tableSize);
        U32 s, position = 0;
        for (s=0; s<maxSV1; s++) {
            int i;
            for (i=0; i<normalizedCounter[s]; i++) {
                tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
                position = (position + step) & tableMask;
                while (position > highThreshold) position = (position + step) & tableMask;   /* lowprob area */
        }   }
        if (position!=0) return ERROR(GENERIC);   /* position must reach all cells once, otherwise normalizedCounter is incorrect */
    }
 
    /* Build Decoding table */
    {   U32 u;
        for (u=0; u<tableSize; u++) {
            FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
            U32 const nextState = symbolNext[symbol]++;
            tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32(nextState) );
            tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
    }   }
 
    return 0;
}

Referenced by FSE_decompress_wksp().

FSE_compress()

FSE_PUBLIC_API size_t FSE_compress	(	void *	dst,
		size_t	dstCapacity,
		const void *	src,
		size_t	srcSize
	)

FSE_compress() : Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. 'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize).

Returns

: size of compressed data (<= dstCapacity). Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead. if FSE_isError(return), compression failed (more details using FSE_getErrorName())

Definition at line 695 of file fse_compress.c.

{
    return FSE_compress2(dst, dstCapacity, src, srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG);
}

FSE_compress2()

FSE_PUBLIC_API size_t FSE_compress2	(	void *	dst,
		size_t	dstSize,
		const void *	src,
		size_t	srcSize,
		unsigned	maxSymbolValue,
		unsigned	tableLog
	)

FSE_compress2() : Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog' Both parameters can be defined as '0' to mean : use default value

Returns

: size of compressed data Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!! if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression. if FSE_isError(return), it's an error code.

Definition at line 687 of file fse_compress.c.

{
    fseWkspMax_t scratchBuffer;
    DEBUG_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE));   /* compilation failures here means scratchBuffer is not large enough */
    if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
    return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, &scratchBuffer, sizeof(scratchBuffer));
}

Referenced by FSE_compress().

FSE_compress_usingCTable()

FSE_PUBLIC_API size_t FSE_compress_usingCTable	(	void *	dst,
		size_t	dstCapacity,
		const void *	src,
		size_t	srcSize,
		const FSE_CTable *	ct
	)

FSE_compress_usingCTable(): Compress src using ct into dst which must be already allocated.

Returns

: size of compressed data (<= dstCapacity), or 0 if compressed data could not fit into dst, or an errorCode, which can be tested using FSE_isError()

Definition at line 616 of file fse_compress.c.

{
    unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize));
 
    if (fast)
        return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1);
    else
        return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0);
}

Referenced by FSE_compress_wksp(), and HUF_compressWeights().

FSE_compressBound()

FSE_PUBLIC_API size_t FSE_compressBound

(

size_t

size

)

Definition at line 629 of file fse_compress.c.

{ return FSE_COMPRESSBOUND(size); }

FSE_createCTable()

FSE_PUBLIC_API FSE_CTable * FSE_createCTable	(	unsigned	maxSymbolValue,
		unsigned	tableLog
	)

Definition at line 308 of file fse_compress.c.

{
    size_t size;
    if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
    size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
    return (FSE_CTable*)ExAllocatePoolWithTag(PagedPool, size, FSEC_ALLOC_TAG);
}

FSE_createDTable()

FSE_PUBLIC_API FSE_DTable * FSE_createDTable

(

unsigned

tableLog

)

Definition at line 62 of file fse_decompress.c.

{
    if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
    return (FSE_DTable*)ExAllocatePoolWithTag(PagedPool, FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32), FSED_ALLOC_TAG);
}

FSE_decompress()

FSE_PUBLIC_API size_t FSE_decompress	(	void *	dst,
		size_t	dstCapacity,
		const void *	cSrc,
		size_t	cSrcSize
	)

FSE_decompress(): Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', into already allocated destination buffer 'dst', of size 'dstCapacity'.

Returns

: size of regenerated data (<= maxDstSize), or an error code, which can be tested using FSE_isError() .

Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!! Why ? : making this distinction requires a header. Header management is intentionally delegated to the user layer, which can better manage special cases.

Definition at line 281 of file fse_decompress.c.

{
    DTable_max_t dt;   /* Static analyzer seems unable to understand this table will be properly initialized later */
    return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, dt, FSE_MAX_TABLELOG);
}

FSE_decompress_usingDTable()

FSE_PUBLIC_API size_t FSE_decompress_usingDTable	(	void *	dst,
		size_t	dstCapacity,
		const void *	cSrc,
		size_t	cSrcSize,
		const FSE_DTable *	dt
	)

FSE_decompress_usingDTable(): Decompress compressed source cSrc of size cSrcSize using dt into dst which must be already allocated.

Returns

: size of regenerated data (necessarily <= dstCapacity), or an errorCode, which can be tested using FSE_isError()

Definition at line 243 of file fse_decompress.c.

{
    const void* ptr = dt;
    const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
    const U32 fastMode = DTableH->fastMode;
 
    /* select fast mode (static) */
    if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
    return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
}

Referenced by FSE_decompress_wksp().

FSE_freeCTable()

FSE_PUBLIC_API void FSE_freeCTable

(

FSE_CTable *

ct

)

Definition at line 316 of file fse_compress.c.

{ ExFreePool(ct); }

FSE_freeDTable()

FSE_PUBLIC_API void FSE_freeDTable

(

FSE_DTable *

dt

)

Definition at line 68 of file fse_decompress.c.

{
    ExFreePool(dt);
}

FSE_getErrorName()

FSE_PUBLIC_API const char * FSE_getErrorName

(

size_t

code

)

Definition at line 32 of file entropy_common.c.

{ return ERR_getErrorName(code); }

FSE_isError()

FSE_PUBLIC_API unsigned FSE_isError

(

size_t

code

)

Definition at line 31 of file entropy_common.c.

{ return ERR_isError(code); }

FSE_NCountWriteBound()

FSE_PUBLIC_API size_t FSE_NCountWriteBound	(	unsigned	maxSymbolValue,
		unsigned	tableLog
	)

FSE_NCountWriteBound(): Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. Typically useful for allocation purpose.

Definition at line 189 of file fse_compress.c.

{
    size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 3;
    return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND;  /* maxSymbolValue==0 ? use default */
}

Referenced by FSE_writeNCount().

FSE_normalizeCount()

FSE_PUBLIC_API size_t FSE_normalizeCount	(	short *	normalizedCounter,
		unsigned	tableLog,
		const unsigned *	count,
		size_t	srcSize,
		unsigned	maxSymbolValue
	)

FSE_normalizeCount(): normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).

Returns

: tableLog, or an errorCode, which can be tested using FSE_isError()

Definition at line 438 of file fse_compress.c.

{
    /* Sanity checks */
    if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
    if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC);   /* Unsupported size */
    if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);   /* Unsupported size */
    if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC);   /* Too small tableLog, compression potentially impossible */
 
    {   static U32 const rtbTable[] = {     0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 };
        U64 const scale = 62 - tableLog;
        U64 const step = ((U64)1<<62) / total;   /* <== here, one division ! */
        U64 const vStep = 1ULL<<(scale-20);
        int stillToDistribute = 1<<tableLog;
        unsigned s;
        unsigned largest=0;
        short largestP=0;
        U32 lowThreshold = (U32)(total >> tableLog);
 
        for (s=0; s<=maxSymbolValue; s++) {
            if (count[s] == total) return 0;   /* rle special case */
            if (count[s] == 0) { normalizedCounter[s]=0; continue; }
            if (count[s] <= lowThreshold) {
                normalizedCounter[s] = -1;
                stillToDistribute--;
            } else {
                short proba = (short)((count[s]*step) >> scale);
                if (proba<8) {
                    U64 restToBeat = vStep * rtbTable[proba];
                    proba += (count[s]*step) - ((U64)proba<<scale) > restToBeat;
                }
                if (proba > largestP) { largestP=proba; largest=s; }
                normalizedCounter[s] = proba;
                stillToDistribute -= proba;
        }   }
        if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) {
            /* corner case, need another normalization method */
            size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue);
            if (FSE_isError(errorCode)) return errorCode;
        }
        else normalizedCounter[largest] += (short)stillToDistribute;
    }
 
#if 0
    {   /* Print Table (debug) */
        U32 s;
        U32 nTotal = 0;
        for (s=0; s<=maxSymbolValue; s++)
            RAWLOG(2, "%3i: %4i \n", s, normalizedCounter[s]);
        for (s=0; s<=maxSymbolValue; s++)
            nTotal += abs(normalizedCounter[s]);
        if (nTotal != (1U<<tableLog))
            RAWLOG(2, "Warning !!! Total == %u != %u !!!", nTotal, 1U<<tableLog);
        getchar();
    }
#endif
 
    return tableLog;
}

Referenced by FSE_compress_wksp(), HUF_compressWeights(), ZSTD_buildCTable(), and ZSTD_NCountCost().

FSE_optimalTableLog()

FSE_PUBLIC_API unsigned FSE_optimalTableLog	(	unsigned	maxTableLog,
		size_t	srcSize,
		unsigned	maxSymbolValue
	)

FSE_compress() does the following:

1. count symbol occurrence from source[] into table count[] (see hist.h)
2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
3. save normalized counters to memory buffer using writeNCount()
4. build encoding table 'CTable' from normalized counters
5. encode the data stream using encoding table 'CTable'

FSE_decompress() does the following:

1. read normalized counters with readNCount()
2. build decoding table 'DTable' from normalized counters
3. decode the data stream using decoding table 'DTable'

The following API allows targeting specific sub-functions for advanced tasks. For example, it's possible to compress several blocks using the same 'CTable', or to save and provide normalized distribution using external method.

FSE_optimalTableLog(): dynamically downsize 'tableLog' when conditions are met. It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.

Returns

: recommended tableLog (necessarily <= 'maxTableLog')

Definition at line 342 of file fse_compress.c.

{
    return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2);
}

Referenced by FSE_compress_wksp(), HUF_compressWeights(), ZSTD_buildCTable(), and ZSTD_NCountCost().

FSE_readNCount()

FSE_PUBLIC_API size_t FSE_readNCount	(	short *	normalizedCounter,
		unsigned *	maxSymbolValuePtr,
		unsigned *	tableLogPtr,
		const void *	rBuffer,
		size_t	rBuffSize
	)

Tutorial :

The first step is to count all symbols. FSE_count() does this job very fast. Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. 'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) FSE_count() will return the number of occurrence of the most frequent symbol. This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).

The next step is to normalize the frequencies. FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. It also guarantees a minimum of 1 to any Symbol with frequency >= 1. You can use 'tableLog'==0 to mean "use default tableLog value". If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(), which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default").

The result of FSE_normalizeCount() will be saved into a table, called 'normalizedCounter', which is a table of signed short. 'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. The return value is tableLog if everything proceeded as expected. It is 0 if there is a single symbol within distribution. If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).

'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount(). 'buffer' must be already allocated. For guaranteed success, buffer size must be at least FSE_headerBound(). The result of the function is the number of bytes written into 'buffer'. If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small).

'normalizedCounter' can then be used to create the compression table 'CTable'. The space required by 'CTable' must be already allocated, using FSE_createCTable(). You can then use FSE_buildCTable() to fill 'CTable'. If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()).

'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' The function returns the size of compressed data (without header), necessarily <= dstCapacity. If it returns '0', compressed data could not fit into 'dst'. If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).

FSE_readNCount(): Read compactly saved 'normalizedCounter' from 'rBuffer'.

Returns

: size read from 'rBuffer', or an errorCode, which can be tested using FSE_isError(). maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values

Definition at line 41 of file entropy_common.c.

{
    const BYTE* const istart = (const BYTE*) headerBuffer;
    const BYTE* const iend = istart + hbSize;
    const BYTE* ip = istart;
    int nbBits;
    int remaining;
    int threshold;
    U32 bitStream;
    int bitCount;
    unsigned charnum = 0;
    int previous0 = 0;
 
    if (hbSize < 4) {
        /* This function only works when hbSize >= 4 */
        char buffer[4];
        memset(buffer, 0, sizeof(buffer));
        memcpy(buffer, headerBuffer, hbSize);
        {   size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr,
                                                    buffer, sizeof(buffer));
            if (FSE_isError(countSize)) return countSize;
            if (countSize > hbSize) return ERROR(corruption_detected);
            return countSize;
    }   }
    assert(hbSize >= 4);
 
    /* init */
    memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0]));   /* all symbols not present in NCount have a frequency of 0 */
    bitStream = MEM_readLE32(ip);
    nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG;   /* extract tableLog */
    if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge);
    bitStream >>= 4;
    bitCount = 4;
    *tableLogPtr = nbBits;
    remaining = (1<<nbBits)+1;
    threshold = 1<<nbBits;
    nbBits++;
 
    while ((remaining>1) & (charnum<=*maxSVPtr)) {
        if (previous0) {
            unsigned n0 = charnum;
            while ((bitStream & 0xFFFF) == 0xFFFF) {
                n0 += 24;
                if (ip < iend-5) {
                    ip += 2;
                    bitStream = MEM_readLE32(ip) >> bitCount;
                } else {
                    bitStream >>= 16;
                    bitCount   += 16;
            }   }
            while ((bitStream & 3) == 3) {
                n0 += 3;
                bitStream >>= 2;
                bitCount += 2;
            }
            n0 += bitStream & 3;
            bitCount += 2;
            if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall);
            while (charnum < n0) normalizedCounter[charnum++] = 0;
            if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
                assert((bitCount >> 3) <= 3); /* For first condition to work */
                ip += bitCount>>3;
                bitCount &= 7;
                bitStream = MEM_readLE32(ip) >> bitCount;
            } else {
                bitStream >>= 2;
        }   }
        {   int const max = (2*threshold-1) - remaining;
            int count;
 
            if ((bitStream & (threshold-1)) < (U32)max) {
                count = bitStream & (threshold-1);
                bitCount += nbBits-1;
            } else {
                count = bitStream & (2*threshold-1);
                if (count >= threshold) count -= max;
                bitCount += nbBits;
            }
 
            count--;   /* extra accuracy */
            remaining -= count < 0 ? -count : count;   /* -1 means +1 */
            normalizedCounter[charnum++] = (short)count;
            previous0 = !count;
            while (remaining < threshold) {
                nbBits--;
                threshold >>= 1;
            }
 
            if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
                ip += bitCount>>3;
                bitCount &= 7;
            } else {
                bitCount -= (int)(8 * (iend - 4 - ip));
                ip = iend - 4;
            }
            bitStream = MEM_readLE32(ip) >> (bitCount & 31);
    }   }   /* while ((remaining>1) & (charnum<=*maxSVPtr)) */
    if (remaining != 1) return ERROR(corruption_detected);
    if (bitCount > 32) return ERROR(corruption_detected);
    *maxSVPtr = charnum-1;
 
    ip += (bitCount+7)>>3;
    return ip-istart;
}

Referenced by FSE_decompress_wksp(), FSE_readNCount(), ZSTD_loadCEntropy(), and ZSTD_loadDEntropy().

FSE_versionNumber()

FSE_PUBLIC_API unsigned FSE_versionNumber

(

void

)

library version number; to be used when checking dll version

Definition at line 27 of file entropy_common.c.

{ return FSE_VERSION_NUMBER; }

FSE_writeNCount()

FSE_PUBLIC_API size_t FSE_writeNCount	(	void *	buffer,
		size_t	bufferSize,
		const short *	normalizedCounter,
		unsigned	maxSymbolValue,
		unsigned	tableLog
	)

FSE_writeNCount(): Compactly save 'normalizedCounter' into 'buffer'.

Returns

: size of the compressed table, or an errorCode, which can be tested using FSE_isError().

Definition at line 291 of file fse_compress.c.

{
    if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);   /* Unsupported */
    if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC);   /* Unsupported */
 
    if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
        return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
 
    return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1 /* write in buffer is safe */);
}

Referenced by FSE_compress_wksp(), HUF_compressWeights(), ZSTD_buildCTable(), and ZSTD_NCountCost().