zstd_compress.c File Reference

#include <limits.h>
#include <string.h>
#include "cpu.h"
#include "mem.h"
#include "hist.h"
#include "fse.h"
#include "huf.h"
#include "zstd_compress_internal.h"
#include "zstd_compress_sequences.h"
#include "zstd_compress_literals.h"
#include "zstd_fast.h"
#include "zstd_double_fast.h"
#include "zstd_lazy.h"
#include "zstd_opt.h"
#include "zstd_ldm.h"
#include "zstd_compress_superblock.h"

Include dependency graph for zstd_compress.c:

Go to the source code of this file.

Classes
struct	ZSTD_CDict_s

Macros
#define	FSE_STATIC_LINKING_ONLY   /* FSE_encodeSymbol */
#define	HUF_STATIC_LINKING_ONLY
#define	BOUNDCHECK(cParam, val)
#define	CLAMP_TYPE(cParam, val, type)
#define	CLAMP(cParam, val)   CLAMP_TYPE(cParam, val, unsigned)
#define	ZSTD_INDEXOVERFLOW_MARGIN   (16 MB)
#define	ZSTD_ROWSIZE   16
#define	ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF   (128 KB)
#define	ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER   (6)
#define	ZSTD_MAX_CLEVEL   22

Enumerations
enum	ZSTD_buffered_policy_e { ZSTDb_not_buffered , ZSTDb_buffered }
enum	ZSTD_compResetPolicy_e { ZSTDcrp_makeClean , ZSTDcrp_leaveDirty }
enum	ZSTD_indexResetPolicy_e { ZSTDirp_continue , ZSTDirp_reset }
enum	ZSTD_resetTarget_e { ZSTD_resetTarget_CDict , ZSTD_resetTarget_CCtx }
enum	ZSTD_buildSeqStore_e { ZSTDbss_compress , ZSTDbss_noCompress }

Functions
size_t	ZSTD_compressBound (size_t srcSize)
ZSTD_CCtx *	ZSTD_createCCtx (void)
static void	ZSTD_initCCtx (ZSTD_CCtx *cctx, ZSTD_customMem memManager)
ZSTD_CCtx *	ZSTD_createCCtx_advanced (ZSTD_customMem customMem)
ZSTD_CCtx *	ZSTD_initStaticCCtx (void *workspace, size_t workspaceSize)
static void	ZSTD_clearAllDicts (ZSTD_CCtx *cctx)
static size_t	ZSTD_sizeof_localDict (ZSTD_localDict dict)
static void	ZSTD_freeCCtxContent (ZSTD_CCtx *cctx)
size_t	ZSTD_freeCCtx (ZSTD_CCtx *cctx)
static size_t	ZSTD_sizeof_mtctx (const ZSTD_CCtx *cctx)
size_t	ZSTD_sizeof_CCtx (const ZSTD_CCtx *cctx)
size_t	ZSTD_sizeof_CStream (const ZSTD_CStream *zcs)
const seqStore_t *	ZSTD_getSeqStore (const ZSTD_CCtx *ctx)
static ZSTD_CCtx_params	ZSTD_makeCCtxParamsFromCParams (ZSTD_compressionParameters cParams)
static ZSTD_CCtx_params *	ZSTD_createCCtxParams_advanced (ZSTD_customMem customMem)
ZSTD_CCtx_params *	ZSTD_createCCtxParams (void)
size_t	ZSTD_freeCCtxParams (ZSTD_CCtx_params *params)
size_t	ZSTD_CCtxParams_reset (ZSTD_CCtx_params *params)
size_t	ZSTD_CCtxParams_init (ZSTD_CCtx_params *cctxParams, int compressionLevel)
size_t	ZSTD_CCtxParams_init_advanced (ZSTD_CCtx_params *cctxParams, ZSTD_parameters params)
static ZSTD_CCtx_params	ZSTD_assignParamsToCCtxParams (const ZSTD_CCtx_params *cctxParams, const ZSTD_parameters *params)
ZSTD_bounds	ZSTD_cParam_getBounds (ZSTD_cParameter param)
static size_t	ZSTD_cParam_clampBounds (ZSTD_cParameter cParam, int *value)
static int	ZSTD_isUpdateAuthorized (ZSTD_cParameter param)
size_t	ZSTD_CCtx_setParameter (ZSTD_CCtx *cctx, ZSTD_cParameter param, int value)
size_t	ZSTD_CCtxParams_setParameter (ZSTD_CCtx_params *CCtxParams, ZSTD_cParameter param, int value)
size_t	ZSTD_CCtx_getParameter (ZSTD_CCtx *cctx, ZSTD_cParameter param, int *value)
size_t	ZSTD_CCtxParams_getParameter (ZSTD_CCtx_params *CCtxParams, ZSTD_cParameter param, int *value)
size_t	ZSTD_CCtx_setParametersUsingCCtxParams (ZSTD_CCtx *cctx, const ZSTD_CCtx_params *params)
ZSTDLIB_API size_t	ZSTD_CCtx_setPledgedSrcSize (ZSTD_CCtx *cctx, unsigned long long pledgedSrcSize)
static size_t	ZSTD_initLocalDict (ZSTD_CCtx *cctx)
size_t	ZSTD_CCtx_loadDictionary_advanced (ZSTD_CCtx *cctx, const void *dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType)
ZSTDLIB_API size_t	ZSTD_CCtx_loadDictionary_byReference (ZSTD_CCtx *cctx, const void *dict, size_t dictSize)
ZSTDLIB_API size_t	ZSTD_CCtx_loadDictionary (ZSTD_CCtx *cctx, const void *dict, size_t dictSize)
size_t	ZSTD_CCtx_refCDict (ZSTD_CCtx *cctx, const ZSTD_CDict *cdict)
size_t	ZSTD_CCtx_refPrefix (ZSTD_CCtx *cctx, const void *prefix, size_t prefixSize)
size_t	ZSTD_CCtx_refPrefix_advanced (ZSTD_CCtx *cctx, const void *prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType)
size_t	ZSTD_CCtx_reset (ZSTD_CCtx *cctx, ZSTD_ResetDirective reset)
size_t	ZSTD_checkCParams (ZSTD_compressionParameters cParams)
static ZSTD_compressionParameters	ZSTD_clampCParams (ZSTD_compressionParameters cParams)
U32	ZSTD_cycleLog (U32 hashLog, ZSTD_strategy strat)
static ZSTD_compressionParameters	ZSTD_adjustCParams_internal (ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize)
ZSTD_compressionParameters	ZSTD_adjustCParams (ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize)
static ZSTD_compressionParameters	ZSTD_getCParams_internal (int compressionLevel, unsigned long long srcSizeHint, size_t dictSize)
static ZSTD_parameters	ZSTD_getParams_internal (int compressionLevel, unsigned long long srcSizeHint, size_t dictSize)
ZSTD_compressionParameters	ZSTD_getCParamsFromCCtxParams (const ZSTD_CCtx_params *CCtxParams, U64 srcSizeHint, size_t dictSize)
static size_t	ZSTD_sizeof_matchState (const ZSTD_compressionParameters *const cParams, const U32 forCCtx)
size_t	ZSTD_estimateCCtxSize_usingCCtxParams (const ZSTD_CCtx_params *params)
size_t	ZSTD_estimateCCtxSize_usingCParams (ZSTD_compressionParameters cParams)
static size_t	ZSTD_estimateCCtxSize_internal (int compressionLevel)
size_t	ZSTD_estimateCCtxSize (int compressionLevel)
size_t	ZSTD_estimateCStreamSize_usingCCtxParams (const ZSTD_CCtx_params *params)
size_t	ZSTD_estimateCStreamSize_usingCParams (ZSTD_compressionParameters cParams)
static size_t	ZSTD_estimateCStreamSize_internal (int compressionLevel)
size_t	ZSTD_estimateCStreamSize (int compressionLevel)
ZSTD_frameProgression	ZSTD_getFrameProgression (const ZSTD_CCtx *cctx)
size_t	ZSTD_toFlushNow (ZSTD_CCtx *cctx)
static void	ZSTD_assertEqualCParams (ZSTD_compressionParameters cParams1, ZSTD_compressionParameters cParams2)
void	ZSTD_reset_compressedBlockState (ZSTD_compressedBlockState_t *bs)
static void	ZSTD_invalidateMatchState (ZSTD_matchState_t *ms)
static size_t	ZSTD_reset_matchState (ZSTD_matchState_t *ms, ZSTD_cwksp *ws, const ZSTD_compressionParameters *cParams, const ZSTD_compResetPolicy_e crp, const ZSTD_indexResetPolicy_e forceResetIndex, const ZSTD_resetTarget_e forWho)
static int	ZSTD_indexTooCloseToMax (ZSTD_window_t w)
static size_t	ZSTD_resetCCtx_internal (ZSTD_CCtx *zc, ZSTD_CCtx_params params, U64 const pledgedSrcSize, ZSTD_compResetPolicy_e const crp, ZSTD_buffered_policy_e const zbuff)
void	ZSTD_invalidateRepCodes (ZSTD_CCtx *cctx)
static int	ZSTD_shouldAttachDict (const ZSTD_CDict *cdict, const ZSTD_CCtx_params *params, U64 pledgedSrcSize)
static size_t	ZSTD_resetCCtx_byAttachingCDict (ZSTD_CCtx *cctx, const ZSTD_CDict *cdict, ZSTD_CCtx_params params, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff)
static size_t	ZSTD_resetCCtx_byCopyingCDict (ZSTD_CCtx *cctx, const ZSTD_CDict *cdict, ZSTD_CCtx_params params, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff)
static size_t	ZSTD_resetCCtx_usingCDict (ZSTD_CCtx *cctx, const ZSTD_CDict *cdict, const ZSTD_CCtx_params *params, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff)
static size_t	ZSTD_copyCCtx_internal (ZSTD_CCtx *dstCCtx, const ZSTD_CCtx *srcCCtx, ZSTD_frameParameters fParams, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff)
size_t	ZSTD_copyCCtx (ZSTD_CCtx *dstCCtx, const ZSTD_CCtx *srcCCtx, unsigned long long pledgedSrcSize)
FORCE_INLINE_TEMPLATE void	ZSTD_reduceTable_internal (U32 *const table, U32 const size, U32 const reducerValue, int const preserveMark)
static void	ZSTD_reduceTable (U32 *const table, U32 const size, U32 const reducerValue)
static void	ZSTD_reduceTable_btlazy2 (U32 *const table, U32 const size, U32 const reducerValue)
static void	ZSTD_reduceIndex (ZSTD_matchState_t *ms, ZSTD_CCtx_params const *params, const U32 reducerValue)
void	ZSTD_seqToCodes (const seqStore_t *seqStorePtr)
static int	ZSTD_useTargetCBlockSize (const ZSTD_CCtx_params *cctxParams)
MEM_STATIC size_t	ZSTD_compressSequences_internal (seqStore_t *seqStorePtr, const ZSTD_entropyCTables_t *prevEntropy, ZSTD_entropyCTables_t *nextEntropy, const ZSTD_CCtx_params *cctxParams, void *dst, size_t dstCapacity, void *entropyWorkspace, size_t entropyWkspSize, const int bmi2)
MEM_STATIC size_t	ZSTD_compressSequences (seqStore_t *seqStorePtr, const ZSTD_entropyCTables_t *prevEntropy, ZSTD_entropyCTables_t *nextEntropy, const ZSTD_CCtx_params *cctxParams, void *dst, size_t dstCapacity, size_t srcSize, void *entropyWorkspace, size_t entropyWkspSize, int bmi2)
ZSTD_blockCompressor	ZSTD_selectBlockCompressor (ZSTD_strategy strat, ZSTD_dictMode_e dictMode)
static void	ZSTD_storeLastLiterals (seqStore_t *seqStorePtr, const BYTE *anchor, size_t lastLLSize)
void	ZSTD_resetSeqStore (seqStore_t *ssPtr)
static size_t	ZSTD_buildSeqStore (ZSTD_CCtx *zc, const void *src, size_t srcSize)
static void	ZSTD_copyBlockSequences (ZSTD_CCtx *zc)
size_t	ZSTD_getSequences (ZSTD_CCtx *zc, ZSTD_Sequence *outSeqs, size_t outSeqsSize, const void *src, size_t srcSize)
static int	ZSTD_isRLE (const BYTE *ip, size_t length)
static int	ZSTD_maybeRLE (seqStore_t const *seqStore)
static void	ZSTD_confirmRepcodesAndEntropyTables (ZSTD_CCtx *zc)
static size_t	ZSTD_compressBlock_internal (ZSTD_CCtx *zc, void *dst, size_t dstCapacity, const void *src, size_t srcSize, U32 frame)
static size_t	ZSTD_compressBlock_targetCBlockSize_body (ZSTD_CCtx *zc, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const size_t bss, U32 lastBlock)
static size_t	ZSTD_compressBlock_targetCBlockSize (ZSTD_CCtx *zc, void *dst, size_t dstCapacity, const void *src, size_t srcSize, U32 lastBlock)
static void	ZSTD_overflowCorrectIfNeeded (ZSTD_matchState_t *ms, ZSTD_cwksp *ws, ZSTD_CCtx_params const *params, void const *ip, void const *iend)
static size_t	ZSTD_compress_frameChunk (ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, U32 lastFrameChunk)
static size_t	ZSTD_writeFrameHeader (void *dst, size_t dstCapacity, const ZSTD_CCtx_params *params, U64 pledgedSrcSize, U32 dictID)
size_t	ZSTD_writeLastEmptyBlock (void *dst, size_t dstCapacity)
size_t	ZSTD_referenceExternalSequences (ZSTD_CCtx *cctx, rawSeq *seq, size_t nbSeq)
static size_t	ZSTD_compressContinue_internal (ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, U32 frame, U32 lastFrameChunk)
size_t	ZSTD_compressContinue (ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
size_t	ZSTD_getBlockSize (const ZSTD_CCtx *cctx)
size_t	ZSTD_compressBlock (ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
static size_t	ZSTD_loadDictionaryContent (ZSTD_matchState_t *ms, ldmState_t *ls, ZSTD_cwksp *ws, ZSTD_CCtx_params const *params, const void *src, size_t srcSize, ZSTD_dictTableLoadMethod_e dtlm)
static size_t	ZSTD_checkDictNCount (short *normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue)
size_t	ZSTD_loadCEntropy (ZSTD_compressedBlockState_t *bs, void *workspace, short *offcodeNCount, unsigned *offcodeMaxValue, const void *const dict, size_t dictSize)
static size_t	ZSTD_loadZstdDictionary (ZSTD_compressedBlockState_t *bs, ZSTD_matchState_t *ms, ZSTD_cwksp *ws, ZSTD_CCtx_params const *params, const void *dict, size_t dictSize, ZSTD_dictTableLoadMethod_e dtlm, void *workspace)
static size_t	ZSTD_compress_insertDictionary (ZSTD_compressedBlockState_t *bs, ZSTD_matchState_t *ms, ldmState_t *ls, ZSTD_cwksp *ws, const ZSTD_CCtx_params *params, const void *dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, ZSTD_dictTableLoadMethod_e dtlm, void *workspace)
static size_t	ZSTD_compressBegin_internal (ZSTD_CCtx *cctx, const void *dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, ZSTD_dictTableLoadMethod_e dtlm, const ZSTD_CDict *cdict, const ZSTD_CCtx_params *params, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff)
size_t	ZSTD_compressBegin_advanced_internal (ZSTD_CCtx *cctx, const void *dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, ZSTD_dictTableLoadMethod_e dtlm, const ZSTD_CDict *cdict, const ZSTD_CCtx_params *params, unsigned long long pledgedSrcSize)
size_t	ZSTD_compressBegin_advanced (ZSTD_CCtx *cctx, const void *dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize)
size_t	ZSTD_compressBegin_usingDict (ZSTD_CCtx *cctx, const void *dict, size_t dictSize, int compressionLevel)
size_t	ZSTD_compressBegin (ZSTD_CCtx *cctx, int compressionLevel)
static size_t	ZSTD_writeEpilogue (ZSTD_CCtx *cctx, void *dst, size_t dstCapacity)
size_t	ZSTD_compressEnd (ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
static size_t	ZSTD_compress_internal (ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize, const ZSTD_parameters *params)
size_t	ZSTD_compress_advanced (ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize, ZSTD_parameters params)
size_t	ZSTD_compress_advanced_internal (ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize, const ZSTD_CCtx_params *params)
size_t	ZSTD_compress_usingDict (ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize, int compressionLevel)
size_t	ZSTD_compressCCtx (ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, int compressionLevel)
size_t	ZSTD_compress (void *dst, size_t dstCapacity, const void *src, size_t srcSize, int compressionLevel)
size_t	ZSTD_estimateCDictSize_advanced (size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod)
size_t	ZSTD_estimateCDictSize (size_t dictSize, int compressionLevel)
size_t	ZSTD_sizeof_CDict (const ZSTD_CDict *cdict)
static size_t	ZSTD_initCDict_internal (ZSTD_CDict *cdict, const void *dictBuffer, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, ZSTD_compressionParameters cParams)
ZSTD_CDict *	ZSTD_createCDict_advanced (const void *dictBuffer, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, ZSTD_compressionParameters cParams, ZSTD_customMem customMem)
ZSTD_CDict *	ZSTD_createCDict (const void *dict, size_t dictSize, int compressionLevel)
ZSTD_CDict *	ZSTD_createCDict_byReference (const void *dict, size_t dictSize, int compressionLevel)
size_t	ZSTD_freeCDict (ZSTD_CDict *cdict)
const ZSTD_CDict *	ZSTD_initStaticCDict (void *workspace, size_t workspaceSize, const void *dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, ZSTD_compressionParameters cParams)
ZSTD_compressionParameters	ZSTD_getCParamsFromCDict (const ZSTD_CDict *cdict)
size_t	ZSTD_compressBegin_usingCDict_advanced (ZSTD_CCtx *const cctx, const ZSTD_CDict *const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize)
size_t	ZSTD_compressBegin_usingCDict (ZSTD_CCtx *cctx, const ZSTD_CDict *cdict)
size_t	ZSTD_compress_usingCDict_advanced (ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const ZSTD_CDict *cdict, ZSTD_frameParameters fParams)
size_t	ZSTD_compress_usingCDict (ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const ZSTD_CDict *cdict)
ZSTD_CStream *	ZSTD_createCStream (void)
ZSTD_CStream *	ZSTD_initStaticCStream (void *workspace, size_t workspaceSize)
ZSTD_CStream *	ZSTD_createCStream_advanced (ZSTD_customMem customMem)
size_t	ZSTD_freeCStream (ZSTD_CStream *zcs)
size_t	ZSTD_CStreamInSize (void)
size_t	ZSTD_CStreamOutSize (void)
static size_t	ZSTD_resetCStream_internal (ZSTD_CStream *cctx, const void *const dict, size_t const dictSize, ZSTD_dictContentType_e const dictContentType, const ZSTD_CDict *const cdict, ZSTD_CCtx_params params, unsigned long long const pledgedSrcSize)
size_t	ZSTD_resetCStream (ZSTD_CStream *zcs, unsigned long long pss)
size_t	ZSTD_initCStream_internal (ZSTD_CStream *zcs, const void *dict, size_t dictSize, const ZSTD_CDict *cdict, const ZSTD_CCtx_params *params, unsigned long long pledgedSrcSize)
size_t	ZSTD_initCStream_usingCDict_advanced (ZSTD_CStream *zcs, const ZSTD_CDict *cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize)
size_t	ZSTD_initCStream_usingCDict (ZSTD_CStream *zcs, const ZSTD_CDict *cdict)
size_t	ZSTD_initCStream_advanced (ZSTD_CStream *zcs, const void *dict, size_t dictSize, ZSTD_parameters params, unsigned long long pss)
size_t	ZSTD_initCStream_usingDict (ZSTD_CStream *zcs, const void *dict, size_t dictSize, int compressionLevel)
size_t	ZSTD_initCStream_srcSize (ZSTD_CStream *zcs, int compressionLevel, unsigned long long pss)
size_t	ZSTD_initCStream (ZSTD_CStream *zcs, int compressionLevel)
static size_t	ZSTD_nextInputSizeHint (const ZSTD_CCtx *cctx)
static size_t	ZSTD_compressStream_generic (ZSTD_CStream *zcs, ZSTD_outBuffer *output, ZSTD_inBuffer *input, ZSTD_EndDirective const flushMode)
static size_t	ZSTD_nextInputSizeHint_MTorST (const ZSTD_CCtx *cctx)
size_t	ZSTD_compressStream (ZSTD_CStream *zcs, ZSTD_outBuffer *output, ZSTD_inBuffer *input)
size_t	ZSTD_compressStream2 (ZSTD_CCtx *cctx, ZSTD_outBuffer *output, ZSTD_inBuffer *input, ZSTD_EndDirective endOp)
size_t	ZSTD_compressStream2_simpleArgs (ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, size_t *dstPos, const void *src, size_t srcSize, size_t *srcPos, ZSTD_EndDirective endOp)
size_t	ZSTD_compress2 (ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
size_t	ZSTD_flushStream (ZSTD_CStream *zcs, ZSTD_outBuffer *output)
size_t	ZSTD_endStream (ZSTD_CStream *zcs, ZSTD_outBuffer *output)
int	ZSTD_maxCLevel (void)
int	ZSTD_minCLevel (void)
ZSTD_compressionParameters	ZSTD_getCParams (int compressionLevel, unsigned long long srcSizeHint, size_t dictSize)
ZSTD_parameters	ZSTD_getParams (int compressionLevel, unsigned long long srcSizeHint, size_t dictSize)

Variables
static const size_t	attachDictSizeCutoffs [ZSTD_STRATEGY_MAX+1]
static const ZSTD_compressionParameters	ZSTD_defaultCParameters [4][ZSTD_MAX_CLEVEL+1]

Macro Definition Documentation

BOUNDCHECK

#define BOUNDCHECK	(		cParam,
			val
	)

Value:

    { \
    RETURN_ERROR_IF(!ZSTD_cParam_withinBounds(cParam,val), \
                    parameter_outOfBound, "Param out of bounds"); \
}

Definition at line 439 of file zstd_compress.c.

CLAMP

#define CLAMP	(		cParam,
			val
	)		CLAMP_TYPE(cParam, val, unsigned)

CLAMP_TYPE

#define CLAMP_TYPE	(		cParam,
			val,
			type
	)

Value:

        {                                \
        ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam);         \
        if ((int)val<bounds.lowerBound) val=(type)bounds.lowerBound;      \
        else if ((int)val>bounds.upperBound) val=(type)bounds.upperBound; \
    }

FSE_STATIC_LINKING_ONLY

#define FSE_STATIC_LINKING_ONLY   /* FSE_encodeSymbol */

Definition at line 19 of file zstd_compress.c.

HUF_STATIC_LINKING_ONLY

#define HUF_STATIC_LINKING_ONLY

Definition at line 21 of file zstd_compress.c.

ZSTD_INDEXOVERFLOW_MARGIN

#define ZSTD_INDEXOVERFLOW_MARGIN   (16 MB)

Definition at line 1411 of file zstd_compress.c.

ZSTD_MAX_CLEVEL

#define ZSTD_MAX_CLEVEL   22

Definition at line 4115 of file zstd_compress.c.

ZSTD_ROWSIZE

#define ZSTD_ROWSIZE   16

Definition at line 1848 of file zstd_compress.c.

ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER

#define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER   (6)

Definition at line 3077 of file zstd_compress.c.

ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF

#define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF   (128 KB)

Definition at line 3076 of file zstd_compress.c.

Enumeration Type Documentation

ZSTD_buffered_policy_e

enum ZSTD_buffered_policy_e

Indicates whether this compression proceeds directly from user-provided source buffer to user-provided destination buffer (ZSTDb_not_buffered), or whether the context needs to buffer the input/output (ZSTDb_buffered).

Enumerator
ZSTDb_not_buffered
ZSTDb_buffered

Definition at line 1312 of file zstd_compress.c.

             {
    ZSTDb_not_buffered,
    ZSTDb_buffered
} ZSTD_buffered_policy_e;

ZSTD_buildSeqStore_e

enum ZSTD_buildSeqStore_e

Enumerator
ZSTDbss_compress
ZSTDbss_noCompress

Definition at line 2240 of file zstd_compress.c.

{ ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e;

ZSTD_compResetPolicy_e

enum ZSTD_compResetPolicy_e

Controls, for this matchState reset, whether the tables need to be cleared / prepared for the coming compression (ZSTDcrp_makeClean), or whether the tables can be left unclean (ZSTDcrp_leaveDirty), because we know that a subsequent operation will overwrite the table space anyways (e.g., copying the matchState contents in from a CDict).

Enumerator
ZSTDcrp_makeClean
ZSTDcrp_leaveDirty

Definition at line 1324 of file zstd_compress.c.

             {
    ZSTDcrp_makeClean,
    ZSTDcrp_leaveDirty
} ZSTD_compResetPolicy_e;

ZSTD_indexResetPolicy_e

enum ZSTD_indexResetPolicy_e

Controls, for this matchState reset, whether indexing can continue where it left off (ZSTDirp_continue), or whether it needs to be restarted from zero (ZSTDirp_reset).

Enumerator
ZSTDirp_continue
ZSTDirp_reset

Definition at line 1334 of file zstd_compress.c.

             {
    ZSTDirp_continue,
    ZSTDirp_reset
} ZSTD_indexResetPolicy_e;

ZSTD_resetTarget_e

enum ZSTD_resetTarget_e

Enumerator
ZSTD_resetTarget_CDict
ZSTD_resetTarget_CCtx

Definition at line 1339 of file zstd_compress.c.

             {
    ZSTD_resetTarget_CDict,
    ZSTD_resetTarget_CCtx
} ZSTD_resetTarget_e;

Function Documentation

ZSTD_adjustCParams()

ZSTD_compressionParameters ZSTD_adjustCParams	(	ZSTD_compressionParameters	cPar,
		unsigned long long	srcSize,
		size_t	dictSize
	)

Definition at line 1065 of file zstd_compress.c.

{
    cPar = ZSTD_clampCParams(cPar);   /* resulting cPar is necessarily valid (all parameters within range) */
    if (srcSize == 0) srcSize = ZSTD_CONTENTSIZE_UNKNOWN;
    return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize);
}

ZSTD_adjustCParams_internal()

static ZSTD_compressionParameters ZSTD_adjustCParams_internal ( ZSTD_compressionParameters  cPar, unsigned long long  srcSize, size_t  dictSize  )

static

ZSTD_adjustCParams_internal() : optimize cPar for a specified input (srcSize and dictSize). mostly downsize to reduce memory consumption and initialization latency. srcSize can be ZSTD_CONTENTSIZE_UNKNOWN when not known. note : srcSize==0 means 0! condition : cPar is presumed validated (can be checked using ZSTD_checkCParams()).

Definition at line 1032 of file zstd_compress.c.

{
    static const U64 minSrcSize = 513; /* (1<<9) + 1 */
    static const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1);
    assert(ZSTD_checkCParams(cPar)==0);
 
    if (dictSize && srcSize == ZSTD_CONTENTSIZE_UNKNOWN)
        srcSize = minSrcSize;
 
    /* resize windowLog if input is small enough, to use less memory */
    if ( (srcSize < maxWindowResize)
      && (dictSize < maxWindowResize) )  {
        U32 const tSize = (U32)(srcSize + dictSize);
        static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN;
        U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN :
                            ZSTD_highbit32(tSize-1) + 1;
        if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
    }
    if (cPar.hashLog > cPar.windowLog+1) cPar.hashLog = cPar.windowLog+1;
    {   U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy);
        if (cycleLog > cPar.windowLog)
            cPar.chainLog -= (cycleLog - cPar.windowLog);
    }
 
    if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN)
        cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN;  /* minimum wlog required for valid frame header */
 
    return cPar;
}

Referenced by ZSTD_adjustCParams(), ZSTD_getCParams_internal(), ZSTD_getCParamsFromCCtxParams(), and ZSTD_resetCCtx_byAttachingCDict().

ZSTD_assertEqualCParams()

static void ZSTD_assertEqualCParams ( ZSTD_compressionParameters  cParams1, ZSTD_compressionParameters  cParams2  )

static

Definition at line 1268 of file zstd_compress.c.

{
    (void)cParams1;
    (void)cParams2;
    assert(cParams1.windowLog    == cParams2.windowLog);
    assert(cParams1.chainLog     == cParams2.chainLog);
    assert(cParams1.hashLog      == cParams2.hashLog);
    assert(cParams1.searchLog    == cParams2.searchLog);
    assert(cParams1.minMatch     == cParams2.minMatch);
    assert(cParams1.targetLength == cParams2.targetLength);
    assert(cParams1.strategy     == cParams2.strategy);
}

Referenced by ZSTD_buildSeqStore(), and ZSTD_loadDictionaryContent().

ZSTD_assignParamsToCCtxParams()

static ZSTD_CCtx_params ZSTD_assignParamsToCCtxParams ( const ZSTD_CCtx_params *  cctxParams, const ZSTD_parameters *  params  )

static

Definition at line 257 of file zstd_compress.c.

{
    ZSTD_CCtx_params ret = *cctxParams;
    assert(!ZSTD_checkCParams(params->cParams));
    ret.cParams = params->cParams;
    ret.fParams = params->fParams;
    ret.compressionLevel = ZSTD_CLEVEL_DEFAULT;   /* should not matter, as all cParams are presumed properly defined */
    return ret;
}

Referenced by ZSTD_compress_internal(), ZSTD_compress_usingDict(), ZSTD_compressBegin_advanced(), ZSTD_compressBegin_usingDict(), and ZSTD_initCStream_advanced().

ZSTD_buildSeqStore()

static size_t ZSTD_buildSeqStore ( ZSTD_CCtx *  zc, const void *  src, size_t  srcSize  )

static

Definition at line 2242 of file zstd_compress.c.

{
    ZSTD_matchState_t* const ms = &zc->blockState.matchState;
    DEBUGLOG(5, "ZSTD_buildSeqStore (srcSize=%zu)", srcSize);
    assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
    /* Assert that we have correctly flushed the ctx params into the ms's copy */
    ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams);
    if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) {
        ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.minMatch);
        return ZSTDbss_noCompress; /* don't even attempt compression below a certain srcSize */
    }
    ZSTD_resetSeqStore(&(zc->seqStore));
    /* required for optimal parser to read stats from dictionary */
    ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy;
    /* tell the optimal parser how we expect to compress literals */
    ms->opt.literalCompressionMode = zc->appliedParams.literalCompressionMode;
    /* a gap between an attached dict and the current window is not safe,
     * they must remain adjacent,
     * and when that stops being the case, the dict must be unset */
    assert(ms->dictMatchState == NULL || ms->loadedDictEnd == ms->window.dictLimit);
 
    /* limited update after a very long match */
    {   const BYTE* const base = ms->window.base;
        const BYTE* const istart = (const BYTE*)src;
        const U32 current = (U32)(istart-base);
        if (sizeof(ptrdiff_t)==8) assert(istart - base < (ptrdiff_t)(U32)(-1));   /* ensure no overflow */
        if (current > ms->nextToUpdate + 384)
            ms->nextToUpdate = current - MIN(192, (U32)(current - ms->nextToUpdate - 384));
    }
 
    /* select and store sequences */
    {   ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms);
        size_t lastLLSize;
        {   int i;
            for (i = 0; i < ZSTD_REP_NUM; ++i)
                zc->blockState.nextCBlock->rep[i] = zc->blockState.prevCBlock->rep[i];
        }
        if (zc->externSeqStore.pos < zc->externSeqStore.size) {
            assert(!zc->appliedParams.ldmParams.enableLdm);
            /* Updates ldmSeqStore.pos */
            lastLLSize =
                ZSTD_ldm_blockCompress(&zc->externSeqStore,
                                       ms, &zc->seqStore,
                                       zc->blockState.nextCBlock->rep,
                                       src, srcSize);
            assert(zc->externSeqStore.pos <= zc->externSeqStore.size);
        } else if (zc->appliedParams.ldmParams.enableLdm) {
            rawSeqStore_t ldmSeqStore = {NULL, 0, 0, 0};
 
            ldmSeqStore.seq = zc->ldmSequences;
            ldmSeqStore.capacity = zc->maxNbLdmSequences;
            /* Updates ldmSeqStore.size */
            FORWARD_IF_ERROR(ZSTD_ldm_generateSequences(&zc->ldmState, &ldmSeqStore,
                                               &zc->appliedParams.ldmParams,
                                               src, srcSize), "");
            /* Updates ldmSeqStore.pos */
            lastLLSize =
                ZSTD_ldm_blockCompress(&ldmSeqStore,
                                       ms, &zc->seqStore,
                                       zc->blockState.nextCBlock->rep,
                                       src, srcSize);
            assert(ldmSeqStore.pos == ldmSeqStore.size);
        } else {   /* not long range mode */
            ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, dictMode);
            lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize);
        }
        {   const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize;
            ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize);
    }   }
    return ZSTDbss_compress;
}

Referenced by ZSTD_compressBlock_internal(), and ZSTD_compressBlock_targetCBlockSize().

ZSTD_CCtx_getParameter()

size_t ZSTD_CCtx_getParameter	(	ZSTD_CCtx *	cctx,
		ZSTD_cParameter	param,
		int *	value
	)

Definition at line 714 of file zstd_compress.c.

{
    return ZSTD_CCtxParams_getParameter(&cctx->requestedParams, param, value);
}

ZSTD_CCtx_loadDictionary()

ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary	(	ZSTD_CCtx *	cctx,
		const void *	dict,
		size_t	dictSize
	)

ZSTD_CCtx_loadDictionary() : Create an internal CDict from dict buffer. Decompression will have to use same dictionary.

Returns

: 0, or an error code (which can be tested with ZSTD_isError()). Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, meaning "return to no-dictionary mode". Note 1 : Dictionary is sticky, it will be used for all future compressed frames. To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters). Note 2 : Loading a dictionary involves building tables. It's also a CPU consuming operation, with non-negligible impact on latency. Tables are dependent on compression parameters, and for this reason, compression parameters can no longer be changed after loading a dictionary. Note 3 :dict content will be copied internally. Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. In such a case, dictionary buffer must outlive its users. Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() to precisely select how dictionary content must be interpreted.

Definition at line 924 of file zstd_compress.c.

{
    return ZSTD_CCtx_loadDictionary_advanced(
            cctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto);
}

Referenced by ZSTD_initCStream_advanced(), ZSTD_initCStream_internal(), and ZSTD_initCStream_usingDict().

ZSTD_CCtx_loadDictionary_advanced()

size_t ZSTD_CCtx_loadDictionary_advanced	(	ZSTD_CCtx *	cctx,
		const void *	dict,
		size_t	dictSize,
		ZSTD_dictLoadMethod_e	dictLoadMethod,
		ZSTD_dictContentType_e	dictContentType
	)

Definition at line 891 of file zstd_compress.c.

{
    RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
                    "Can't load a dictionary when ctx is not in init stage.");
    RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
                    "no malloc for static CCtx");
    DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize);
    ZSTD_clearAllDicts(cctx);  /* in case one already exists */
    if (dict == NULL || dictSize == 0)  /* no dictionary mode */
        return 0;
    if (dictLoadMethod == ZSTD_dlm_byRef) {
        cctx->localDict.dict = dict;
    } else {
        void* dictBuffer = ZSTD_malloc(dictSize, cctx->customMem);
        RETURN_ERROR_IF(!dictBuffer, memory_allocation, "NULL pointer!");
        memcpy(dictBuffer, dict, dictSize);
        cctx->localDict.dictBuffer = dictBuffer;
        cctx->localDict.dict = dictBuffer;
    }
    cctx->localDict.dictSize = dictSize;
    cctx->localDict.dictContentType = dictContentType;
    return 0;
}

Referenced by ZSTD_CCtx_loadDictionary(), and ZSTD_CCtx_loadDictionary_byReference().

ZSTD_CCtx_loadDictionary_byReference()

ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference	(	ZSTD_CCtx *	cctx,
		const void *	dict,
		size_t	dictSize
	)

Definition at line 917 of file zstd_compress.c.

{
    return ZSTD_CCtx_loadDictionary_advanced(
            cctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto);
}

ZSTD_CCtx_refCDict()

size_t ZSTD_CCtx_refCDict	(	ZSTD_CCtx *	cctx,
		const ZSTD_CDict *	cdict
	)

ZSTD_CCtx_refCDict() : Reference a prepared dictionary, to be used for all next compressed frames. Note that compression parameters are enforced from within CDict, and supersede any compression parameter previously set within CCtx. The parameters ignored are labled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. The ignored parameters will be used again if the CCtx is returned to no-dictionary mode. The dictionary will remain valid for future compressed frames using same CCtx.

Returns

: 0, or an error code (which can be tested with ZSTD_isError()). Special : Referencing a NULL CDict means "return to no-dictionary mode". Note 1 : Currently, only one dictionary can be managed. Referencing a new dictionary effectively "discards" any previous one. Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx.

Definition at line 931 of file zstd_compress.c.

{
    RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
                    "Can't ref a dict when ctx not in init stage.");
    /* Free the existing local cdict (if any) to save memory. */
    ZSTD_clearAllDicts(cctx);
    cctx->cdict = cdict;
    return 0;
}

Referenced by ZSTD_initCStream(), ZSTD_initCStream_internal(), ZSTD_initCStream_srcSize(), ZSTD_initCStream_usingCDict(), and ZSTD_initCStream_usingCDict_advanced().

ZSTD_CCtx_refPrefix()

size_t ZSTD_CCtx_refPrefix	(	ZSTD_CCtx *	cctx,
		const void *	prefix,
		size_t	prefixSize
	)

ZSTD_CCtx_refPrefix() : Reference a prefix (single-usage dictionary) for next compressed frame. A prefix is only used once. Tables are discarded at end of frame (ZSTD_e_end). Decompression will need same prefix to properly regenerate data. Compressing with a prefix is similar in outcome as performing a diff and compressing it, but performs much faster, especially during decompression (compression speed is tunable with compression level).

Returns

: 0, or an error code (which can be tested with ZSTD_isError()). Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary Note 1 : Prefix buffer is referenced. It must outlive compression. Its content must remain unmodified during compression. Note 2 : If the intention is to diff some large src data blob with some prior version of itself, ensure that the window size is large enough to contain the entire source. See ZSTD_c_windowLog. Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. It's a CPU consuming operation, with non-negligible impact on latency. If there is a need to use the same prefix multiple times, consider loadDictionary instead. Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent). Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation.

Definition at line 941 of file zstd_compress.c.

{
    return ZSTD_CCtx_refPrefix_advanced(cctx, prefix, prefixSize, ZSTD_dct_rawContent);
}

ZSTD_CCtx_refPrefix_advanced()

size_t ZSTD_CCtx_refPrefix_advanced	(	ZSTD_CCtx *	cctx,
		const void *	prefix,
		size_t	prefixSize,
		ZSTD_dictContentType_e	dictContentType
	)

Definition at line 946 of file zstd_compress.c.

{
    RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
                    "Can't ref a prefix when ctx not in init stage.");
    ZSTD_clearAllDicts(cctx);
    if (prefix != NULL && prefixSize > 0) {
        cctx->prefixDict.dict = prefix;
        cctx->prefixDict.dictSize = prefixSize;
        cctx->prefixDict.dictContentType = dictContentType;
    }
    return 0;
}

Referenced by ZSTD_CCtx_refPrefix().

ZSTD_CCtx_reset()

size_t ZSTD_CCtx_reset	(	ZSTD_CCtx *	cctx,
		ZSTD_ResetDirective	reset
	)

ZSTD_CCtx_reset() : Also dumps dictionary

Definition at line 962 of file zstd_compress.c.

{
    if ( (reset == ZSTD_reset_session_only)
      || (reset == ZSTD_reset_session_and_parameters) ) {
        cctx->streamStage = zcss_init;
        cctx->pledgedSrcSizePlusOne = 0;
    }
    if ( (reset == ZSTD_reset_parameters)
      || (reset == ZSTD_reset_session_and_parameters) ) {
        RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
                        "Can't reset parameters only when not in init stage.");
        ZSTD_clearAllDicts(cctx);
        return ZSTD_CCtxParams_reset(&cctx->requestedParams);
    }
    return 0;
}

Referenced by ZSTD_compress2(), ZSTD_compressStream2(), ZSTD_compressStream_generic(), ZSTD_initCCtx(), ZSTD_initCStream(), ZSTD_initCStream_advanced(), ZSTD_initCStream_internal(), ZSTD_initCStream_srcSize(), ZSTD_initCStream_usingCDict(), ZSTD_initCStream_usingCDict_advanced(), ZSTD_initCStream_usingDict(), and ZSTD_resetCStream().

ZSTD_CCtx_setParameter()

size_t ZSTD_CCtx_setParameter	(	ZSTD_CCtx *	cctx,
		ZSTD_cParameter	param,
		int	value
	)

ZSTD_CCtx_setParameter() : Set one compression parameter, selected by enum ZSTD_cParameter. All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds(). Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). Setting a parameter is generally only possible during frame initialization (before starting compression). Exception : when using multi-threading mode (nbWorkers >= 1), the following parameters can be updated during compression (within same frame): => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. new parameters will be active for next job only (after a flush()).

Returns

: an error code (which can be tested using ZSTD_isError()).

Definition at line 482 of file zstd_compress.c.

{
    DEBUGLOG(4, "ZSTD_CCtx_setParameter (%i, %i)", (int)param, value);
    if (cctx->streamStage != zcss_init) {
        if (ZSTD_isUpdateAuthorized(param)) {
            cctx->cParamsChanged = 1;
        } else {
            RETURN_ERROR(stage_wrong, "can only set params in ctx init stage");
    }   }
 
    switch(param)
    {
    case ZSTD_c_nbWorkers:
        RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported,
                        "MT not compatible with static alloc");
        break;
 
    case ZSTD_c_compressionLevel:
    case ZSTD_c_windowLog:
    case ZSTD_c_hashLog:
    case ZSTD_c_chainLog:
    case ZSTD_c_searchLog:
    case ZSTD_c_minMatch:
    case ZSTD_c_targetLength:
    case ZSTD_c_strategy:
    case ZSTD_c_ldmHashRateLog:
    case ZSTD_c_format:
    case ZSTD_c_contentSizeFlag:
    case ZSTD_c_checksumFlag:
    case ZSTD_c_dictIDFlag:
    case ZSTD_c_forceMaxWindow:
    case ZSTD_c_forceAttachDict:
    case ZSTD_c_literalCompressionMode:
    case ZSTD_c_jobSize:
    case ZSTD_c_overlapLog:
    case ZSTD_c_rsyncable:
    case ZSTD_c_enableLongDistanceMatching:
    case ZSTD_c_ldmHashLog:
    case ZSTD_c_ldmMinMatch:
    case ZSTD_c_ldmBucketSizeLog:
    case ZSTD_c_targetCBlockSize:
    case ZSTD_c_srcSizeHint:
        break;
 
    default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
    }
    return ZSTD_CCtxParams_setParameter(&cctx->requestedParams, param, value);
}

Referenced by ZSTD_initCStream(), ZSTD_initCStream_srcSize(), and ZSTD_initCStream_usingDict().

ZSTD_CCtx_setParametersUsingCCtxParams()

size_t ZSTD_CCtx_setParametersUsingCCtxParams	(	ZSTD_CCtx *	cctx,
		const ZSTD_CCtx_params *	params
	)

ZSTD_CCtx_setParametersUsingCCtxParams() : just applies params into cctx no action is performed, parameters are merely stored. If ZSTDMT is enabled, parameters are pushed to cctx->mtctx. This is possible even if a compression is ongoing. In which case, new parameters will be applied on the fly, starting with next compression job.

Definition at line 830 of file zstd_compress.c.

{
    DEBUGLOG(4, "ZSTD_CCtx_setParametersUsingCCtxParams");
    RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
                    "The context is in the wrong stage!");
    RETURN_ERROR_IF(cctx->cdict, stage_wrong,
                    "Can't override parameters with cdict attached (some must "
                    "be inherited from the cdict).");
 
    cctx->requestedParams = *params;
    return 0;
}

ZSTD_CCtx_setPledgedSrcSize()

ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize	(	ZSTD_CCtx *	cctx,
		unsigned long long	pledgedSrcSize
	)

ZSTD_CCtx_setPledgedSrcSize() : Total input data size to be compressed as a single frame. Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag. This value will also be controlled at end of frame, and trigger an error if not respected.

Returns

: 0, or an error code (which can be tested with ZSTD_isError()). Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame. In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame. Note 2 : pledgedSrcSize is only valid once, for the next frame. It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN. Note 3 : Whenever all input data is provided and consumed in a single round, for example with ZSTD_compress2(), or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end), this value is automatically overridden by srcSize instead.

Definition at line 844 of file zstd_compress.c.

{
    DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %u bytes", (U32)pledgedSrcSize);
    RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
                    "Can't set pledgedSrcSize when not in init stage.");
    cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1;
    return 0;
}

Referenced by ZSTD_initCStream_advanced(), ZSTD_initCStream_internal(), ZSTD_initCStream_srcSize(), ZSTD_initCStream_usingCDict_advanced(), and ZSTD_resetCStream().

ZSTD_CCtxParams_getParameter()

size_t ZSTD_CCtxParams_getParameter	(	ZSTD_CCtx_params *	CCtxParams,
		ZSTD_cParameter	param,
		int *	value
	)

Definition at line 719 of file zstd_compress.c.

{
    switch(param)
    {
    case ZSTD_c_format :
        *value = CCtxParams->format;
        break;
    case ZSTD_c_compressionLevel :
        *value = CCtxParams->compressionLevel;
        break;
    case ZSTD_c_windowLog :
        *value = (int)CCtxParams->cParams.windowLog;
        break;
    case ZSTD_c_hashLog :
        *value = (int)CCtxParams->cParams.hashLog;
        break;
    case ZSTD_c_chainLog :
        *value = (int)CCtxParams->cParams.chainLog;
        break;
    case ZSTD_c_searchLog :
        *value = CCtxParams->cParams.searchLog;
        break;
    case ZSTD_c_minMatch :
        *value = CCtxParams->cParams.minMatch;
        break;
    case ZSTD_c_targetLength :
        *value = CCtxParams->cParams.targetLength;
        break;
    case ZSTD_c_strategy :
        *value = (unsigned)CCtxParams->cParams.strategy;
        break;
    case ZSTD_c_contentSizeFlag :
        *value = CCtxParams->fParams.contentSizeFlag;
        break;
    case ZSTD_c_checksumFlag :
        *value = CCtxParams->fParams.checksumFlag;
        break;
    case ZSTD_c_dictIDFlag :
        *value = !CCtxParams->fParams.noDictIDFlag;
        break;
    case ZSTD_c_forceMaxWindow :
        *value = CCtxParams->forceWindow;
        break;
    case ZSTD_c_forceAttachDict :
        *value = CCtxParams->attachDictPref;
        break;
    case ZSTD_c_literalCompressionMode :
        *value = CCtxParams->literalCompressionMode;
        break;
    case ZSTD_c_nbWorkers :
#ifndef ZSTD_MULTITHREAD
        assert(CCtxParams->nbWorkers == 0);
#endif
        *value = CCtxParams->nbWorkers;
        break;
    case ZSTD_c_jobSize :
#ifndef ZSTD_MULTITHREAD
        RETURN_ERROR(parameter_unsupported, "not compiled with multithreading");
#else
        assert(CCtxParams->jobSize <= INT_MAX);
        *value = (int)CCtxParams->jobSize;
        break;
#endif
    case ZSTD_c_overlapLog :
#ifndef ZSTD_MULTITHREAD
        RETURN_ERROR(parameter_unsupported, "not compiled with multithreading");
#else
        *value = CCtxParams->overlapLog;
        break;
#endif
    case ZSTD_c_rsyncable :
#ifndef ZSTD_MULTITHREAD
        RETURN_ERROR(parameter_unsupported, "not compiled with multithreading");
#else
        *value = CCtxParams->rsyncable;
        break;
#endif
    case ZSTD_c_enableLongDistanceMatching :
        *value = CCtxParams->ldmParams.enableLdm;
        break;
    case ZSTD_c_ldmHashLog :
        *value = CCtxParams->ldmParams.hashLog;
        break;
    case ZSTD_c_ldmMinMatch :
        *value = CCtxParams->ldmParams.minMatchLength;
        break;
    case ZSTD_c_ldmBucketSizeLog :
        *value = CCtxParams->ldmParams.bucketSizeLog;
        break;
    case ZSTD_c_ldmHashRateLog :
        *value = CCtxParams->ldmParams.hashRateLog;
        break;
    case ZSTD_c_targetCBlockSize :
        *value = (int)CCtxParams->targetCBlockSize;
        break;
    case ZSTD_c_srcSizeHint :
        *value = (int)CCtxParams->srcSizeHint;
        break;
    default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
    }
    return 0;
}

Referenced by ZSTD_CCtx_getParameter().

ZSTD_CCtxParams_init()

size_t ZSTD_CCtxParams_init	(	ZSTD_CCtx_params *	cctxParams,
		int	compressionLevel
	)

Definition at line 235 of file zstd_compress.c.

                                                                                {
    RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!");
    memset(cctxParams, 0, sizeof(*cctxParams));
    cctxParams->compressionLevel = compressionLevel;
    cctxParams->fParams.contentSizeFlag = 1;
    return 0;
}

Referenced by ZSTD_CCtxParams_reset().

ZSTD_CCtxParams_init_advanced()

size_t ZSTD_CCtxParams_init_advanced	(	ZSTD_CCtx_params *	cctxParams,
		ZSTD_parameters	params
	)

Definition at line 243 of file zstd_compress.c.

{
    RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!");
    FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , "");
    memset(cctxParams, 0, sizeof(*cctxParams));
    assert(!ZSTD_checkCParams(params.cParams));
    cctxParams->cParams = params.cParams;
    cctxParams->fParams = params.fParams;
    cctxParams->compressionLevel = ZSTD_CLEVEL_DEFAULT;   /* should not matter, as all cParams are presumed properly defined */
    return 0;
}

ZSTD_CCtxParams_reset()

size_t ZSTD_CCtxParams_reset

(

ZSTD_CCtx_params *

params

)

Definition at line 230 of file zstd_compress.c.

{
    return ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT);
}

Referenced by ZSTD_CCtx_reset().

ZSTD_CCtxParams_setParameter()

size_t ZSTD_CCtxParams_setParameter	(	ZSTD_CCtx_params *	CCtxParams,
		ZSTD_cParameter	param,
		int	value
	)

Definition at line 531 of file zstd_compress.c.

{
    DEBUGLOG(4, "ZSTD_CCtxParams_setParameter (%i, %i)", (int)param, value);
    switch(param)
    {
    case ZSTD_c_format :
        BOUNDCHECK(ZSTD_c_format, value);
        CCtxParams->format = (ZSTD_format_e)value;
        return (size_t)CCtxParams->format;
 
    case ZSTD_c_compressionLevel : {
        FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), "");
        if (value) {  /* 0 : does not change current level */
            CCtxParams->compressionLevel = value;
        }
        if (CCtxParams->compressionLevel >= 0) return (size_t)CCtxParams->compressionLevel;
        return 0;  /* return type (size_t) cannot represent negative values */
    }
 
    case ZSTD_c_windowLog :
        if (value!=0)   /* 0 => use default */
            BOUNDCHECK(ZSTD_c_windowLog, value);
        CCtxParams->cParams.windowLog = (U32)value;
        return CCtxParams->cParams.windowLog;
 
    case ZSTD_c_hashLog :
        if (value!=0)   /* 0 => use default */
            BOUNDCHECK(ZSTD_c_hashLog, value);
        CCtxParams->cParams.hashLog = (U32)value;
        return CCtxParams->cParams.hashLog;
 
    case ZSTD_c_chainLog :
        if (value!=0)   /* 0 => use default */
            BOUNDCHECK(ZSTD_c_chainLog, value);
        CCtxParams->cParams.chainLog = (U32)value;
        return CCtxParams->cParams.chainLog;
 
    case ZSTD_c_searchLog :
        if (value!=0)   /* 0 => use default */
            BOUNDCHECK(ZSTD_c_searchLog, value);
        CCtxParams->cParams.searchLog = (U32)value;
        return (size_t)value;
 
    case ZSTD_c_minMatch :
        if (value!=0)   /* 0 => use default */
            BOUNDCHECK(ZSTD_c_minMatch, value);
        CCtxParams->cParams.minMatch = value;
        return CCtxParams->cParams.minMatch;
 
    case ZSTD_c_targetLength :
        BOUNDCHECK(ZSTD_c_targetLength, value);
        CCtxParams->cParams.targetLength = value;
        return CCtxParams->cParams.targetLength;
 
    case ZSTD_c_strategy :
        if (value!=0)   /* 0 => use default */
            BOUNDCHECK(ZSTD_c_strategy, value);
        CCtxParams->cParams.strategy = (ZSTD_strategy)value;
        return (size_t)CCtxParams->cParams.strategy;
 
    case ZSTD_c_contentSizeFlag :
        /* Content size written in frame header _when known_ (default:1) */
        DEBUGLOG(4, "set content size flag = %u", (value!=0));
        CCtxParams->fParams.contentSizeFlag = value != 0;
        return CCtxParams->fParams.contentSizeFlag;
 
    case ZSTD_c_checksumFlag :
        /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */
        CCtxParams->fParams.checksumFlag = value != 0;
        return CCtxParams->fParams.checksumFlag;
 
    case ZSTD_c_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */
        DEBUGLOG(4, "set dictIDFlag = %u", (value!=0));
        CCtxParams->fParams.noDictIDFlag = !value;
        return !CCtxParams->fParams.noDictIDFlag;
 
    case ZSTD_c_forceMaxWindow :
        CCtxParams->forceWindow = (value != 0);
        return CCtxParams->forceWindow;
 
    case ZSTD_c_forceAttachDict : {
        const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value;
        BOUNDCHECK(ZSTD_c_forceAttachDict, pref);
        CCtxParams->attachDictPref = pref;
        return CCtxParams->attachDictPref;
    }
 
    case ZSTD_c_literalCompressionMode : {
        const ZSTD_literalCompressionMode_e lcm = (ZSTD_literalCompressionMode_e)value;
        BOUNDCHECK(ZSTD_c_literalCompressionMode, lcm);
        CCtxParams->literalCompressionMode = lcm;
        return CCtxParams->literalCompressionMode;
    }
 
    case ZSTD_c_nbWorkers :
#ifndef ZSTD_MULTITHREAD
        RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading");
        return 0;
#else
        FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), "");
        CCtxParams->nbWorkers = value;
        return CCtxParams->nbWorkers;
#endif
 
    case ZSTD_c_jobSize :
#ifndef ZSTD_MULTITHREAD
        RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading");
        return 0;
#else
        /* Adjust to the minimum non-default value. */
        if (value != 0 && value < ZSTDMT_JOBSIZE_MIN)
            value = ZSTDMT_JOBSIZE_MIN;
        FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), "");
        assert(value >= 0);
        CCtxParams->jobSize = value;
        return CCtxParams->jobSize;
#endif
 
    case ZSTD_c_overlapLog :
#ifndef ZSTD_MULTITHREAD
        RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading");
        return 0;
#else
        FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), "");
        CCtxParams->overlapLog = value;
        return CCtxParams->overlapLog;
#endif
 
    case ZSTD_c_rsyncable :
#ifndef ZSTD_MULTITHREAD
        RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading");
        return 0;
#else
        FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), "");
        CCtxParams->rsyncable = value;
        return CCtxParams->rsyncable;
#endif
 
    case ZSTD_c_enableLongDistanceMatching :
        CCtxParams->ldmParams.enableLdm = (value!=0);
        return CCtxParams->ldmParams.enableLdm;
 
    case ZSTD_c_ldmHashLog :
        if (value!=0)   /* 0 ==> auto */
            BOUNDCHECK(ZSTD_c_ldmHashLog, value);
        CCtxParams->ldmParams.hashLog = value;
        return CCtxParams->ldmParams.hashLog;
 
    case ZSTD_c_ldmMinMatch :
        if (value!=0)   /* 0 ==> default */
            BOUNDCHECK(ZSTD_c_ldmMinMatch, value);
        CCtxParams->ldmParams.minMatchLength = value;
        return CCtxParams->ldmParams.minMatchLength;
 
    case ZSTD_c_ldmBucketSizeLog :
        if (value!=0)   /* 0 ==> default */
            BOUNDCHECK(ZSTD_c_ldmBucketSizeLog, value);
        CCtxParams->ldmParams.bucketSizeLog = value;
        return CCtxParams->ldmParams.bucketSizeLog;
 
    case ZSTD_c_ldmHashRateLog :
        RETURN_ERROR_IF(value > ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN,
                        parameter_outOfBound, "Param out of bounds!");
        CCtxParams->ldmParams.hashRateLog = value;
        return CCtxParams->ldmParams.hashRateLog;
 
    case ZSTD_c_targetCBlockSize :
        if (value!=0)   /* 0 ==> default */
            BOUNDCHECK(ZSTD_c_targetCBlockSize, value);
        CCtxParams->targetCBlockSize = value;
        return CCtxParams->targetCBlockSize;
 
    case ZSTD_c_srcSizeHint :
        if (value!=0)    /* 0 ==> default */
            BOUNDCHECK(ZSTD_c_srcSizeHint, value);
        CCtxParams->srcSizeHint = value;
        return CCtxParams->srcSizeHint;
 
    default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
    }
}

Referenced by ZSTD_CCtx_setParameter().

ZSTD_checkCParams()

size_t ZSTD_checkCParams

(

ZSTD_compressionParameters

cParams

)

ZSTD_checkCParams() : control CParam values remain within authorized range.

Returns

: 0, or an error code if one value is beyond authorized range

Definition at line 983 of file zstd_compress.c.

{
    BOUNDCHECK(ZSTD_c_windowLog, (int)cParams.windowLog);
    BOUNDCHECK(ZSTD_c_chainLog,  (int)cParams.chainLog);
    BOUNDCHECK(ZSTD_c_hashLog,   (int)cParams.hashLog);
    BOUNDCHECK(ZSTD_c_searchLog, (int)cParams.searchLog);
    BOUNDCHECK(ZSTD_c_minMatch,  (int)cParams.minMatch);
    BOUNDCHECK(ZSTD_c_targetLength,(int)cParams.targetLength);
    BOUNDCHECK(ZSTD_c_strategy,  cParams.strategy);
    return 0;
}

Referenced by ZSTD_adjustCParams_internal(), ZSTD_assignParamsToCCtxParams(), ZSTD_CCtxParams_init_advanced(), ZSTD_compress_advanced(), ZSTD_compressBegin_advanced_internal(), ZSTD_compressBegin_internal(), ZSTD_getBlockSize(), ZSTD_getCParamsFromCCtxParams(), ZSTD_initCDict_internal(), ZSTD_initCStream_advanced(), ZSTD_initCStream_internal(), ZSTD_makeCCtxParamsFromCParams(), ZSTD_resetCCtx_internal(), and ZSTD_resetCStream_internal().

ZSTD_checkDictNCount()

static size_t ZSTD_checkDictNCount ( short *  normalizedCounter, unsigned  dictMaxSymbolValue, unsigned  maxSymbolValue  )

static

Definition at line 2893 of file zstd_compress.c.

                                                                                                                   {
    U32 s;
    RETURN_ERROR_IF(dictMaxSymbolValue < maxSymbolValue, dictionary_corrupted, "dict fse tables don't have all symbols");
    for (s = 0; s <= maxSymbolValue; ++s) {
        RETURN_ERROR_IF(normalizedCounter[s] == 0, dictionary_corrupted, "dict fse tables don't have all symbols");
    }
    return 0;
}

Referenced by ZSTD_loadCEntropy(), and ZSTD_loadZstdDictionary().

ZSTD_clampCParams()

static ZSTD_compressionParameters ZSTD_clampCParams ( ZSTD_compressionParameters  cParams )

static

ZSTD_clampCParams() : make CParam values within valid range.

Returns

: valid CParams

Definition at line 999 of file zstd_compress.c.

{
#   define CLAMP_TYPE(cParam, val, type) {                                \
        ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam);         \
        if ((int)val<bounds.lowerBound) val=(type)bounds.lowerBound;      \
        else if ((int)val>bounds.upperBound) val=(type)bounds.upperBound; \
    }
#   define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, unsigned)
    CLAMP(ZSTD_c_windowLog, cParams.windowLog);
    CLAMP(ZSTD_c_chainLog,  cParams.chainLog);
    CLAMP(ZSTD_c_hashLog,   cParams.hashLog);
    CLAMP(ZSTD_c_searchLog, cParams.searchLog);
    CLAMP(ZSTD_c_minMatch,  cParams.minMatch);
    CLAMP(ZSTD_c_targetLength,cParams.targetLength);
    CLAMP_TYPE(ZSTD_c_strategy,cParams.strategy, ZSTD_strategy);
    return cParams;
}

Referenced by ZSTD_adjustCParams().

ZSTD_clearAllDicts()

static void ZSTD_clearAllDicts ( ZSTD_CCtx *  cctx )

static

Clears and frees all of the dictionaries in the CCtx.

Definition at line 120 of file zstd_compress.c.

{
    ZSTD_free(cctx->localDict.dictBuffer, cctx->customMem);
    ZSTD_freeCDict(cctx->localDict.cdict);
    memset(&cctx->localDict, 0, sizeof(cctx->localDict));
    memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict));
    cctx->cdict = NULL;
}

Referenced by ZSTD_CCtx_loadDictionary_advanced(), ZSTD_CCtx_refCDict(), ZSTD_CCtx_refPrefix_advanced(), ZSTD_CCtx_reset(), and ZSTD_freeCCtxContent().

ZSTD_compress()

size_t ZSTD_compress	(	void *	dst,
		size_t	dstCapacity,
		const void *	src,
		size_t	srcSize,
		int	compressionLevel
	)

ZSTD_compress() : Compresses src content as a single zstd compressed frame into already allocated dst. Hint : compression runs faster if dstCapacity >= ZSTD_compressBound(srcSize).

Returns

: compressed size written into dst (<= `dstCapacity), or an error code if it fails (which can be tested using ZSTD_isError()).

Definition at line 3307 of file zstd_compress.c.

{
    size_t result;
    ZSTD_CCtx ctxBody;
    ZSTD_initCCtx(&ctxBody, ZSTD_defaultCMem);
    result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel);
    ZSTD_freeCCtxContent(&ctxBody);   /* can't free ctxBody itself, as it's on stack; free only heap content */
    return result;
}

ZSTD_compress2()

size_t ZSTD_compress2	(	ZSTD_CCtx *	cctx,
		void *	dst,
		size_t	dstCapacity,
		const void *	src,
		size_t	srcSize
	)

ZSTD_compress2() : Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. ZSTD_compress2() always starts a new frame. Should cctx hold data from a previously unfinished frame, everything about it is forgotten.

• Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
• The function is always blocking, returns when compression is completed. Hint : compression runs faster if dstCapacity >= ZSTD_compressBound(srcSize).

  Returns

  : compressed size written into dst (<= `dstCapacity), or an error code if it fails (which can be tested using ZSTD_isError()).

Definition at line 4064 of file zstd_compress.c.

{
    DEBUGLOG(4, "ZSTD_compress2 (srcSize=%u)", (unsigned)srcSize);
    ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
    {   size_t oPos = 0;
        size_t iPos = 0;
        size_t const result = ZSTD_compressStream2_simpleArgs(cctx,
                                        dst, dstCapacity, &oPos,
                                        src, srcSize, &iPos,
                                        ZSTD_e_end);
        FORWARD_IF_ERROR(result, "ZSTD_compressStream2_simpleArgs failed");
        if (result != 0) {  /* compression not completed, due to lack of output space */
            assert(oPos == dstCapacity);
            RETURN_ERROR(dstSize_tooSmall, "");
        }
        assert(iPos == srcSize);   /* all input is expected consumed */
        return oPos;
    }
}

Referenced by ZSTD_getSequences().

ZSTD_compress_advanced()

size_t ZSTD_compress_advanced	(	ZSTD_CCtx *	cctx,
		void *	dst,
		size_t	dstCapacity,
		const void *	src,
		size_t	srcSize,
		const void *	dict,
		size_t	dictSize,
		ZSTD_parameters	params
	)

Definition at line 3254 of file zstd_compress.c.

{
    DEBUGLOG(4, "ZSTD_compress_advanced");
    FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), "");
    return ZSTD_compress_internal(cctx,
                                  dst, dstCapacity,
                                  src, srcSize,
                                  dict, dictSize,
                                  &params);
}

ZSTD_compress_advanced_internal()

size_t ZSTD_compress_advanced_internal	(	ZSTD_CCtx *	cctx,
		void *	dst,
		size_t	dstCapacity,
		const void *	src,
		size_t	srcSize,
		const void *	dict,
		size_t	dictSize,
		const ZSTD_CCtx_params *	params
	)

Definition at line 3270 of file zstd_compress.c.

{
    DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", (unsigned)srcSize);
    FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx,
                         dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL,
                         params, srcSize, ZSTDb_not_buffered) , "");
    return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
}

Referenced by ZSTD_compress_internal(), and ZSTD_compress_usingDict().

ZSTD_compress_frameChunk()

static size_t ZSTD_compress_frameChunk ( ZSTD_CCtx *  cctx, void *  dst, size_t  dstCapacity, const void *  src, size_t  srcSize, U32  lastFrameChunk  )

static

ZSTD_compress_frameChunk() : Compress a chunk of data into one or multiple blocks. All blocks will be terminated, all input will be consumed. Function will issue an error if there is not enough dstCapacity to hold the compressed content. Frame is supposed already started (header already produced)

Returns

: compressed size, or an error code

Definition at line 2582 of file zstd_compress.c.

{
    size_t blockSize = cctx->blockSize;
    size_t remaining = srcSize;
    const BYTE* ip = (const BYTE*)src;
    BYTE* const ostart = (BYTE*)dst;
    BYTE* op = ostart;
    U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog;
 
    assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX);
 
    DEBUGLOG(5, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize);
    if (cctx->appliedParams.fParams.checksumFlag && srcSize)
        XXH64_update(&cctx->xxhState, src, srcSize);
 
    while (remaining) {
        ZSTD_matchState_t* const ms = &cctx->blockState.matchState;
        U32 const lastBlock = lastFrameChunk & (blockSize >= remaining);
 
        RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE,
                        dstSize_tooSmall,
                        "not enough space to store compressed block");
        if (remaining < blockSize) blockSize = remaining;
 
        ZSTD_overflowCorrectIfNeeded(
            ms, &cctx->workspace, &cctx->appliedParams, ip, ip + blockSize);
        ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState);
 
        /* Ensure hash/chain table insertion resumes no sooner than lowlimit */
        if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit;
 
        {   size_t cSize;
            if (ZSTD_useTargetCBlockSize(&cctx->appliedParams)) {
                cSize = ZSTD_compressBlock_targetCBlockSize(cctx, op, dstCapacity, ip, blockSize, lastBlock);
                FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize failed");
                assert(cSize > 0);
                assert(cSize <= blockSize + ZSTD_blockHeaderSize);
            } else {
                cSize = ZSTD_compressBlock_internal(cctx,
                                        op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize,
                                        ip, blockSize, 1 /* frame */);
                FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_internal failed");
 
                if (cSize == 0) {  /* block is not compressible */
                    cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
                    FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed");
                } else {
                    U32 const cBlockHeader = cSize == 1 ?
                        lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) :
                        lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
                    MEM_writeLE24(op, cBlockHeader);
                    cSize += ZSTD_blockHeaderSize;
                }
            }
 
 
            ip += blockSize;
            assert(remaining >= blockSize);
            remaining -= blockSize;
            op += cSize;
            assert(dstCapacity >= cSize);
            dstCapacity -= cSize;
            cctx->isFirstBlock = 0;
            DEBUGLOG(5, "ZSTD_compress_frameChunk: adding a block of size %u",
                        (unsigned)cSize);
    }   }
 
    if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending;
    return (size_t)(op-ostart);
}

Referenced by ZSTD_compressContinue_internal().

ZSTD_compress_insertDictionary()

static size_t ZSTD_compress_insertDictionary ( ZSTD_compressedBlockState_t *  bs, ZSTD_matchState_t *  ms, ldmState_t *  ls, ZSTD_cwksp *  ws, const ZSTD_CCtx_params *  params, const void *  dict, size_t  dictSize, ZSTD_dictContentType_e  dictContentType, ZSTD_dictTableLoadMethod_e  dtlm, void *  workspace  )

static

ZSTD_compress_insertDictionary() :

Returns

: dictID, or an error code

Definition at line 3039 of file zstd_compress.c.

{
    DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize);
    if ((dict==NULL) || (dictSize<8)) {
        RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, "");
        return 0;
    }
 
    ZSTD_reset_compressedBlockState(bs);
 
    /* dict restricted modes */
    if (dictContentType == ZSTD_dct_rawContent)
        return ZSTD_loadDictionaryContent(ms, ls, ws, params, dict, dictSize, dtlm);
 
    if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) {
        if (dictContentType == ZSTD_dct_auto) {
            DEBUGLOG(4, "raw content dictionary detected");
            return ZSTD_loadDictionaryContent(
                ms, ls, ws, params, dict, dictSize, dtlm);
        }
        RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, "");
        assert(0);   /* impossible */
    }
 
    /* dict as full zstd dictionary */
    return ZSTD_loadZstdDictionary(
        bs, ms, ws, params, dict, dictSize, dtlm, workspace);
}

Referenced by ZSTD_compressBegin_internal(), and ZSTD_initCDict_internal().

ZSTD_compress_internal()

static size_t ZSTD_compress_internal ( ZSTD_CCtx *  cctx, void *  dst, size_t  dstCapacity, const void *  src, size_t  srcSize, const void *  dict, size_t  dictSize, const ZSTD_parameters *  params  )

static

Definition at line 3238 of file zstd_compress.c.

{
    ZSTD_CCtx_params const cctxParams =
            ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, params);
    DEBUGLOG(4, "ZSTD_compress_internal");
    return ZSTD_compress_advanced_internal(cctx,
                                           dst, dstCapacity,
                                           src, srcSize,
                                           dict, dictSize,
                                           &cctxParams);
}

Referenced by ZSTD_compress_advanced().

ZSTD_compress_usingCDict()

size_t ZSTD_compress_usingCDict	(	ZSTD_CCtx *	cctx,
		void *	dst,
		size_t	dstCapacity,
		const void *	src,
		size_t	srcSize,
		const ZSTD_CDict *	cdict
	)

ZSTD_compress_usingCDict() : Compression using a digested Dictionary. Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. Note that compression parameters are decided at CDict creation time while frame parameters are hardcoded

Definition at line 3596 of file zstd_compress.c.

{
    ZSTD_frameParameters const fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
    return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, fParams);
}

ZSTD_compress_usingCDict_advanced()

size_t ZSTD_compress_usingCDict_advanced	(	ZSTD_CCtx *	cctx,
		void *	dst,
		size_t	dstCapacity,
		const void *	src,
		size_t	srcSize,
		const ZSTD_CDict *	cdict,
		ZSTD_frameParameters	fParams
	)

Definition at line 3582 of file zstd_compress.c.

{
    FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize), "");   /* will check if cdict != NULL */
    return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
}

Referenced by ZSTD_compress_usingCDict().

ZSTD_compress_usingDict()

size_t ZSTD_compress_usingDict	(	ZSTD_CCtx *	ctx,
		void *	dst,
		size_t	dstCapacity,
		const void *	src,
		size_t	srcSize,
		const void *	dict,
		size_t	dictSize,
		int	compressionLevel
	)

ZSTD_compress_usingDict() : Compression at an explicit compression level using a Dictionary. A dictionary can be any arbitrary data segment (also called a prefix), or a buffer with specified information (see dictBuilder/zdict.h). Note : This function loads the dictionary, resulting in significant startup delay. It's intended for a dictionary used only once. Note 2 : When dict == NULL || dictSize < 8 no dictionary is used.

Definition at line 3284 of file zstd_compress.c.

{
    ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0);
    ZSTD_CCtx_params cctxParams = ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, &params);
    DEBUGLOG(4, "ZSTD_compress_usingDict (srcSize=%u)", (unsigned)srcSize);
    assert(params.fParams.contentSizeFlag == 1);
    return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctxParams);
}

Referenced by ZSTD_compressCCtx().

ZSTD_compressBegin()

size_t ZSTD_compressBegin	(	ZSTD_CCtx *	cctx,
		int	compressionLevel
	)

Definition at line 3164 of file zstd_compress.c.

{
    return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel);
}

ZSTD_compressBegin_advanced()

size_t ZSTD_compressBegin_advanced	(	ZSTD_CCtx *	cctx,
		const void *	dict,
		size_t	dictSize,
		ZSTD_parameters	params,
		unsigned long long	pledgedSrcSize
	)

ZSTD_compressBegin_advanced() :

Returns

: 0, or an error code

Definition at line 3142 of file zstd_compress.c.

{
    ZSTD_CCtx_params const cctxParams =
            ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, &params);
    return ZSTD_compressBegin_advanced_internal(cctx,
                                            dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast,
                                            NULL /*cdict*/,
                                            &cctxParams, pledgedSrcSize);
}

ZSTD_compressBegin_advanced_internal()

size_t ZSTD_compressBegin_advanced_internal	(	ZSTD_CCtx *	cctx,
		const void *	dict,
		size_t	dictSize,
		ZSTD_dictContentType_e	dictContentType,
		ZSTD_dictTableLoadMethod_e	dtlm,
		const ZSTD_CDict *	cdict,
		const ZSTD_CCtx_params *	params,
		unsigned long long	pledgedSrcSize
	)

Definition at line 3122 of file zstd_compress.c.

{
    DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params->cParams.windowLog);
    /* compression parameters verification and optimization */
    FORWARD_IF_ERROR( ZSTD_checkCParams(params->cParams) , "");
    return ZSTD_compressBegin_internal(cctx,
                                       dict, dictSize, dictContentType, dtlm,
                                       cdict,
                                       params, pledgedSrcSize,
                                       ZSTDb_not_buffered);
}

Referenced by ZSTD_compressBegin_advanced().

ZSTD_compressBegin_internal()

static size_t ZSTD_compressBegin_internal ( ZSTD_CCtx *  cctx, const void *  dict, size_t  dictSize, ZSTD_dictContentType_e  dictContentType, ZSTD_dictTableLoadMethod_e  dtlm, const ZSTD_CDict *  cdict, const ZSTD_CCtx_params *  params, U64  pledgedSrcSize, ZSTD_buffered_policy_e  zbuff  )

static

ZSTD_compressBegin_internal() :

Returns

: 0, or an error code

Definition at line 3081 of file zstd_compress.c.

{
    DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params->cParams.windowLog);
    /* params are supposed to be fully validated at this point */
    assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams)));
    assert(!((dict) && (cdict)));  /* either dict or cdict, not both */
    if ( (cdict)
      && (cdict->dictContentSize > 0)
      && ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF
        || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER
        || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN
        || cdict->compressionLevel == 0)
      && (params->attachDictPref != ZSTD_dictForceLoad) ) {
        return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff);
    }
 
    FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, *params, pledgedSrcSize,
                                     ZSTDcrp_makeClean, zbuff) , "");
    {   size_t const dictID = cdict ?
                ZSTD_compress_insertDictionary(
                        cctx->blockState.prevCBlock, &cctx->blockState.matchState,
                        &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent,
                        cdict->dictContentSize, dictContentType, dtlm,
                        cctx->entropyWorkspace)
              : ZSTD_compress_insertDictionary(
                        cctx->blockState.prevCBlock, &cctx->blockState.matchState,
                        &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, dict, dictSize,
                        dictContentType, dtlm, cctx->entropyWorkspace);
        FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed");
        assert(dictID <= UINT_MAX);
        cctx->dictID = (U32)dictID;
    }
    return 0;
}

Referenced by ZSTD_compress_advanced_internal(), ZSTD_compressBegin_advanced_internal(), ZSTD_compressBegin_usingCDict_advanced(), ZSTD_compressBegin_usingDict(), and ZSTD_resetCStream_internal().

ZSTD_compressBegin_usingCDict()

size_t ZSTD_compressBegin_usingCDict	(	ZSTD_CCtx *	cctx,
		const ZSTD_CDict *	cdict
	)

Definition at line 3575 of file zstd_compress.c.

{
    ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
    DEBUGLOG(4, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag);
    return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN);
}

ZSTD_compressBegin_usingCDict_advanced()

size_t ZSTD_compressBegin_usingCDict_advanced	(	ZSTD_CCtx *const	cctx,
		const ZSTD_CDict *const	cdict,
		ZSTD_frameParameters const	fParams,
		unsigned long long const	pledgedSrcSize
	)

Definition at line 3538 of file zstd_compress.c.

{
    DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced");
    RETURN_ERROR_IF(cdict==NULL, dictionary_wrong, "NULL pointer!");
    {   ZSTD_CCtx_params params = cctx->requestedParams;
        params.cParams = ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF
                        || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER
                        || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN
                        || cdict->compressionLevel == 0 )
                      && (params.attachDictPref != ZSTD_dictForceLoad) ?
                ZSTD_getCParamsFromCDict(cdict)
              : ZSTD_getCParams(cdict->compressionLevel,
                                pledgedSrcSize,
                                cdict->dictContentSize);
        /* Increase window log to fit the entire dictionary and source if the
         * source size is known. Limit the increase to 19, which is the
         * window log for compression level 1 with the largest source size.
         */
        if (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN) {
            U32 const limitedSrcSize = (U32)MIN(pledgedSrcSize, 1U << 19);
            U32 const limitedSrcLog = limitedSrcSize > 1 ? ZSTD_highbit32(limitedSrcSize - 1) + 1 : 1;
            params.cParams.windowLog = MAX(params.cParams.windowLog, limitedSrcLog);
        }
        params.fParams = fParams;
        return ZSTD_compressBegin_internal(cctx,
                                           NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast,
                                           cdict,
                                           &params, pledgedSrcSize,
                                           ZSTDb_not_buffered);
    }
}

Referenced by ZSTD_compress_usingCDict_advanced(), and ZSTD_compressBegin_usingCDict().

ZSTD_compressBegin_usingDict()

size_t ZSTD_compressBegin_usingDict	(	ZSTD_CCtx *	cctx,
		const void *	dict,
		size_t	dictSize,
		int	compressionLevel
	)

Definition at line 3154 of file zstd_compress.c.

{
    ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
    ZSTD_CCtx_params const cctxParams =
            ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, &params);
    DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize);
    return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL,
                                       &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered);
}

Referenced by ZSTD_compressBegin().

ZSTD_compressBlock()

size_t ZSTD_compressBlock	(	ZSTD_CCtx *	cctx,
		void *	dst,
		size_t	dstCapacity,
		const void *	src,
		size_t	srcSize
	)

Definition at line 2808 of file zstd_compress.c.

{
    DEBUGLOG(5, "ZSTD_compressBlock: srcSize = %u", (unsigned)srcSize);
    { size_t const blockSizeMax = ZSTD_getBlockSize(cctx);
      RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong, "input is larger than a block"); }
 
    return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */);
}

ZSTD_compressBlock_internal()

static size_t ZSTD_compressBlock_internal ( ZSTD_CCtx *  zc, void *  dst, size_t  dstCapacity, const void *  src, size_t  srcSize, U32  frame  )

static

Definition at line 2414 of file zstd_compress.c.

{
    /* This the upper bound for the length of an rle block.
     * This isn't the actual upper bound. Finding the real threshold
     * needs further investigation.
     */
    const U32 rleMaxLength = 25;
    size_t cSize;
    const BYTE* ip = (const BYTE*)src;
    BYTE* op = (BYTE*)dst;
    DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)",
                (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit,
                (unsigned)zc->blockState.matchState.nextToUpdate);
 
    {   const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize);
        FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed");
        if (bss == ZSTDbss_noCompress) { cSize = 0; goto out; }
    }
 
    if (zc->seqCollector.collectSequences) {
        ZSTD_copyBlockSequences(zc);
        return 0;
    }
 
    /* encode sequences and literals */
    cSize = ZSTD_compressSequences(&zc->seqStore,
            &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy,
            &zc->appliedParams,
            dst, dstCapacity,
            srcSize,
            zc->entropyWorkspace, HUF_WORKSPACE_SIZE /* statically allocated in resetCCtx */,
            zc->bmi2);
 
    if (frame &&
        /* We don't want to emit our first block as a RLE even if it qualifies because
         * doing so will cause the decoder (cli only) to throw a "should consume all input error."
         * This is only an issue for zstd <= v1.4.3
         */
        !zc->isFirstBlock &&
        cSize < rleMaxLength &&
        ZSTD_isRLE(ip, srcSize))
    {
        cSize = 1;
        op[0] = ip[0];
    }
 
out:
    if (!ZSTD_isError(cSize) && cSize > 1) {
        ZSTD_confirmRepcodesAndEntropyTables(zc);
    }
    /* We check that dictionaries have offset codes available for the first
     * block. After the first block, the offcode table might not have large
     * enough codes to represent the offsets in the data.
     */
    if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
        zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
 
    return cSize;
}

Referenced by ZSTD_compress_frameChunk(), and ZSTD_compressContinue_internal().

ZSTD_compressBlock_targetCBlockSize()

static size_t ZSTD_compressBlock_targetCBlockSize ( ZSTD_CCtx *  zc, void *  dst, size_t  dstCapacity, const void *  src, size_t  srcSize, U32  lastBlock  )

static

Definition at line 2531 of file zstd_compress.c.

{
    size_t cSize = 0;
    const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize);
    DEBUGLOG(5, "ZSTD_compressBlock_targetCBlockSize (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u, srcSize=%zu)",
                (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockState.matchState.nextToUpdate, srcSize);
    FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed");
 
    cSize = ZSTD_compressBlock_targetCBlockSize_body(zc, dst, dstCapacity, src, srcSize, bss, lastBlock);
    FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize_body failed");
 
    if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
        zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
 
    return cSize;
}

Referenced by ZSTD_compress_frameChunk().

ZSTD_compressBlock_targetCBlockSize_body()

static size_t ZSTD_compressBlock_targetCBlockSize_body ( ZSTD_CCtx *  zc, void *  dst, size_t  dstCapacity, const void *  src, size_t  srcSize, const size_t  bss, U32  lastBlock  )

static

Definition at line 2476 of file zstd_compress.c.

{
    DEBUGLOG(6, "Attempting ZSTD_compressSuperBlock()");
    if (bss == ZSTDbss_compress) {
        if (/* We don't want to emit our first block as a RLE even if it qualifies because
            * doing so will cause the decoder (cli only) to throw a "should consume all input error."
            * This is only an issue for zstd <= v1.4.3
            */
            !zc->isFirstBlock &&
            ZSTD_maybeRLE(&zc->seqStore) &&
            ZSTD_isRLE((BYTE const*)src, srcSize))
        {
            return ZSTD_rleCompressBlock(dst, dstCapacity, *(BYTE const*)src, srcSize, lastBlock);
        }
        /* Attempt superblock compression.
         *
         * Note that compressed size of ZSTD_compressSuperBlock() is not bound by the
         * standard ZSTD_compressBound(). This is a problem, because even if we have
         * space now, taking an extra byte now could cause us to run out of space later
         * and violate ZSTD_compressBound().
         *
         * Define blockBound(blockSize) = blockSize + ZSTD_blockHeaderSize.
         *
         * In order to respect ZSTD_compressBound() we must attempt to emit a raw
         * uncompressed block in these cases:
         *   * cSize == 0: Return code for an uncompressed block.
         *   * cSize == dstSize_tooSmall: We may have expanded beyond blockBound(srcSize).
         *     ZSTD_noCompressBlock() will return dstSize_tooSmall if we are really out of
         *     output space.
         *   * cSize >= blockBound(srcSize): We have expanded the block too much so
         *     emit an uncompressed block.
         */
        {
            size_t const cSize = ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock);
            if (cSize != ERROR(dstSize_tooSmall)) {
                size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy);
                FORWARD_IF_ERROR(cSize, "ZSTD_compressSuperBlock failed");
                if (cSize != 0 && cSize < maxCSize + ZSTD_blockHeaderSize) {
                    ZSTD_confirmRepcodesAndEntropyTables(zc);
                    return cSize;
                }
            }
        }
    }
 
    DEBUGLOG(6, "Resorting to ZSTD_noCompressBlock()");
    /* Superblock compression failed, attempt to emit a single no compress block.
     * The decoder will be able to stream this block since it is uncompressed.
     */
    return ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock);
}

Referenced by ZSTD_compressBlock_targetCBlockSize().

ZSTD_compressBound()

size_t ZSTD_compressBound

(

size_t

srcSize

)

maximum compressed size in worst case single-pass scenario

Definition at line 44 of file zstd_compress.c.

                                          {
    return ZSTD_COMPRESSBOUND(srcSize);
}

Referenced by ZSTD_compressStream_generic(), ZSTD_CStreamOutSize(), ZSTD_estimateCStreamSize_usingCCtxParams(), ZSTD_getSequences(), and ZSTD_resetCCtx_internal().

ZSTD_compressCCtx()

size_t ZSTD_compressCCtx	(	ZSTD_CCtx *	cctx,
		void *	dst,
		size_t	dstCapacity,
		const void *	src,
		size_t	srcSize,
		int	compressionLevel
	)

ZSTD_compressCCtx() : Same as ZSTD_compress(), using an explicit ZSTD_CCtx. Important : in order to behave similarly to ZSTD_compress(), this function compresses at requested compression level, ignoring any other parameter . If any advanced parameter was set using the advanced API, they will all be reset. Only compressionLevel remains.

Definition at line 3297 of file zstd_compress.c.

{
    DEBUGLOG(4, "ZSTD_compressCCtx (srcSize=%u)", (unsigned)srcSize);
    assert(cctx != NULL);
    return ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, srcSize, NULL, 0, compressionLevel);
}

Referenced by ZSTD_compress().

ZSTD_compressContinue()

size_t ZSTD_compressContinue	(	ZSTD_CCtx *	cctx,
		void *	dst,
		size_t	dstCapacity,
		const void *	src,
		size_t	srcSize
	)

Definition at line 2792 of file zstd_compress.c.

{
    DEBUGLOG(5, "ZSTD_compressContinue (srcSize=%u)", (unsigned)srcSize);
    return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */);
}

Referenced by ZSTD_compressStream_generic().

ZSTD_compressContinue_internal()

static size_t ZSTD_compressContinue_internal ( ZSTD_CCtx *  cctx, void *  dst, size_t  dstCapacity, const void *  src, size_t  srcSize, U32  frame, U32  lastFrameChunk  )

static

Definition at line 2732 of file zstd_compress.c.

{
    ZSTD_matchState_t* const ms = &cctx->blockState.matchState;
    size_t fhSize = 0;
 
    DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u, srcSize: %u",
                cctx->stage, (unsigned)srcSize);
    RETURN_ERROR_IF(cctx->stage==ZSTDcs_created, stage_wrong,
                    "missing init (ZSTD_compressBegin)");
 
    if (frame && (cctx->stage==ZSTDcs_init)) {
        fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams,
                                       cctx->pledgedSrcSizePlusOne-1, cctx->dictID);
        FORWARD_IF_ERROR(fhSize, "ZSTD_writeFrameHeader failed");
        assert(fhSize <= dstCapacity);
        dstCapacity -= fhSize;
        dst = (char*)dst + fhSize;
        cctx->stage = ZSTDcs_ongoing;
    }
 
    if (!srcSize) return fhSize;  /* do not generate an empty block if no input */
 
    if (!ZSTD_window_update(&ms->window, src, srcSize)) {
        ms->nextToUpdate = ms->window.dictLimit;
    }
    if (cctx->appliedParams.ldmParams.enableLdm) {
        ZSTD_window_update(&cctx->ldmState.window, src, srcSize);
    }
 
    if (!frame) {
        /* overflow check and correction for block mode */
        ZSTD_overflowCorrectIfNeeded(
            ms, &cctx->workspace, &cctx->appliedParams,
            src, (BYTE const*)src + srcSize);
    }
 
    DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSize);
    {   size_t const cSize = frame ?
                             ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) :
                             ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize, 0 /* frame */);
        FORWARD_IF_ERROR(cSize, "%s", frame ? "ZSTD_compress_frameChunk failed" : "ZSTD_compressBlock_internal failed");
        cctx->consumedSrcSize += srcSize;
        cctx->producedCSize += (cSize + fhSize);
        assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0));
        if (cctx->pledgedSrcSizePlusOne != 0) {  /* control src size */
            ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1);
            RETURN_ERROR_IF(
                cctx->consumedSrcSize+1 > cctx->pledgedSrcSizePlusOne,
                srcSize_wrong,
                "error : pledgedSrcSize = %u, while realSrcSize >= %u",
                (unsigned)cctx->pledgedSrcSizePlusOne-1,
                (unsigned)cctx->consumedSrcSize);
        }
        return cSize + fhSize;
    }
}

Referenced by ZSTD_compressBlock(), ZSTD_compressContinue(), and ZSTD_compressEnd().

ZSTD_compressEnd()

size_t ZSTD_compressEnd	(	ZSTD_CCtx *	cctx,
		void *	dst,
		size_t	dstCapacity,
		const void *	src,
		size_t	srcSize
	)

Definition at line 3212 of file zstd_compress.c.

{
    size_t endResult;
    size_t const cSize = ZSTD_compressContinue_internal(cctx,
                                dst, dstCapacity, src, srcSize,
                                1 /* frame mode */, 1 /* last chunk */);
    FORWARD_IF_ERROR(cSize, "ZSTD_compressContinue_internal failed");
    endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize);
    FORWARD_IF_ERROR(endResult, "ZSTD_writeEpilogue failed");
    assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0));
    if (cctx->pledgedSrcSizePlusOne != 0) {  /* control src size */
        ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1);
        DEBUGLOG(4, "end of frame : controlling src size");
        RETURN_ERROR_IF(
            cctx->pledgedSrcSizePlusOne != cctx->consumedSrcSize+1,
            srcSize_wrong,
             "error : pledgedSrcSize = %u, while realSrcSize = %u",
            (unsigned)cctx->pledgedSrcSizePlusOne-1,
            (unsigned)cctx->consumedSrcSize);
    }
    return cSize + endResult;
}

Referenced by ZSTD_compress_advanced_internal(), ZSTD_compress_usingCDict_advanced(), and ZSTD_compressStream_generic().

ZSTD_compressSequences()

MEM_STATIC size_t ZSTD_compressSequences	(	seqStore_t *	seqStorePtr,
		const ZSTD_entropyCTables_t *	prevEntropy,
		ZSTD_entropyCTables_t *	nextEntropy,
		const ZSTD_CCtx_params *	cctxParams,
		void *	dst,
		size_t	dstCapacity,
		size_t	srcSize,
		void *	entropyWorkspace,
		size_t	entropyWkspSize,
		int	bmi2
	)

Definition at line 2151 of file zstd_compress.c.

{
    size_t const cSize = ZSTD_compressSequences_internal(
                            seqStorePtr, prevEntropy, nextEntropy, cctxParams,
                            dst, dstCapacity,
                            entropyWorkspace, entropyWkspSize, bmi2);
    if (cSize == 0) return 0;
    /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block.
     * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block.
     */
    if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity))
        return 0;  /* block not compressed */
    FORWARD_IF_ERROR(cSize, "ZSTD_compressSequences_internal failed");
 
    /* Check compressibility */
    {   size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy);
        if (cSize >= maxCSize) return 0;  /* block not compressed */
    }
 
    return cSize;
}

Referenced by ZSTD_compressBlock_internal().

ZSTD_compressSequences_internal()

MEM_STATIC size_t ZSTD_compressSequences_internal	(	seqStore_t *	seqStorePtr,
		const ZSTD_entropyCTables_t *	prevEntropy,
		ZSTD_entropyCTables_t *	nextEntropy,
		const ZSTD_CCtx_params *	cctxParams,
		void *	dst,
		size_t	dstCapacity,
		void *	entropyWorkspace,
		size_t	entropyWkspSize,
		const int	bmi2
	)

Definition at line 1964 of file zstd_compress.c.

{
    const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN;
    ZSTD_strategy const strategy = cctxParams->cParams.strategy;
    unsigned count[MaxSeq+1];
    FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable;
    FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable;
    FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable;
    U32 LLtype, Offtype, MLtype;   /* compressed, raw or rle */
    const seqDef* const sequences = seqStorePtr->sequencesStart;
    const BYTE* const ofCodeTable = seqStorePtr->ofCode;
    const BYTE* const llCodeTable = seqStorePtr->llCode;
    const BYTE* const mlCodeTable = seqStorePtr->mlCode;
    BYTE* const ostart = (BYTE*)dst;
    BYTE* const oend = ostart + dstCapacity;
    BYTE* op = ostart;
    size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
    BYTE* seqHead;
    BYTE* lastNCount = NULL;
 
    DEBUGLOG(5, "ZSTD_compressSequences_internal (nbSeq=%zu)", nbSeq);
    ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog)));
 
    /* Compress literals */
    {   const BYTE* const literals = seqStorePtr->litStart;
        size_t const litSize = (size_t)(seqStorePtr->lit - literals);
        size_t const cSize = ZSTD_compressLiterals(
                                    &prevEntropy->huf, &nextEntropy->huf,
                                    cctxParams->cParams.strategy,
                                    ZSTD_disableLiteralsCompression(cctxParams),
                                    op, dstCapacity,
                                    literals, litSize,
                                    entropyWorkspace, entropyWkspSize,
                                    bmi2);
        FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed");
        assert(cSize <= dstCapacity);
        op += cSize;
    }
 
    /* Sequences Header */
    RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/,
                    dstSize_tooSmall, "Can't fit seq hdr in output buf!");
    if (nbSeq < 128) {
        *op++ = (BYTE)nbSeq;
    } else if (nbSeq < LONGNBSEQ) {
        op[0] = (BYTE)((nbSeq>>8) + 0x80);
        op[1] = (BYTE)nbSeq;
        op+=2;
    } else {
        op[0]=0xFF;
        MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ));
        op+=3;
    }
    assert(op <= oend);
    if (nbSeq==0) {
        /* Copy the old tables over as if we repeated them */
        memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse));
        return (size_t)(op - ostart);
    }
 
    /* seqHead : flags for FSE encoding type */
    seqHead = op++;
    assert(op <= oend);
 
    /* convert length/distances into codes */
    ZSTD_seqToCodes(seqStorePtr);
    /* build CTable for Literal Lengths */
    {   unsigned max = MaxLL;
        size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize);   /* can't fail */
        DEBUGLOG(5, "Building LL table");
        nextEntropy->fse.litlength_repeatMode = prevEntropy->fse.litlength_repeatMode;
        LLtype = ZSTD_selectEncodingType(&nextEntropy->fse.litlength_repeatMode,
                                        count, max, mostFrequent, nbSeq,
                                        LLFSELog, prevEntropy->fse.litlengthCTable,
                                        LL_defaultNorm, LL_defaultNormLog,
                                        ZSTD_defaultAllowed, strategy);
        assert(set_basic < set_compressed && set_rle < set_compressed);
        assert(!(LLtype < set_compressed && nextEntropy->fse.litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
        {   size_t const countSize = ZSTD_buildCTable(
                op, (size_t)(oend - op),
                CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype,
                count, max, llCodeTable, nbSeq,
                LL_defaultNorm, LL_defaultNormLog, MaxLL,
                prevEntropy->fse.litlengthCTable,
                sizeof(prevEntropy->fse.litlengthCTable),
                entropyWorkspace, entropyWkspSize);
            FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for LitLens failed");
            if (LLtype == set_compressed)
                lastNCount = op;
            op += countSize;
            assert(op <= oend);
    }   }
    /* build CTable for Offsets */
    {   unsigned max = MaxOff;
        size_t const mostFrequent = HIST_countFast_wksp(
            count, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize);  /* can't fail */
        /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */
        ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed;
        DEBUGLOG(5, "Building OF table");
        nextEntropy->fse.offcode_repeatMode = prevEntropy->fse.offcode_repeatMode;
        Offtype = ZSTD_selectEncodingType(&nextEntropy->fse.offcode_repeatMode,
                                        count, max, mostFrequent, nbSeq,
                                        OffFSELog, prevEntropy->fse.offcodeCTable,
                                        OF_defaultNorm, OF_defaultNormLog,
                                        defaultPolicy, strategy);
        assert(!(Offtype < set_compressed && nextEntropy->fse.offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */
        {   size_t const countSize = ZSTD_buildCTable(
                op, (size_t)(oend - op),
                CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype,
                count, max, ofCodeTable, nbSeq,
                OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
                prevEntropy->fse.offcodeCTable,
                sizeof(prevEntropy->fse.offcodeCTable),
                entropyWorkspace, entropyWkspSize);
            FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for Offsets failed");
            if (Offtype == set_compressed)
                lastNCount = op;
            op += countSize;
            assert(op <= oend);
    }   }
    /* build CTable for MatchLengths */
    {   unsigned max = MaxML;
        size_t const mostFrequent = HIST_countFast_wksp(
            count, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize);   /* can't fail */
        DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op));
        nextEntropy->fse.matchlength_repeatMode = prevEntropy->fse.matchlength_repeatMode;
        MLtype = ZSTD_selectEncodingType(&nextEntropy->fse.matchlength_repeatMode,
                                        count, max, mostFrequent, nbSeq,
                                        MLFSELog, prevEntropy->fse.matchlengthCTable,
                                        ML_defaultNorm, ML_defaultNormLog,
                                        ZSTD_defaultAllowed, strategy);
        assert(!(MLtype < set_compressed && nextEntropy->fse.matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
        {   size_t const countSize = ZSTD_buildCTable(
                op, (size_t)(oend - op),
                CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype,
                count, max, mlCodeTable, nbSeq,
                ML_defaultNorm, ML_defaultNormLog, MaxML,
                prevEntropy->fse.matchlengthCTable,
                sizeof(prevEntropy->fse.matchlengthCTable),
                entropyWorkspace, entropyWkspSize);
            FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for MatchLengths failed");
            if (MLtype == set_compressed)
                lastNCount = op;
            op += countSize;
            assert(op <= oend);
    }   }
 
    *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
 
    {   size_t const bitstreamSize = ZSTD_encodeSequences(
                                        op, (size_t)(oend - op),
                                        CTable_MatchLength, mlCodeTable,
                                        CTable_OffsetBits, ofCodeTable,
                                        CTable_LitLength, llCodeTable,
                                        sequences, nbSeq,
                                        longOffsets, bmi2);
        FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed");
        op += bitstreamSize;
        assert(op <= oend);
        /* zstd versions <= 1.3.4 mistakenly report corruption when
         * FSE_readNCount() receives a buffer < 4 bytes.
         * Fixed by https://github.com/facebook/zstd/pull/1146.
         * This can happen when the last set_compressed table present is 2
         * bytes and the bitstream is only one byte.
         * In this exceedingly rare case, we will simply emit an uncompressed
         * block, since it isn't worth optimizing.
         */
        if (lastNCount && (op - lastNCount) < 4) {
            /* NCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */
            assert(op - lastNCount == 3);
            DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by "
                        "emitting an uncompressed block.");
            return 0;
        }
    }
 
    DEBUGLOG(5, "compressed block size : %u", (unsigned)(op - ostart));
    return (size_t)(op - ostart);
}

Referenced by ZSTD_compressSequences().

ZSTD_compressStream()

size_t ZSTD_compressStream	(	ZSTD_CStream *	zcs,
		ZSTD_outBuffer *	output,
		ZSTD_inBuffer *	input
	)

Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). NOTE: The return value is different. ZSTD_compressStream() returns a hint for the next read size (if non-zero and not an error). ZSTD_compressStream2() returns the minimum nb of bytes left to flush (if non-zero and not an error).

Definition at line 3956 of file zstd_compress.c.

{
    FORWARD_IF_ERROR( ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue) , "");
    return ZSTD_nextInputSizeHint_MTorST(zcs);
}

Referenced by zstd_compress().

ZSTD_compressStream2()

size_t ZSTD_compressStream2	(	ZSTD_CCtx *	cctx,
		ZSTD_outBuffer *	output,
		ZSTD_inBuffer *	input,
		ZSTD_EndDirective	endOp
	)

ZSTD_compressStream2() : Behaves about the same as ZSTD_compressStream, with additional control on end directive.

• Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
• Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
• output->pos must be <= dstCapacity, input->pos must be <= srcSize
• output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
• When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
• When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available, and then immediately returns, just indicating that there is some data remaining to be flushed. The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
• Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.

• Returns

  provides a minimum amount of data remaining to be flushed from internal buffers or an error code, which can be tested using ZSTD_isError(). if

  != 0, flush is not fully completed, there is still some data left within internal buffers. This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. For ZSTD_e_end,

  == 0 when internal buffers are fully flushed and frame is completed.

• after a ZSTD_e_end directive, if internal buffer is not fully flushed (

  Returns

  != 0), only ZSTD_e_end or ZSTD_e_flush operations are allowed. Before starting a new compression job, or changing compression parameters, it is required to fully flush internal buffers.

Definition at line 3963 of file zstd_compress.c.

{
    DEBUGLOG(5, "ZSTD_compressStream2, endOp=%u ", (unsigned)endOp);
    /* check conditions */
    RETURN_ERROR_IF(output->pos > output->size, GENERIC, "invalid buffer");
    RETURN_ERROR_IF(input->pos  > input->size, GENERIC, "invalid buffer");
    assert(cctx!=NULL);
 
    /* transparent initialization stage */
    if (cctx->streamStage == zcss_init) {
        ZSTD_CCtx_params params = cctx->requestedParams;
        ZSTD_prefixDict const prefixDict = cctx->prefixDict;
        FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */
        memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict));   /* single usage */
        assert(prefixDict.dict==NULL || cctx->cdict==NULL);    /* only one can be set */
        DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage");
        if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = input->size + 1;  /* auto-fix pledgedSrcSize */
        params.cParams = ZSTD_getCParamsFromCCtxParams(
                &cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/);
 
 
#ifdef ZSTD_MULTITHREAD
        if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) {
            params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */
        }
        if (params.nbWorkers > 0) {
            /* mt context creation */
            if (cctx->mtctx == NULL) {
                DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u",
                            params.nbWorkers);
                cctx->mtctx = ZSTDMT_createCCtx_advanced((U32)params.nbWorkers, cctx->customMem);
                RETURN_ERROR_IF(cctx->mtctx == NULL, memory_allocation, "NULL pointer!");
            }
            /* mt compression */
            DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbWorkers=%u", params.nbWorkers);
            FORWARD_IF_ERROR( ZSTDMT_initCStream_internal(
                        cctx->mtctx,
                        prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType,
                        cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) , "");
            cctx->streamStage = zcss_load;
            cctx->appliedParams.nbWorkers = params.nbWorkers;
        } else
#endif
        {   FORWARD_IF_ERROR( ZSTD_resetCStream_internal(cctx,
                            prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType,
                            cctx->cdict,
                            params, cctx->pledgedSrcSizePlusOne-1) , "");
            assert(cctx->streamStage == zcss_load);
            assert(cctx->appliedParams.nbWorkers == 0);
    }   }
    /* end of transparent initialization stage */
 
    /* compression stage */
#ifdef ZSTD_MULTITHREAD
    if (cctx->appliedParams.nbWorkers > 0) {
        int const forceMaxProgress = (endOp == ZSTD_e_flush || endOp == ZSTD_e_end);
        size_t flushMin;
        assert(forceMaxProgress || endOp == ZSTD_e_continue /* Protection for a new flush type */);
        if (cctx->cParamsChanged) {
            ZSTDMT_updateCParams_whileCompressing(cctx->mtctx, &cctx->requestedParams);
            cctx->cParamsChanged = 0;
        }
        do {
            flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp);
            if ( ZSTD_isError(flushMin)
              || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */
                ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
            }
            FORWARD_IF_ERROR(flushMin, "ZSTDMT_compressStream_generic failed");
        } while (forceMaxProgress && flushMin != 0 && output->pos < output->size);
        DEBUGLOG(5, "completed ZSTD_compressStream2 delegating to ZSTDMT_compressStream_generic");
        /* Either we don't require maximum forward progress, we've finished the
         * flush, or we are out of output space.
         */
        assert(!forceMaxProgress || flushMin == 0 || output->pos == output->size);
        return flushMin;
    }
#endif
    FORWARD_IF_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) , "");
    DEBUGLOG(5, "completed ZSTD_compressStream2");
    return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */
}

Referenced by ZSTD_compressStream(), ZSTD_compressStream2_simpleArgs(), ZSTD_endStream(), and ZSTD_flushStream().

ZSTD_compressStream2_simpleArgs()

size_t ZSTD_compressStream2_simpleArgs	(	ZSTD_CCtx *	cctx,
		void *	dst,
		size_t	dstCapacity,
		size_t *	dstPos,
		const void *	src,
		size_t	srcSize,
		size_t *	srcPos,
		ZSTD_EndDirective	endOp
	)

Definition at line 4049 of file zstd_compress.c.

{
    ZSTD_outBuffer output = { dst, dstCapacity, *dstPos };
    ZSTD_inBuffer  input  = { src, srcSize, *srcPos };
    /* ZSTD_compressStream2() will check validity of dstPos and srcPos */
    size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp);
    *dstPos = output.pos;
    *srcPos = input.pos;
    return cErr;
}

Referenced by ZSTD_compress2().

ZSTD_compressStream_generic()

static size_t ZSTD_compressStream_generic ( ZSTD_CStream *  zcs, ZSTD_outBuffer *  output, ZSTD_inBuffer *  input, ZSTD_EndDirective const  flushMode  )

static

ZSTD_compressStream_generic(): internal function for all compressStream() variants non-static, because can be called from zstdmt_compress.c

Returns

: hint size for next input

Definition at line 3803 of file zstd_compress.c.

{
    const char* const istart = (const char*)input->src;
    const char* const iend = input->size != 0 ? istart + input->size : istart;
    const char* ip = input->pos != 0 ? istart + input->pos : istart;
    char* const ostart = (char*)output->dst;
    char* const oend = output->size != 0 ? ostart + output->size : ostart;
    char* op = output->pos != 0 ? ostart + output->pos : ostart;
    U32 someMoreWork = 1;
 
    /* check expectations */
    DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%u", (unsigned)flushMode);
    assert(zcs->inBuff != NULL);
    assert(zcs->inBuffSize > 0);
    assert(zcs->outBuff !=  NULL);
    assert(zcs->outBuffSize > 0);
    assert(output->pos <= output->size);
    assert(input->pos <= input->size);
 
    while (someMoreWork) {
        switch(zcs->streamStage)
        {
        case zcss_init:
            RETURN_ERROR(init_missing, "call ZSTD_initCStream() first!");
 
        case zcss_load:
            if ( (flushMode == ZSTD_e_end)
              && ((size_t)(oend-op) >= ZSTD_compressBound(iend-ip))  /* enough dstCapacity */
              && (zcs->inBuffPos == 0) ) {
                /* shortcut to compression pass directly into output buffer */
                size_t const cSize = ZSTD_compressEnd(zcs,
                                                op, oend-op, ip, iend-ip);
                DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize);
                FORWARD_IF_ERROR(cSize, "ZSTD_compressEnd failed");
                ip = iend;
                op += cSize;
                zcs->frameEnded = 1;
                ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
                someMoreWork = 0; break;
            }
            /* complete loading into inBuffer */
            {   size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos;
                size_t const loaded = ZSTD_limitCopy(
                                        zcs->inBuff + zcs->inBuffPos, toLoad,
                                        ip, iend-ip);
                zcs->inBuffPos += loaded;
                if (loaded != 0)
                    ip += loaded;
                if ( (flushMode == ZSTD_e_continue)
                  && (zcs->inBuffPos < zcs->inBuffTarget) ) {
                    /* not enough input to fill full block : stop here */
                    someMoreWork = 0; break;
                }
                if ( (flushMode == ZSTD_e_flush)
                  && (zcs->inBuffPos == zcs->inToCompress) ) {
                    /* empty */
                    someMoreWork = 0; break;
                }
            }
            /* compress current block (note : this stage cannot be stopped in the middle) */
            DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode);
            {   void* cDst;
                size_t cSize;
                size_t const iSize = zcs->inBuffPos - zcs->inToCompress;
                size_t oSize = oend-op;
                unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend);
                if (oSize >= ZSTD_compressBound(iSize))
                    cDst = op;   /* compress into output buffer, to skip flush stage */
                else
                    cDst = zcs->outBuff, oSize = zcs->outBuffSize;
                cSize = lastBlock ?
                        ZSTD_compressEnd(zcs, cDst, oSize,
                                    zcs->inBuff + zcs->inToCompress, iSize) :
                        ZSTD_compressContinue(zcs, cDst, oSize,
                                    zcs->inBuff + zcs->inToCompress, iSize);
                FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed");
                zcs->frameEnded = lastBlock;
                /* prepare next block */
                zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize;
                if (zcs->inBuffTarget > zcs->inBuffSize)
                    zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize;
                DEBUGLOG(5, "inBuffTarget:%u / inBuffSize:%u",
                         (unsigned)zcs->inBuffTarget, (unsigned)zcs->inBuffSize);
                if (!lastBlock)
                    assert(zcs->inBuffTarget <= zcs->inBuffSize);
                zcs->inToCompress = zcs->inBuffPos;
                if (cDst == op) {  /* no need to flush */
                    op += cSize;
                    if (zcs->frameEnded) {
                        DEBUGLOG(5, "Frame completed directly in outBuffer");
                        someMoreWork = 0;
                        ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
                    }
                    break;
                }
                zcs->outBuffContentSize = cSize;
                zcs->outBuffFlushedSize = 0;
                zcs->streamStage = zcss_flush; /* pass-through to flush stage */
            }
        /* fall-through */
        case zcss_flush:
            DEBUGLOG(5, "flush stage");
            {   size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
                size_t const flushed = ZSTD_limitCopy(op, (size_t)(oend-op),
                            zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
                DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u",
                            (unsigned)toFlush, (unsigned)(oend-op), (unsigned)flushed);
                if (flushed)
                    op += flushed;
                zcs->outBuffFlushedSize += flushed;
                if (toFlush!=flushed) {
                    /* flush not fully completed, presumably because dst is too small */
                    assert(op==oend);
                    someMoreWork = 0;
                    break;
                }
                zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
                if (zcs->frameEnded) {
                    DEBUGLOG(5, "Frame completed on flush");
                    someMoreWork = 0;
                    ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
                    break;
                }
                zcs->streamStage = zcss_load;
                break;
            }
 
        default: /* impossible */
            assert(0);
        }
    }
 
    input->pos = ip - istart;
    output->pos = op - ostart;
    if (zcs->frameEnded) return 0;
    return ZSTD_nextInputSizeHint(zcs);
}

Referenced by ZSTD_compressStream2().

ZSTD_confirmRepcodesAndEntropyTables()

static void ZSTD_confirmRepcodesAndEntropyTables ( ZSTD_CCtx *  zc )

static

Definition at line 2407 of file zstd_compress.c.

{
    ZSTD_compressedBlockState_t* const tmp = zc->blockState.prevCBlock;
    zc->blockState.prevCBlock = zc->blockState.nextCBlock;
    zc->blockState.nextCBlock = tmp;
}

Referenced by ZSTD_compressBlock_internal(), and ZSTD_compressBlock_targetCBlockSize_body().

ZSTD_copyBlockSequences()

static void ZSTD_copyBlockSequences ( ZSTD_CCtx *  zc )

static

Definition at line 2314 of file zstd_compress.c.

{
    const seqStore_t* seqStore = ZSTD_getSeqStore(zc);
    const seqDef* seqs = seqStore->sequencesStart;
    size_t seqsSize = seqStore->sequences - seqs;
 
    ZSTD_Sequence* outSeqs = &zc->seqCollector.seqStart[zc->seqCollector.seqIndex];
    size_t i; size_t position; int repIdx;
 
    assert(zc->seqCollector.seqIndex + 1 < zc->seqCollector.maxSequences);
    for (i = 0, position = 0; i < seqsSize; ++i) {
        outSeqs[i].offset = seqs[i].offset;
        outSeqs[i].litLength = seqs[i].litLength;
        outSeqs[i].matchLength = seqs[i].matchLength + MINMATCH;
 
        if (i == seqStore->longLengthPos) {
            if (seqStore->longLengthID == 1) {
                outSeqs[i].litLength += 0x10000;
            } else if (seqStore->longLengthID == 2) {
                outSeqs[i].matchLength += 0x10000;
            }
        }
 
        if (outSeqs[i].offset <= ZSTD_REP_NUM) {
            outSeqs[i].rep = outSeqs[i].offset;
            repIdx = (unsigned int)i - outSeqs[i].offset;
 
            if (outSeqs[i].litLength == 0) {
                if (outSeqs[i].offset < 3) {
                    --repIdx;
                } else {
                    repIdx = (unsigned int)i - 1;
                }
                ++outSeqs[i].rep;
            }
            assert(repIdx >= -3);
            outSeqs[i].offset = repIdx >= 0 ? outSeqs[repIdx].offset : repStartValue[-repIdx - 1];
            if (outSeqs[i].rep == 4) {
                --outSeqs[i].offset;
            }
        } else {
            outSeqs[i].offset -= ZSTD_REP_NUM;
        }
 
        position += outSeqs[i].litLength;
        outSeqs[i].matchPos = (unsigned int)position;
        position += outSeqs[i].matchLength;
    }
    zc->seqCollector.seqIndex += seqsSize;
}

Referenced by ZSTD_compressBlock_internal().

ZSTD_copyCCtx()

size_t ZSTD_copyCCtx	(	ZSTD_CCtx *	dstCCtx,
		const ZSTD_CCtx *	srcCCtx,
		unsigned long long	pledgedSrcSize
	)

ZSTD_copyCCtx() : Duplicate an existing context srcCCtx into another one dstCCtx. Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). pledgedSrcSize==0 means "unknown".

Returns

: 0, or an error code

Definition at line 1834 of file zstd_compress.c.

{
    ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
    ZSTD_buffered_policy_e const zbuff = (ZSTD_buffered_policy_e)(srcCCtx->inBuffSize>0);
    ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered==1);
    if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
    fParams.contentSizeFlag = (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN);
 
    return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx,
                                fParams, pledgedSrcSize,
                                zbuff);
}

ZSTD_copyCCtx_internal()

static size_t ZSTD_copyCCtx_internal ( ZSTD_CCtx *  dstCCtx, const ZSTD_CCtx *  srcCCtx, ZSTD_frameParameters  fParams, U64  pledgedSrcSize, ZSTD_buffered_policy_e  zbuff  )

static

ZSTD_copyCCtx_internal() : Duplicate an existing context srcCCtx into another one dstCCtx. Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). The "context", in this case, refers to the hash and chain tables, entropy tables, and dictionary references. windowLog value is enforced if != 0, otherwise value is copied from srcCCtx.

Returns

: 0, or an error code

Definition at line 1768 of file zstd_compress.c.

{
    DEBUGLOG(5, "ZSTD_copyCCtx_internal");
    RETURN_ERROR_IF(srcCCtx->stage!=ZSTDcs_init, stage_wrong,
                    "Can't copy a ctx that's not in init stage.");
 
    memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
    {   ZSTD_CCtx_params params = dstCCtx->requestedParams;
        /* Copy only compression parameters related to tables. */
        params.cParams = srcCCtx->appliedParams.cParams;
        params.fParams = fParams;
        ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize,
                                ZSTDcrp_leaveDirty, zbuff);
        assert(dstCCtx->appliedParams.cParams.windowLog == srcCCtx->appliedParams.cParams.windowLog);
        assert(dstCCtx->appliedParams.cParams.strategy == srcCCtx->appliedParams.cParams.strategy);
        assert(dstCCtx->appliedParams.cParams.hashLog == srcCCtx->appliedParams.cParams.hashLog);
        assert(dstCCtx->appliedParams.cParams.chainLog == srcCCtx->appliedParams.cParams.chainLog);
        assert(dstCCtx->blockState.matchState.hashLog3 == srcCCtx->blockState.matchState.hashLog3);
    }
 
    ZSTD_cwksp_mark_tables_dirty(&dstCCtx->workspace);
 
    /* copy tables */
    {   size_t const chainSize = (srcCCtx->appliedParams.cParams.strategy == ZSTD_fast) ? 0 : ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog);
        size_t const hSize =  (size_t)1 << srcCCtx->appliedParams.cParams.hashLog;
        int const h3log = srcCCtx->blockState.matchState.hashLog3;
        size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0;
 
        memcpy(dstCCtx->blockState.matchState.hashTable,
               srcCCtx->blockState.matchState.hashTable,
               hSize * sizeof(U32));
        memcpy(dstCCtx->blockState.matchState.chainTable,
               srcCCtx->blockState.matchState.chainTable,
               chainSize * sizeof(U32));
        memcpy(dstCCtx->blockState.matchState.hashTable3,
               srcCCtx->blockState.matchState.hashTable3,
               h3Size * sizeof(U32));
    }
 
    ZSTD_cwksp_mark_tables_clean(&dstCCtx->workspace);
 
    /* copy dictionary offsets */
    {
        const ZSTD_matchState_t* srcMatchState = &srcCCtx->blockState.matchState;
        ZSTD_matchState_t* dstMatchState = &dstCCtx->blockState.matchState;
        dstMatchState->window       = srcMatchState->window;
        dstMatchState->nextToUpdate = srcMatchState->nextToUpdate;
        dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd;
    }
    dstCCtx->dictID = srcCCtx->dictID;
 
    /* copy block state */
    memcpy(dstCCtx->blockState.prevCBlock, srcCCtx->blockState.prevCBlock, sizeof(*srcCCtx->blockState.prevCBlock));
 
    return 0;
}

Referenced by ZSTD_copyCCtx().

ZSTD_cParam_clampBounds()

static size_t ZSTD_cParam_clampBounds ( ZSTD_cParameter  cParam, int *  value  )

static

Definition at line 430 of file zstd_compress.c.

{
    ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam);
    if (ZSTD_isError(bounds.error)) return bounds.error;
    if (*value < bounds.lowerBound) *value = bounds.lowerBound;
    if (*value > bounds.upperBound) *value = bounds.upperBound;
    return 0;
}

Referenced by ZSTD_CCtxParams_setParameter().

ZSTD_cParam_getBounds()

ZSTD_bounds ZSTD_cParam_getBounds

(

ZSTD_cParameter

cParam

)

ZSTD_cParam_getBounds() : All parameters must belong to an interval with lower and upper bounds, otherwise they will either trigger an error or be automatically clamped.

Returns

: a structure, ZSTD_bounds, which contains

• an error status field, which must be tested using ZSTD_isError()
• lower and upper bounds, both inclusive

Definition at line 268 of file zstd_compress.c.

{
    ZSTD_bounds bounds = { 0, 0, 0 };
 
    switch(param)
    {
    case ZSTD_c_compressionLevel:
        bounds.lowerBound = ZSTD_minCLevel();
        bounds.upperBound = ZSTD_maxCLevel();
        return bounds;
 
    case ZSTD_c_windowLog:
        bounds.lowerBound = ZSTD_WINDOWLOG_MIN;
        bounds.upperBound = ZSTD_WINDOWLOG_MAX;
        return bounds;
 
    case ZSTD_c_hashLog:
        bounds.lowerBound = ZSTD_HASHLOG_MIN;
        bounds.upperBound = ZSTD_HASHLOG_MAX;
        return bounds;
 
    case ZSTD_c_chainLog:
        bounds.lowerBound = ZSTD_CHAINLOG_MIN;
        bounds.upperBound = ZSTD_CHAINLOG_MAX;
        return bounds;
 
    case ZSTD_c_searchLog:
        bounds.lowerBound = ZSTD_SEARCHLOG_MIN;
        bounds.upperBound = ZSTD_SEARCHLOG_MAX;
        return bounds;
 
    case ZSTD_c_minMatch:
        bounds.lowerBound = ZSTD_MINMATCH_MIN;
        bounds.upperBound = ZSTD_MINMATCH_MAX;
        return bounds;
 
    case ZSTD_c_targetLength:
        bounds.lowerBound = ZSTD_TARGETLENGTH_MIN;
        bounds.upperBound = ZSTD_TARGETLENGTH_MAX;
        return bounds;
 
    case ZSTD_c_strategy:
        bounds.lowerBound = ZSTD_STRATEGY_MIN;
        bounds.upperBound = ZSTD_STRATEGY_MAX;
        return bounds;
 
    case ZSTD_c_contentSizeFlag:
        bounds.lowerBound = 0;
        bounds.upperBound = 1;
        return bounds;
 
    case ZSTD_c_checksumFlag:
        bounds.lowerBound = 0;
        bounds.upperBound = 1;
        return bounds;
 
    case ZSTD_c_dictIDFlag:
        bounds.lowerBound = 0;
        bounds.upperBound = 1;
        return bounds;
 
    case ZSTD_c_nbWorkers:
        bounds.lowerBound = 0;
#ifdef ZSTD_MULTITHREAD
        bounds.upperBound = ZSTDMT_NBWORKERS_MAX;
#else
        bounds.upperBound = 0;
#endif
        return bounds;
 
    case ZSTD_c_jobSize:
        bounds.lowerBound = 0;
#ifdef ZSTD_MULTITHREAD
        bounds.upperBound = ZSTDMT_JOBSIZE_MAX;
#else
        bounds.upperBound = 0;
#endif
        return bounds;
 
    case ZSTD_c_overlapLog:
#ifdef ZSTD_MULTITHREAD
        bounds.lowerBound = ZSTD_OVERLAPLOG_MIN;
        bounds.upperBound = ZSTD_OVERLAPLOG_MAX;
#else
        bounds.lowerBound = 0;
        bounds.upperBound = 0;
#endif
        return bounds;
 
    case ZSTD_c_enableLongDistanceMatching:
        bounds.lowerBound = 0;
        bounds.upperBound = 1;
        return bounds;
 
    case ZSTD_c_ldmHashLog:
        bounds.lowerBound = ZSTD_LDM_HASHLOG_MIN;
        bounds.upperBound = ZSTD_LDM_HASHLOG_MAX;
        return bounds;
 
    case ZSTD_c_ldmMinMatch:
        bounds.lowerBound = ZSTD_LDM_MINMATCH_MIN;
        bounds.upperBound = ZSTD_LDM_MINMATCH_MAX;
        return bounds;
 
    case ZSTD_c_ldmBucketSizeLog:
        bounds.lowerBound = ZSTD_LDM_BUCKETSIZELOG_MIN;
        bounds.upperBound = ZSTD_LDM_BUCKETSIZELOG_MAX;
        return bounds;
 
    case ZSTD_c_ldmHashRateLog:
        bounds.lowerBound = ZSTD_LDM_HASHRATELOG_MIN;
        bounds.upperBound = ZSTD_LDM_HASHRATELOG_MAX;
        return bounds;
 
    /* experimental parameters */
    case ZSTD_c_rsyncable:
        bounds.lowerBound = 0;
        bounds.upperBound = 1;
        return bounds;
 
    case ZSTD_c_forceMaxWindow :
        bounds.lowerBound = 0;
        bounds.upperBound = 1;
        return bounds;
 
    case ZSTD_c_format:
        ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless);
        bounds.lowerBound = ZSTD_f_zstd1;
        bounds.upperBound = ZSTD_f_zstd1_magicless;   /* note : how to ensure at compile time that this is the highest value enum ? */
        return bounds;
 
    case ZSTD_c_forceAttachDict:
        ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceCopy);
        bounds.lowerBound = ZSTD_dictDefaultAttach;
        bounds.upperBound = ZSTD_dictForceLoad;       /* note : how to ensure at compile time that this is the highest value enum ? */
        return bounds;
 
    case ZSTD_c_literalCompressionMode:
        ZSTD_STATIC_ASSERT(ZSTD_lcm_auto < ZSTD_lcm_huffman && ZSTD_lcm_huffman < ZSTD_lcm_uncompressed);
        bounds.lowerBound = ZSTD_lcm_auto;
        bounds.upperBound = ZSTD_lcm_uncompressed;
        return bounds;
 
    case ZSTD_c_targetCBlockSize:
        bounds.lowerBound = ZSTD_TARGETCBLOCKSIZE_MIN;
        bounds.upperBound = ZSTD_TARGETCBLOCKSIZE_MAX;
        return bounds;
 
    case ZSTD_c_srcSizeHint:
        bounds.lowerBound = ZSTD_SRCSIZEHINT_MIN;
        bounds.upperBound = ZSTD_SRCSIZEHINT_MAX;
        return bounds;
 
    default:
        bounds.error = ERROR(parameter_unsupported);
        return bounds;
    }
}

Referenced by ZSTD_cParam_clampBounds().

ZSTD_createCCtx()

ZSTD_CCtx * ZSTD_createCCtx

(

void

)

Definition at line 64 of file zstd_compress.c.

{
    return ZSTD_createCCtx_advanced(ZSTD_defaultCMem);
}

ZSTD_createCCtx_advanced()

ZSTD_CCtx * ZSTD_createCCtx_advanced

(

ZSTD_customMem

customMem

)

Definition at line 81 of file zstd_compress.c.

{
    ZSTD_STATIC_ASSERT(zcss_init==0);
    ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1));
    if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
    {   ZSTD_CCtx* const cctx = (ZSTD_CCtx*)ZSTD_malloc(sizeof(ZSTD_CCtx), customMem);
        if (!cctx) return NULL;
        ZSTD_initCCtx(cctx, customMem);
        return cctx;
    }
}

Referenced by ZSTD_createCCtx(), and ZSTD_createCStream_advanced().

ZSTD_createCCtxParams()

ZSTD_CCtx_params * ZSTD_createCCtxParams

(

void

)

Definition at line 218 of file zstd_compress.c.

{
    return ZSTD_createCCtxParams_advanced(ZSTD_defaultCMem);
}

ZSTD_createCCtxParams_advanced()

static ZSTD_CCtx_params * ZSTD_createCCtxParams_advanced ( ZSTD_customMem  customMem )

static

Definition at line 204 of file zstd_compress.c.

{
    ZSTD_CCtx_params* params;
    if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
    params = (ZSTD_CCtx_params*)ZSTD_calloc(
            sizeof(ZSTD_CCtx_params), customMem);
    if (!params) { return NULL; }
    params->customMem = customMem;
    params->compressionLevel = ZSTD_CLEVEL_DEFAULT;
    params->fParams.contentSizeFlag = 1;
    return params;
}

Referenced by ZSTD_createCCtxParams().

ZSTD_createCDict()

ZSTD_CDict * ZSTD_createCDict	(	const void *	dictBuffer,
		size_t	dictSize,
		int	compressionLevel
	)

ZSTD_createCDict() : When compressing multiple messages or blocks using the same dictionary, it's recommended to digest the dictionary only once, since it's a costly operation. ZSTD_createCDict() will create a state from digesting a dictionary. The resulting state can be used for future compression operations with very limited startup cost. ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict. Note 1 : Consider experimental function ZSTD_createCDict_byReference() if you prefer to not duplicate @dictBuffer content. Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer, in which case the only thing that it transports is the @compressionLevel. This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively, expecting a ZSTD_CDict parameter with any data, including those without a known dictionary.

Definition at line 3447 of file zstd_compress.c.

{
    ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
    ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dict, dictSize,
                                                  ZSTD_dlm_byCopy, ZSTD_dct_auto,
                                                  cParams, ZSTD_defaultCMem);
    if (cdict)
        cdict->compressionLevel = compressionLevel == 0 ? ZSTD_CLEVEL_DEFAULT : compressionLevel;
    return cdict;
}

ZSTD_createCDict_advanced()

ZSTD_CDict * ZSTD_createCDict_advanced	(	const void *	dictBuffer,
		size_t	dictSize,
		ZSTD_dictLoadMethod_e	dictLoadMethod,
		ZSTD_dictContentType_e	dictContentType,
		ZSTD_compressionParameters	cParams,
		ZSTD_customMem	customMem
	)

Definition at line 3404 of file zstd_compress.c.

{
    DEBUGLOG(3, "ZSTD_createCDict_advanced, mode %u", (unsigned)dictContentType);
    if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
 
    {   size_t const workspaceSize =
            ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) +
            ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) +
            ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) +
            (dictLoadMethod == ZSTD_dlm_byRef ? 0
             : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*))));
        void* const workspace = ZSTD_malloc(workspaceSize, customMem);
        ZSTD_cwksp ws;
        ZSTD_CDict* cdict;
 
        if (!workspace) {
            ZSTD_free(workspace, customMem);
            return NULL;
        }
 
        ZSTD_cwksp_init(&ws, workspace, workspaceSize);
 
        cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict));
        assert(cdict != NULL);
        ZSTD_cwksp_move(&cdict->workspace, &ws);
        cdict->customMem = customMem;
        cdict->compressionLevel = 0; /* signals advanced API usage */
 
        if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
                                        dictBuffer, dictSize,
                                        dictLoadMethod, dictContentType,
                                        cParams) )) {
            ZSTD_freeCDict(cdict);
            return NULL;
        }
 
        return cdict;
    }
}

Referenced by ZSTD_createCDict(), ZSTD_createCDict_byReference(), and ZSTD_initLocalDict().

ZSTD_createCDict_byReference()

ZSTD_CDict * ZSTD_createCDict_byReference	(	const void *	dict,
		size_t	dictSize,
		int	compressionLevel
	)

Definition at line 3458 of file zstd_compress.c.

{
    ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
    return ZSTD_createCDict_advanced(dict, dictSize,
                                     ZSTD_dlm_byRef, ZSTD_dct_auto,
                                     cParams, ZSTD_defaultCMem);
}

ZSTD_createCStream()

ZSTD_CStream * ZSTD_createCStream

(

void

)

Definition at line 3611 of file zstd_compress.c.

{
    DEBUGLOG(3, "ZSTD_createCStream");
    return ZSTD_createCStream_advanced(ZSTD_defaultCMem);
}

ZSTD_createCStream_advanced()

ZSTD_CStream * ZSTD_createCStream_advanced

(

ZSTD_customMem

customMem

)

Definition at line 3622 of file zstd_compress.c.

{   /* CStream and CCtx are now same object */
    return ZSTD_createCCtx_advanced(customMem);
}

Referenced by zstd_compress(), and ZSTD_createCStream().

ZSTD_CStreamInSize()

size_t ZSTD_CStreamInSize

(

void

)

recommended size for input buffer

Definition at line 3636 of file zstd_compress.c.

{ return ZSTD_BLOCKSIZE_MAX; }

ZSTD_CStreamOutSize()

size_t ZSTD_CStreamOutSize

(

void

)

recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block.

Definition at line 3638 of file zstd_compress.c.

{
    return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ;
}

ZSTD_cycleLog()

U32 ZSTD_cycleLog	(	U32	hashLog,
		ZSTD_strategy	strat
	)

ZSTD_cycleLog() : condition for correct operation : hashLog > 1

Definition at line 1019 of file zstd_compress.c.

{
    U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2);
    return hashLog - btScale;
}

Referenced by ZSTD_adjustCParams_internal(), and ZSTD_overflowCorrectIfNeeded().

ZSTD_endStream()

size_t ZSTD_endStream	(	ZSTD_CStream *	zcs,
		ZSTD_outBuffer *	output
	)

Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end).

Definition at line 4097 of file zstd_compress.c.

{
    ZSTD_inBuffer input = { NULL, 0, 0 };
    size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end);
    FORWARD_IF_ERROR( remainingToFlush , "ZSTD_compressStream2 failed");
    if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush;   /* minimal estimation */
    /* single thread mode : attempt to calculate remaining to flush more precisely */
    {   size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE;
        size_t const checksumSize = (size_t)(zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4);
        size_t const toFlush = remainingToFlush + lastBlockSize + checksumSize;
        DEBUGLOG(4, "ZSTD_endStream : remaining to flush : %u", (unsigned)toFlush);
        return toFlush;
    }
}

Referenced by zstd_compress().

ZSTD_estimateCCtxSize()

size_t ZSTD_estimateCCtxSize

(

int

compressionLevel

)

Definition at line 1179 of file zstd_compress.c.

{
    int level;
    size_t memBudget = 0;
    for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) {
        size_t const newMB = ZSTD_estimateCCtxSize_internal(level);
        if (newMB > memBudget) memBudget = newMB;
    }
    return memBudget;
}

ZSTD_estimateCCtxSize_internal()

static size_t ZSTD_estimateCCtxSize_internal ( int  compressionLevel )

static

Definition at line 1173 of file zstd_compress.c.

{
    ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0);
    return ZSTD_estimateCCtxSize_usingCParams(cParams);
}

Referenced by ZSTD_estimateCCtxSize().

ZSTD_estimateCCtxSize_usingCCtxParams()

size_t ZSTD_estimateCCtxSize_usingCCtxParams

(

const ZSTD_CCtx_params *

params

)

Definition at line 1126 of file zstd_compress.c.

{
    RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only.");
    {   ZSTD_compressionParameters const cParams =
                ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0);
        size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
        U32    const divider = (cParams.minMatch==3) ? 3 : 4;
        size_t const maxNbSeq = blockSize / divider;
        size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
                                + ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(seqDef))
                                + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE));
        size_t const entropySpace = ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE);
        size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t));
        size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 1);
 
        size_t const ldmSpace = ZSTD_ldm_getTableSize(params->ldmParams);
        size_t const ldmSeqSpace = ZSTD_cwksp_alloc_size(ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize) * sizeof(rawSeq));
 
        /* estimateCCtxSize is for one-shot compression. So no buffers should
         * be needed. However, we still allocate two 0-sized buffers, which can
         * take space under ASAN. */
        size_t const bufferSpace = ZSTD_cwksp_alloc_size(0)
                                 + ZSTD_cwksp_alloc_size(0);
 
        size_t const cctxSpace = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx));
 
        size_t const neededSpace =
            cctxSpace +
            entropySpace +
            blockStateSpace +
            ldmSpace +
            ldmSeqSpace +
            matchStateSize +
            tokenSpace +
            bufferSpace;
 
        DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace);
        return neededSpace;
    }
}

Referenced by ZSTD_estimateCCtxSize_usingCParams(), and ZSTD_estimateCStreamSize_usingCCtxParams().

ZSTD_estimateCCtxSize_usingCParams()

size_t ZSTD_estimateCCtxSize_usingCParams

(

ZSTD_compressionParameters

cParams

)

Definition at line 1167 of file zstd_compress.c.

{
    ZSTD_CCtx_params const params = ZSTD_makeCCtxParamsFromCParams(cParams);
    return ZSTD_estimateCCtxSize_usingCCtxParams(&params);
}

Referenced by ZSTD_estimateCCtxSize_internal().

ZSTD_estimateCDictSize()

size_t ZSTD_estimateCDictSize	(	size_t	dictSize,
		int	compressionLevel
	)

Definition at line 3336 of file zstd_compress.c.

{
    ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
    return ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy);
}

ZSTD_estimateCDictSize_advanced()

size_t ZSTD_estimateCDictSize_advanced	(	size_t	dictSize,
		ZSTD_compressionParameters	cParams,
		ZSTD_dictLoadMethod_e	dictLoadMethod
	)

ZSTD_estimateCDictSize_advanced() : Estimate amount of memory that will be needed to create a dictionary with following arguments

Definition at line 3324 of file zstd_compress.c.

{
    DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict));
    return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict))
         + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE)
         + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0)
         + (dictLoadMethod == ZSTD_dlm_byRef ? 0
            : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void *))));
}

Referenced by ZSTD_estimateCDictSize().

ZSTD_estimateCStreamSize()

size_t ZSTD_estimateCStreamSize

(

int

compressionLevel

)

Definition at line 1218 of file zstd_compress.c.

{
    int level;
    size_t memBudget = 0;
    for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) {
        size_t const newMB = ZSTD_estimateCStreamSize_internal(level);
        if (newMB > memBudget) memBudget = newMB;
    }
    return memBudget;
}

ZSTD_estimateCStreamSize_internal()

static size_t ZSTD_estimateCStreamSize_internal ( int  compressionLevel )

static

Definition at line 1212 of file zstd_compress.c.

{
    ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0);
    return ZSTD_estimateCStreamSize_usingCParams(cParams);
}

Referenced by ZSTD_estimateCStreamSize().

ZSTD_estimateCStreamSize_usingCCtxParams()

size_t ZSTD_estimateCStreamSize_usingCCtxParams

(

const ZSTD_CCtx_params *

params

)

Definition at line 1190 of file zstd_compress.c.

{
    RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only.");
    {   ZSTD_compressionParameters const cParams =
                ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0);
        size_t const CCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params);
        size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
        size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize;
        size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
        size_t const streamingSize = ZSTD_cwksp_alloc_size(inBuffSize)
                                   + ZSTD_cwksp_alloc_size(outBuffSize);
 
        return CCtxSize + streamingSize;
    }
}

Referenced by ZSTD_estimateCStreamSize_usingCParams().

ZSTD_estimateCStreamSize_usingCParams()

size_t ZSTD_estimateCStreamSize_usingCParams

(

ZSTD_compressionParameters

cParams

)

Definition at line 1206 of file zstd_compress.c.

{
    ZSTD_CCtx_params const params = ZSTD_makeCCtxParamsFromCParams(cParams);
    return ZSTD_estimateCStreamSize_usingCCtxParams(&params);
}

Referenced by ZSTD_estimateCStreamSize_internal().

ZSTD_flushStream()

size_t ZSTD_flushStream	(	ZSTD_CStream *	zcs,
		ZSTD_outBuffer *	output
	)

ZSTD_flushStream() :

Returns

: amount of data remaining to flush

Definition at line 4090 of file zstd_compress.c.

{
    ZSTD_inBuffer input = { NULL, 0, 0 };
    return ZSTD_compressStream2(zcs, output, &input, ZSTD_e_flush);
}

ZSTD_freeCCtx()

size_t ZSTD_freeCCtx

(

ZSTD_CCtx *

cctx

)

Definition at line 147 of file zstd_compress.c.

{
    if (cctx==NULL) return 0;   /* support free on NULL */
    RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
                    "not compatible with static CCtx");
    {
        int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx);
        ZSTD_freeCCtxContent(cctx);
        if (!cctxInWorkspace) {
            ZSTD_free(cctx, cctx->customMem);
        }
    }
    return 0;
}

Referenced by ZSTD_freeCStream().

ZSTD_freeCCtxContent()

static void ZSTD_freeCCtxContent ( ZSTD_CCtx *  cctx )

static

Definition at line 136 of file zstd_compress.c.

{
    assert(cctx != NULL);
    assert(cctx->staticSize == 0);
    ZSTD_clearAllDicts(cctx);
#ifdef ZSTD_MULTITHREAD
    ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL;
#endif
    ZSTD_cwksp_free(&cctx->workspace, cctx->customMem);
}

Referenced by ZSTD_compress(), and ZSTD_freeCCtx().

ZSTD_freeCCtxParams()

size_t ZSTD_freeCCtxParams

(

ZSTD_CCtx_params *

params

)

Definition at line 223 of file zstd_compress.c.

{
    if (params == NULL) { return 0; }
    ZSTD_free(params, params->customMem);
    return 0;
}

ZSTD_freeCDict()

size_t ZSTD_freeCDict

(

ZSTD_CDict *

CDict

)

ZSTD_freeCDict() : Function frees memory allocated by ZSTD_createCDict().

Definition at line 3466 of file zstd_compress.c.

{
    if (cdict==NULL) return 0;   /* support free on NULL */
    {   ZSTD_customMem const cMem = cdict->customMem;
        int cdictInWorkspace = ZSTD_cwksp_owns_buffer(&cdict->workspace, cdict);
        ZSTD_cwksp_free(&cdict->workspace, cMem);
        if (!cdictInWorkspace) {
            ZSTD_free(cdict, cMem);
        }
        return 0;
    }
}

Referenced by ZSTD_clearAllDicts(), and ZSTD_createCDict_advanced().

ZSTD_freeCStream()

size_t ZSTD_freeCStream

(

ZSTD_CStream *

zcs

)

Definition at line 3627 of file zstd_compress.c.

{
    return ZSTD_freeCCtx(zcs);   /* same object */
}

Referenced by zstd_compress().

ZSTD_getBlockSize()

size_t ZSTD_getBlockSize

(

const ZSTD_CCtx *

cctx

)

Definition at line 2801 of file zstd_compress.c.

{
    ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams;
    assert(!ZSTD_checkCParams(cParams));
    return MIN (ZSTD_BLOCKSIZE_MAX, (U32)1 << cParams.windowLog);
}

Referenced by