15 #ifndef ZSTD_COMPRESS_H 16 #define ZSTD_COMPRESS_H 22 #ifdef ZSTD_MULTITHREAD 23 # include "zstdmt_compress.h" 26 #if defined (__cplusplus) 34 #define kSearchStrength 8 35 #define HASH_READ_SIZE 8 36 #define ZSTD_DUBT_UNSORTED_MARK 1 64 U32 CTable[HUF_CTABLE_SIZE_U32(255)];
256 #ifdef ZSTD_MULTITHREAD 268 void const*
src,
size_t srcSize);
274 static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7,
275 8, 9, 10, 11, 12, 13, 14, 15,
276 16, 16, 17, 17, 18, 18, 19, 19,
277 20, 20, 20, 20, 21, 21, 21, 21,
278 22, 22, 22, 22, 22, 22, 22, 22,
279 23, 23, 23, 23, 23, 23, 23, 23,
280 24, 24, 24, 24, 24, 24, 24, 24,
281 24, 24, 24, 24, 24, 24, 24, 24 };
282 static const U32 LL_deltaCode = 19;
283 return (litLength > 63) ?
ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
291 static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
292 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
293 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
294 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39,
295 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
296 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
297 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
298 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
299 static const U32 ML_deltaCode = 36;
300 return (mlBase > 127) ?
ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase];
310 #if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6) 312 if (g_start==
NULL) g_start = (
const BYTE*)literals;
314 DEBUGLOG(6,
"Cpos%7u :%3u literals, match%4u bytes at offCode%7u",
321 assert(seqStorePtr->
lit + litLength <= seqStorePtr->litStart + seqStorePtr->
maxNbLit);
323 seqStorePtr->
lit += litLength;
326 if (litLength>0xFFFF) {
355 # if defined(_MSC_VER) && defined(_WIN64) 357 _BitScanForward64( &
r, (
U64)
val );
358 return (
unsigned)(
r>>3);
359 # elif defined(__GNUC__) && (__GNUC__ >= 4) 360 return (__builtin_ctzll((
U64)
val) >> 3);
362 static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
363 0, 3, 1, 3, 1, 4, 2, 7,
364 0, 2, 3, 6, 1, 5, 3, 5,
365 1, 3, 4, 4, 2, 5, 6, 7,
366 7, 0, 1, 2, 3, 3, 4, 6,
367 2, 6, 5, 5, 3, 4, 5, 6,
368 7, 1, 2, 4, 6, 4, 4, 5,
369 7, 2, 6, 5, 7, 6, 7, 7 };
370 return DeBruijnBytePos[((
U64)((
val & -(
long long)
val) * 0x0218A392CDABBD3FULL)) >> 58];
373 # if defined(_MSC_VER) 376 return (
unsigned)(
r>>3);
377 # elif defined(__GNUC__) && (__GNUC__ >= 3) 378 return (__builtin_ctz((
U32)
val) >> 3);
380 static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
381 3, 2, 2, 1, 3, 2, 0, 1,
382 3, 3, 1, 2, 2, 2, 2, 0,
383 3, 1, 2, 0, 1, 0, 1, 1 };
384 return DeBruijnBytePos[((
U32)((
val & -(
S32)
val) * 0x077CB531U)) >> 27];
389 # if defined(_MSC_VER) && defined(_WIN64) 391 _BitScanReverse64( &
r,
val );
392 return (
unsigned)(
r>>3);
393 # elif defined(__GNUC__) && (__GNUC__ >= 4) 394 return (__builtin_clzll(
val) >> 3);
397 const unsigned n32 =
sizeof(
size_t)*4;
398 if (!(
val>>n32)) {
r=4; }
else {
r=0;
val>>=n32; }
399 if (!(
val>>16)) {
r+=2;
val>>=8; }
else {
val>>=24; }
404 # if defined(_MSC_VER) 407 return (
unsigned)(
r>>3);
408 # elif defined(__GNUC__) && (__GNUC__ >= 3) 409 return (__builtin_clz((
U32)
val) >> 3);
412 if (!(
val>>16)) {
r=2;
val>>=8; }
else {
r=0;
val>>=24; }
422 const BYTE*
const pStart = pIn;
423 const BYTE*
const pInLoopLimit = pInLimit - (
sizeof(
size_t)-1);
425 if (pIn < pInLoopLimit) {
429 while (pIn < pInLoopLimit) {
431 if (!diff) { pIn+=
sizeof(
size_t); pMatch+=
sizeof(
size_t);
continue; }
433 return (
size_t)(pIn - pStart);
437 if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
438 return (
size_t)(pIn - pStart);
451 if (
match + matchLength != mEnd)
return matchLength;
452 DEBUGLOG(7,
"ZSTD_count_2segments: found a 2-parts match (current length==%zu)", matchLength);
453 DEBUGLOG(7,
"distance from match beginning to end dictionary = %zi", mEnd -
match);
454 DEBUGLOG(7,
"distance from current pos to end buffer = %zi", iEnd -
ip);
455 DEBUGLOG(7,
"next byte : ip==%02X, istart==%02X",
ip[matchLength], *
iStart);
505 #define ZSTD_CURRENT_MAX ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)) 507 #define ZSTD_CHUNKSIZE_MAX \ 570 U32 maxDist,
void const*
src)
591 U32 const cycleMask = (1
U << cycleLog) - 1;
593 U32 const newCurrent = (
current & cycleMask) + maxDist;
595 assert((maxDist & cycleMask) == 0);
598 assert(correction > 1<<28);
600 window->base += correction;
601 window->dictBase += correction;
602 window->lowLimit -= correction;
603 window->dictLimit -= correction;
605 DEBUGLOG(4,
"Correction of 0x%x bytes to lowLimit=0x%x", correction,
630 void const* srcEnd,
U32 maxDist,
631 U32* loadedDictEndPtr,
635 U32 loadedDictEnd = loadedDictEndPtr !=
NULL ? *loadedDictEndPtr : 0;
636 DEBUGLOG(5,
"ZSTD_window_enforceMaxDist: current=%u, maxDist=%u",
current, maxDist);
637 if (
current > maxDist + loadedDictEnd) {
639 if (
window->lowLimit < newLowLimit)
window->lowLimit = newLowLimit;
641 DEBUGLOG(5,
"Update dictLimit to match lowLimit, from %u to %u",
645 if (loadedDictEndPtr)
646 *loadedDictEndPtr = 0;
647 if (dictMatchStatePtr)
648 *dictMatchStatePtr =
NULL;
660 void const*
src,
size_t srcSize)
669 DEBUGLOG(5,
"Non contiguous blocks, new segment starts at %u",
window->dictLimit);
671 assert(distanceFromBase == (
size_t)(
U32)distanceFromBase);
672 window->dictLimit = (
U32)distanceFromBase;
674 window->base =
ip - distanceFromBase;
685 window->lowLimit = lowLimitMax;
686 DEBUGLOG(5,
"Overlapping extDict and input : new lowLimit = %u",
window->lowLimit);
696 U32 const fp_accuracy = 8;
697 U32 const fp_multiplier = (1 << fp_accuracy);
700 U32 const BWeight = hb * fp_multiplier;
703 assert(hb + fp_accuracy < 31);
704 return (
double)
weight / fp_multiplier;
718 #if defined (__cplusplus) 733 const ZSTD_CCtx_params* CCtxParams,
U64 srcSizeHint,
size_t dictSize);
741 const void* dict,
size_t dictSize,
743 ZSTD_CCtx_params
params,
unsigned long long pledgedSrcSize);
752 ZSTD_EndDirective
const flushMode);
761 const void* dict,
size_t dictSize,
762 ZSTD_dictContentType_e dictContentType,
766 unsigned long long pledgedSrcSize);
771 void*
dst,
size_t dstCapacity,
772 const void*
src,
size_t srcSize,
773 const void* dict,
size_t dictSize,
GLsizei GLenum const GLvoid GLsizei GLenum GLbyte GLbyte GLbyte GLdouble GLdouble GLdouble GLfloat GLfloat GLfloat GLint GLint GLint GLshort GLshort GLshort GLubyte GLubyte GLubyte GLuint GLuint GLuint GLushort GLushort GLushort GLbyte GLbyte GLbyte GLbyte GLdouble GLdouble GLdouble GLdouble GLfloat GLfloat GLfloat GLfloat GLint GLint GLint GLint GLshort GLshort GLshort GLshort GLubyte GLubyte GLubyte GLubyte GLuint GLuint GLuint GLuint GLushort GLushort GLushort GLushort GLboolean const GLdouble const GLfloat const GLint const GLshort const GLbyte const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLdouble const GLfloat const GLfloat const GLint const GLint const GLshort const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort GLenum GLenum GLenum GLfloat GLenum GLint GLenum GLenum GLenum GLfloat GLenum GLenum GLint GLenum GLfloat GLenum GLint GLint GLushort GLenum GLenum GLfloat GLenum GLenum GLint GLfloat const GLubyte GLenum GLenum GLenum const GLfloat GLenum GLenum const GLint GLenum GLint GLint GLsizei GLsizei GLint GLenum GLenum const GLvoid GLenum GLenum const GLfloat GLenum GLenum const GLint GLenum GLenum const GLdouble GLenum GLenum const GLfloat GLenum GLenum const GLint GLsizei GLuint GLfloat GLuint GLbitfield GLfloat GLint GLuint GLboolean GLenum GLfloat GLenum GLbitfield GLenum GLfloat GLfloat GLint GLint const GLfloat GLenum GLfloat GLfloat GLint GLint GLfloat GLfloat GLint GLint const GLfloat GLint GLfloat GLfloat GLint GLfloat GLfloat GLint GLfloat GLfloat const GLdouble * u
int workSpaceOversizedDuration
unsigned long long consumedSrcSize
rawSeqStore_t externSeqStore
static size_t ZSTD_hash5Ptr(const void *p, U32 h)
unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask)
static UCHAR ULONG UCHAR ULONG UCHAR * output
MEM_STATIC size_t MEM_readST(const void *memPtr)
static const U64 prime7bytes
unsigned long long pledgedSrcSizePlusOne
MEM_STATIC unsigned MEM_isLittleEndian(void)
size_t ZSTD_compress_advanced_internal(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize, ZSTD_CCtx_params params)
ZSTD_match_t * matchTable
GLdouble GLdouble GLdouble r
ZSTD_OptPrice_e priceType
MEM_STATIC U32 MEM_readLE32(const void *memPtr)
struct XXH64_state_s XXH64_state_t
struct ZSTD_prefixDict_s ZSTD_prefixDict
static size_t ZSTD_hash5(U64 u, U32 h)
MEM_STATIC U32 ZSTD_window_hasExtDict(ZSTD_window_t const window)
MEM_STATIC void ZSTD_window_clear(ZSTD_window_t *window)
MEM_STATIC size_t ZSTD_hash3Ptr(const void *ptr, U32 h)
ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict *cdict)
size_t ZSTD_writeLastEmptyBlock(void *dst, size_t dstCapacity)
size_t outBuffFlushedSize
void ZSTD_resetSeqStore(seqStore_t *ssPtr)
void mls(int argc, const char *argv[])
MEM_STATIC U64 MEM_readLE64(const void *memPtr)
MEM_STATIC unsigned MEM_64bits(void)
ZSTD_prefixDict prefixDict
GLfloat GLfloat GLfloat GLfloat h
static size_t ZSTD_hash4Ptr(const void *ptr, U32 h)
size_t ZSTD_referenceExternalSequences(ZSTD_CCtx *cctx, rawSeq *seq, size_t nbSeq)
static unsigned ZSTD_NbCommonBytes(size_t val)
static const U64 prime5bytes
GLenum const GLfloat * params
ZSTD_compressionStage_e stage
static const U64 prime6bytes
ZSTD_compressionParameters cParams
static size_t ZSTD_hash8Ptr(const void *p, U32 h)
ZSTD_compressionParameters cParams
static const U32 prime4bytes
MEM_STATIC void ZSTD_wildcopy(void *dst, const void *src, ptrdiff_t length)
ZSTD_CCtx_params appliedParams
ZSTD_blockState_t blockState
MEM_STATIC U32 ZSTD_MLcode(U32 mlBase)
MEM_STATIC size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch, const BYTE *const pInLimit)
MEM_STATIC void ZSTD_window_enforceMaxDist(ZSTD_window_t *window, void const *srcEnd, U32 maxDist, U32 *loadedDictEndPtr, const ZSTD_matchState_t **dictMatchStatePtr)
static size_t ZSTD_hash8(U64 u, U32 h)
MEM_STATIC size_t ZSTD_count_2segments(const BYTE *ip, const BYTE *match, const BYTE *iEnd, const BYTE *mEnd, const BYTE *iStart)
static int sum(int x_, int y_)
MEM_STATIC void ZSTD_debugTable(const U32 *table, U32 max)
FSE_repeat litlength_repeatMode
MEM_STATIC void ZSTD_storeSeq(seqStore_t *seqStorePtr, size_t litLength, const void *literals, U32 offsetCode, size_t mlBase)
const ZSTD_matchState_t * dictMatchState
static const U64 prime8bytes
ZSTD_frameParameters fParams
MEM_STATIC double ZSTD_fWeight(U32 rawStat)
static const U32 prime3bytes
ZSTD_dictContentType_e dictContentType
static size_t ZSTD_hash6Ptr(const void *p, U32 h)
MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window, void const *srcEnd)
size_t(* ZSTD_blockCompressor)(ZSTD_matchState_t *bs, seqStore_t *seqStore, U32 rep[ZSTD_REP_NUM], void const *src, size_t srcSize)
MEM_STATIC U32 MEM_read32(const void *memPtr)
ZSTD_entropyCTables_t entropy
ZSTD_optimal_t * priceTable
ZSTD_compressedBlockState_t * nextCBlock
const ZSTD_entropyCTables_t * symbolCosts
ZSTD_matchState_t matchState
static IHTMLWindow2 * window
MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t *window, U32 cycleLog, U32 maxDist, void const *src)
ZSTD_dictTableLoadMethod_e
size_t ZSTD_initCStream_internal(ZSTD_CStream *zcs, const void *dict, size_t dictSize, const ZSTD_CDict *cdict, ZSTD_CCtx_params params, unsigned long long pledgedSrcSize)
MEM_STATIC size_t ZSTD_hashPtr(const void *p, U32 hBits, U32 mls)
static size_t ZSTD_hash6(U64 u, U32 h)
static U32 ZSTD_hash4(U32 u, U32 h)
static size_t ZSTD_hash7(U64 u, U32 h)
static U32 ZSTD_hash3(U32 u, U32 h)
ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMode_e dictMode)
GLenum GLenum GLenum input
size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, ZSTD_dictTableLoadMethod_e dtlm, const ZSTD_CDict *cdict, ZSTD_CCtx_params params, unsigned long long pledgedSrcSize)
MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_matchState_t *ms)
U32 litLengthSumBasePrice
static size_t ZSTD_hash7Ptr(const void *p, U32 h)
unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask)
FSE_repeat matchlength_repeatMode
MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t *window, void const *src, size_t srcSize)
U32 matchLengthSumBasePrice
__kernel_ptrdiff_t ptrdiff_t
MEM_STATIC U32 ZSTD_highbit32(U32 val)
ZSTD_cStreamStage streamStage
ZSTD_CCtx_params requestedParams
unsigned long long producedCSize
ZSTD_dictAttachPref_e attachDictPref
ZSTD_compressedBlockState_t * prevCBlock
ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(const ZSTD_CCtx_params *CCtxParams, U64 srcSizeHint, size_t dictSize)
MEM_STATIC U16 MEM_read16(const void *memPtr)
static struct msdos_boot_sector bs
size_t outBuffContentSize
MEM_STATIC U32 ZSTD_LLcode(U32 litLength)
size_t ZSTD_compressStream_generic(ZSTD_CStream *zcs, ZSTD_outBuffer *output, ZSTD_inBuffer *input, ZSTD_EndDirective const flushMode)
struct task_struct * current
FSE_repeat offcode_repeatMode