18 #define FSE_STATIC_LINKING_ONLY 20 #define HUF_STATIC_LINKING_ONLY 61 memset(cctx, 0,
sizeof(*cctx));
74 if (!customMem.customAlloc ^ !customMem.customFree)
return NULL;
76 if (!cctx)
return NULL;
86 if ((
size_t)workspace & 7)
return NULL;
87 memset(workspace, 0, workspaceSize);
111 #ifdef ZSTD_MULTITHREAD 112 ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx =
NULL;
118 if (cctx==
NULL)
return 0;
128 #ifdef ZSTD_MULTITHREAD 129 return ZSTDMT_sizeof_CCtx(cctx->mtctx);
139 if (cctx==
NULL)
return 0;
154 ZSTD_compressionParameters cParams)
156 ZSTD_CCtx_params cctxParams;
157 memset(&cctxParams, 0,
sizeof(cctxParams));
158 cctxParams.cParams = cParams;
161 cctxParams.fParams.contentSizeFlag = 1;
166 ZSTD_customMem customMem)
169 if (!customMem.customAlloc ^ !customMem.customFree)
return NULL;
171 sizeof(ZSTD_CCtx_params), customMem);
173 params->customMem = customMem;
175 params->fParams.contentSizeFlag = 1;
197 if (!cctxParams) {
return ERROR(GENERIC); }
198 memset(cctxParams, 0,
sizeof(*cctxParams));
199 cctxParams->compressionLevel = compressionLevel;
200 cctxParams->fParams.contentSizeFlag = 1;
206 if (!cctxParams) {
return ERROR(GENERIC); }
208 memset(cctxParams, 0,
sizeof(*cctxParams));
209 cctxParams->cParams =
params.cParams;
210 cctxParams->fParams =
params.fParams;
219 ZSTD_CCtx_params cctxParams, ZSTD_parameters
params)
221 ZSTD_CCtx_params
ret = cctxParams;
229 #define CLAMPCHECK(val,min,max) { \ 230 if (((val)<(min)) | ((val)>(max))) { \ 231 return ERROR(parameter_outOfBound); \ 239 case ZSTD_p_compressionLevel:
241 case ZSTD_p_chainLog:
242 case ZSTD_p_searchLog:
243 case ZSTD_p_minMatch:
244 case ZSTD_p_targetLength:
245 case ZSTD_p_compressionStrategy:
249 case ZSTD_p_windowLog:
250 case ZSTD_p_contentSizeFlag:
251 case ZSTD_p_checksumFlag:
252 case ZSTD_p_dictIDFlag:
253 case ZSTD_p_forceMaxWindow :
254 case ZSTD_p_nbWorkers:
256 case ZSTD_p_overlapSizeLog:
257 case ZSTD_p_enableLongDistanceMatching:
258 case ZSTD_p_ldmHashLog:
259 case ZSTD_p_ldmMinMatch:
260 case ZSTD_p_ldmBucketSizeLog:
261 case ZSTD_p_ldmHashEveryLog:
262 case ZSTD_p_forceAttachDict:
275 return ERROR(stage_wrong);
283 case ZSTD_p_compressionLevel:
287 case ZSTD_p_windowLog:
289 case ZSTD_p_chainLog:
290 case ZSTD_p_searchLog:
291 case ZSTD_p_minMatch:
292 case ZSTD_p_targetLength:
293 case ZSTD_p_compressionStrategy:
297 case ZSTD_p_contentSizeFlag:
298 case ZSTD_p_checksumFlag:
299 case ZSTD_p_dictIDFlag:
302 case ZSTD_p_forceMaxWindow :
307 case ZSTD_p_forceAttachDict:
310 case ZSTD_p_nbWorkers:
312 return ERROR(parameter_unsupported);
317 case ZSTD_p_overlapSizeLog:
320 case ZSTD_p_enableLongDistanceMatching:
321 case ZSTD_p_ldmHashLog:
322 case ZSTD_p_ldmMinMatch:
323 case ZSTD_p_ldmBucketSizeLog:
324 case ZSTD_p_ldmHashEveryLog:
328 default:
return ERROR(parameter_unsupported);
333 ZSTD_CCtx_params* CCtxParams, ZSTD_cParameter
param,
unsigned value)
339 if (
value > (
unsigned)ZSTD_f_zstd1_magicless)
340 return ERROR(parameter_unsupported);
341 CCtxParams->format = (ZSTD_format_e)
value;
342 return (
size_t)CCtxParams->format;
344 case ZSTD_p_compressionLevel : {
348 CCtxParams->compressionLevel = cLevel;
350 if (CCtxParams->compressionLevel >= 0)
return CCtxParams->compressionLevel;
354 case ZSTD_p_windowLog :
357 CCtxParams->cParams.windowLog =
value;
358 return CCtxParams->cParams.windowLog;
360 case ZSTD_p_hashLog :
363 CCtxParams->cParams.hashLog =
value;
364 return CCtxParams->cParams.hashLog;
366 case ZSTD_p_chainLog :
369 CCtxParams->cParams.chainLog =
value;
370 return CCtxParams->cParams.chainLog;
372 case ZSTD_p_searchLog :
375 CCtxParams->cParams.searchLog =
value;
378 case ZSTD_p_minMatch :
381 CCtxParams->cParams.searchLength =
value;
382 return CCtxParams->cParams.searchLength;
384 case ZSTD_p_targetLength :
386 CCtxParams->cParams.targetLength =
value;
387 return CCtxParams->cParams.targetLength;
389 case ZSTD_p_compressionStrategy :
392 CCtxParams->cParams.strategy = (ZSTD_strategy)
value;
393 return (
size_t)CCtxParams->cParams.strategy;
395 case ZSTD_p_contentSizeFlag :
398 CCtxParams->fParams.contentSizeFlag =
value > 0;
399 return CCtxParams->fParams.contentSizeFlag;
401 case ZSTD_p_checksumFlag :
403 CCtxParams->fParams.checksumFlag =
value > 0;
404 return CCtxParams->fParams.checksumFlag;
406 case ZSTD_p_dictIDFlag :
408 CCtxParams->fParams.noDictIDFlag = !
value;
409 return !CCtxParams->fParams.noDictIDFlag;
411 case ZSTD_p_forceMaxWindow :
412 CCtxParams->forceWindow = (
value > 0);
413 return CCtxParams->forceWindow;
415 case ZSTD_p_forceAttachDict :
416 CCtxParams->attachDictPref =
value ?
419 return CCtxParams->attachDictPref;
421 case ZSTD_p_nbWorkers :
422 #ifndef ZSTD_MULTITHREAD 423 if (
value>0)
return ERROR(parameter_unsupported);
426 return ZSTDMT_CCtxParam_setNbWorkers(CCtxParams,
value);
429 case ZSTD_p_jobSize :
430 #ifndef ZSTD_MULTITHREAD 431 return ERROR(parameter_unsupported);
433 return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_jobSize,
value);
436 case ZSTD_p_overlapSizeLog :
437 #ifndef ZSTD_MULTITHREAD 438 return ERROR(parameter_unsupported);
440 return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_overlapSectionLog,
value);
443 case ZSTD_p_enableLongDistanceMatching :
444 CCtxParams->ldmParams.enableLdm = (
value>0);
445 return CCtxParams->ldmParams.enableLdm;
447 case ZSTD_p_ldmHashLog :
450 CCtxParams->ldmParams.hashLog =
value;
451 return CCtxParams->ldmParams.hashLog;
453 case ZSTD_p_ldmMinMatch :
456 CCtxParams->ldmParams.minMatchLength =
value;
457 return CCtxParams->ldmParams.minMatchLength;
459 case ZSTD_p_ldmBucketSizeLog :
460 if (
value > ZSTD_LDM_BUCKETSIZELOG_MAX)
461 return ERROR(parameter_outOfBound);
462 CCtxParams->ldmParams.bucketSizeLog =
value;
463 return CCtxParams->ldmParams.bucketSizeLog;
465 case ZSTD_p_ldmHashEveryLog :
466 if (
value > ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN)
467 return ERROR(parameter_outOfBound);
468 CCtxParams->ldmParams.hashEveryLog =
value;
469 return CCtxParams->ldmParams.hashEveryLog;
471 default:
return ERROR(parameter_unsupported);
481 ZSTD_CCtx_params* CCtxParams, ZSTD_cParameter
param,
unsigned*
value)
486 *
value = CCtxParams->format;
488 case ZSTD_p_compressionLevel :
489 *
value = CCtxParams->compressionLevel;
491 case ZSTD_p_windowLog :
492 *
value = CCtxParams->cParams.windowLog;
494 case ZSTD_p_hashLog :
495 *
value = CCtxParams->cParams.hashLog;
497 case ZSTD_p_chainLog :
498 *
value = CCtxParams->cParams.chainLog;
500 case ZSTD_p_searchLog :
501 *
value = CCtxParams->cParams.searchLog;
503 case ZSTD_p_minMatch :
504 *
value = CCtxParams->cParams.searchLength;
506 case ZSTD_p_targetLength :
507 *
value = CCtxParams->cParams.targetLength;
509 case ZSTD_p_compressionStrategy :
512 case ZSTD_p_contentSizeFlag :
513 *
value = CCtxParams->fParams.contentSizeFlag;
515 case ZSTD_p_checksumFlag :
516 *
value = CCtxParams->fParams.checksumFlag;
518 case ZSTD_p_dictIDFlag :
519 *
value = !CCtxParams->fParams.noDictIDFlag;
521 case ZSTD_p_forceMaxWindow :
522 *
value = CCtxParams->forceWindow;
524 case ZSTD_p_forceAttachDict :
525 *
value = CCtxParams->attachDictPref;
527 case ZSTD_p_nbWorkers :
528 #ifndef ZSTD_MULTITHREAD
529 assert(CCtxParams->nbWorkers == 0);
531 *
value = CCtxParams->nbWorkers;
533 case ZSTD_p_jobSize :
534 #ifndef ZSTD_MULTITHREAD 535 return ERROR(parameter_unsupported);
537 *
value = CCtxParams->jobSize;
540 case ZSTD_p_overlapSizeLog :
541 #ifndef ZSTD_MULTITHREAD 542 return ERROR(parameter_unsupported);
544 *
value = CCtxParams->overlapSizeLog;
547 case ZSTD_p_enableLongDistanceMatching :
548 *
value = CCtxParams->ldmParams.enableLdm;
550 case ZSTD_p_ldmHashLog :
551 *
value = CCtxParams->ldmParams.hashLog;
553 case ZSTD_p_ldmMinMatch :
554 *
value = CCtxParams->ldmParams.minMatchLength;
556 case ZSTD_p_ldmBucketSizeLog :
557 *
value = CCtxParams->ldmParams.bucketSizeLog;
559 case ZSTD_p_ldmHashEveryLog :
560 *
value = CCtxParams->ldmParams.hashEveryLog;
562 default:
return ERROR(parameter_unsupported);
577 DEBUGLOG(4,
"ZSTD_CCtx_setParametersUsingCCtxParams");
587 DEBUGLOG(4,
"ZSTD_CCtx_setPledgedSrcSize to %u bytes", (
U32)pledgedSrcSize);
594 ZSTD_CCtx* cctx,
const void* dict,
size_t dictSize,
595 ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType)
599 DEBUGLOG(4,
"ZSTD_CCtx_loadDictionary_advanced (size: %u)", (
U32)dictSize);
601 if (dict==
NULL || dictSize==0) {
605 ZSTD_compressionParameters
const cParams =
609 dictLoadMethod, dictContentType,
613 return ERROR(memory_allocation);
619 ZSTD_CCtx* cctx,
const void* dict,
size_t dictSize)
622 cctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto);
628 cctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto);
646 ZSTD_CCtx* cctx,
const void* prefix,
size_t prefixSize, ZSTD_dictContentType_e dictContentType)
676 CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
677 CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
678 CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
679 CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
680 CLAMPCHECK(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
682 if (cParams.targetLength > ZSTD_TARGETLENGTH_MAX)
683 return ERROR(parameter_outOfBound);
684 if ((
U32)(cParams.strategy) > (
U32)ZSTD_btultra)
685 return ERROR(parameter_unsupported);
692 static ZSTD_compressionParameters
695 # define CLAMP(val,min,max) { \ 696 if (val<min) val=min; \ 697 else if (val>max) val=max; \ 699 CLAMP(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
700 CLAMP(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
701 CLAMP(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
702 CLAMP(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
703 CLAMP(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
705 if (cParams.targetLength > ZSTD_TARGETLENGTH_MAX)
706 cParams.targetLength = ZSTD_TARGETLENGTH_MAX;
707 CLAMP(cParams.strategy, ZSTD_fast, ZSTD_btultra);
715 U32 const btScale = ((
U32)strat >= (
U32)ZSTD_btlazy2);
716 return hashLog - btScale;
724 static ZSTD_compressionParameters
726 unsigned long long srcSize,
729 static const U64 minSrcSize = 513;
730 static const U64 maxWindowResize = 1
ULL << (ZSTD_WINDOWLOG_MAX-1);
733 if (dictSize && (srcSize+1<2) )
734 srcSize = minSrcSize;
735 else if (srcSize == 0)
739 if ( (srcSize < maxWindowResize)
740 && (dictSize < maxWindowResize) ) {
741 U32 const tSize = (
U32)(srcSize + dictSize);
742 static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN;
743 U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN :
745 if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
747 if (cPar.hashLog > cPar.windowLog+1) cPar.hashLog = cPar.windowLog+1;
749 if (cycleLog > cPar.windowLog)
750 cPar.chainLog -= (cycleLog - cPar.windowLog);
759 ZSTD_compressionParameters
761 unsigned long long srcSize,
769 const ZSTD_CCtx_params* CCtxParams,
U64 srcSizeHint,
size_t dictSize)
771 ZSTD_compressionParameters cParams =
ZSTD_getCParams(CCtxParams->compressionLevel, srcSizeHint, dictSize);
773 if (CCtxParams->cParams.windowLog) cParams.windowLog = CCtxParams->cParams.windowLog;
774 if (CCtxParams->cParams.hashLog) cParams.hashLog = CCtxParams->cParams.hashLog;
775 if (CCtxParams->cParams.chainLog) cParams.chainLog = CCtxParams->cParams.chainLog;
776 if (CCtxParams->cParams.searchLog) cParams.searchLog = CCtxParams->cParams.searchLog;
777 if (CCtxParams->cParams.searchLength) cParams.searchLength = CCtxParams->cParams.searchLength;
778 if (CCtxParams->cParams.targetLength) cParams.targetLength = CCtxParams->cParams.targetLength;
779 if (CCtxParams->cParams.strategy) cParams.strategy = CCtxParams->cParams.strategy;
788 size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((
size_t)1 << cParams->chainLog);
789 size_t const hSize = ((
size_t)1) << cParams->hashLog;
790 U32 const hashLog3 = (forCCtx && cParams->searchLength==3) ?
MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0;
791 size_t const h3Size = ((
size_t)1) << hashLog3;
792 size_t const tableSpace = (chainSize + hSize + h3Size) *
sizeof(
U32);
795 size_t const optSpace = (forCCtx && ((cParams->strategy == ZSTD_btopt) ||
796 (cParams->strategy == ZSTD_btultra)))
799 DEBUGLOG(4,
"chainSize: %u - hSize: %u - h3Size: %u",
800 (
U32)chainSize, (
U32)hSize, (
U32)h3Size);
801 return tableSpace + optSpace;
807 if (
params->nbWorkers > 0) {
return ERROR(GENERIC); }
808 { ZSTD_compressionParameters
const cParams =
810 size_t const blockSize =
MIN(ZSTD_BLOCKSIZE_MAX, (
size_t)1 << cParams.windowLog);
811 U32 const divider = (cParams.searchLength==3) ? 3 : 4;
812 size_t const maxNbSeq = blockSize / divider;
821 size_t const neededSpace = entropySpace + blockStateSpace + tokenSpace +
822 matchStateSize + ldmSpace + ldmSeqSpace;
825 DEBUGLOG(5,
"estimate workSpace : %u", (
U32)neededSpace);
838 ZSTD_compressionParameters
const cParams =
ZSTD_getCParams(compressionLevel, 0, 0);
845 size_t memBudget = 0;
848 if (newMB > memBudget) memBudget = newMB;
855 if (
params->nbWorkers > 0) {
return ERROR(GENERIC); }
857 size_t const blockSize =
MIN(ZSTD_BLOCKSIZE_MAX, (
size_t)1 <<
params->cParams.windowLog);
858 size_t const inBuffSize = ((
size_t)1 <<
params->cParams.windowLog) + blockSize;
860 size_t const streamingSize = inBuffSize + outBuffSize;
862 return CCtxSize + streamingSize;
874 ZSTD_compressionParameters
const cParams =
ZSTD_getCParams(compressionLevel, 0, 0);
881 size_t memBudget = 0;
884 if (newMB > memBudget) memBudget = newMB;
895 #ifdef ZSTD_MULTITHREAD 897 return ZSTDMT_getFrameProgression(cctx->mtctx);
900 { ZSTD_frameProgression fp;
901 size_t const buffered = (cctx->
inBuff ==
NULL) ? 0 :
904 assert(buffered <= ZSTD_BLOCKSIZE_MAX);
910 fp.nbActiveWorkers = 0;
919 #ifdef ZSTD_MULTITHREAD 921 return ZSTDMT_toFlushNow(cctx->mtctx);
931 ZSTD_compressionParameters cParams2)
933 return (cParams1.hashLog == cParams2.hashLog)
934 & (cParams1.chainLog == cParams2.chainLog)
935 & (cParams1.strategy == cParams2.strategy)
936 & ((cParams1.searchLength==3) == (cParams2.searchLength==3));
940 ZSTD_compressionParameters cParams2)
944 assert(cParams1.windowLog == cParams2.windowLog);
945 assert(cParams1.chainLog == cParams2.chainLog);
946 assert(cParams1.hashLog == cParams2.hashLog);
947 assert(cParams1.searchLog == cParams2.searchLog);
948 assert(cParams1.searchLength == cParams2.searchLength);
949 assert(cParams1.targetLength == cParams2.targetLength);
950 assert(cParams1.strategy == cParams2.strategy);
974 ZSTD_compressionParameters cParams2,
977 size_t const windowSize2 =
MAX(1, (
size_t)
MIN(((
U64)1 << cParams2.windowLog), pledgedSrcSize));
978 size_t const blockSize2 =
MIN(ZSTD_BLOCKSIZE_MAX, windowSize2);
979 size_t const maxNbSeq2 = blockSize2 / ((cParams2.searchLength == 3) ? 3 : 4);
980 size_t const maxNbLit2 = blockSize2;
981 size_t const neededBufferSize2 = (buffPol2==
ZSTDb_buffered) ? windowSize2 + blockSize2 : 0;
982 DEBUGLOG(4,
"ZSTD_sufficientBuff: is neededBufferSize2=%u <= bufferSize1=%u",
983 (
U32)neededBufferSize2, (
U32)bufferSize1);
984 DEBUGLOG(4,
"ZSTD_sufficientBuff: is maxNbSeq2=%u <= maxNbSeq1=%u",
985 (
U32)maxNbSeq2, (
U32)maxNbSeq1);
986 DEBUGLOG(4,
"ZSTD_sufficientBuff: is maxNbLit2=%u <= maxNbLit1=%u",
987 (
U32)maxNbLit2, (
U32)maxNbLit1);
988 return (maxNbLit2 <= maxNbLit1)
989 & (maxNbSeq2 <= maxNbSeq1)
990 & (neededBufferSize2 <= bufferSize1);
995 ZSTD_CCtx_params params2,
997 size_t maxNbSeq1,
size_t maxNbLit1,
1001 DEBUGLOG(4,
"ZSTD_equivalentParams: pledgedSrcSize=%u", (
U32)pledgedSrcSize);
1003 DEBUGLOG(4,
"ZSTD_equivalentCParams() == 0");
1007 DEBUGLOG(4,
"ZSTD_equivalentLdmParams() == 0");
1011 params2.cParams, pledgedSrcSize)) {
1012 DEBUGLOG(4,
"ZSTD_sufficientBuff() == 0");
1023 bs->entropy.huf.repeatMode = HUF_repeat_none;
1024 bs->entropy.fse.offcode_repeatMode = FSE_repeat_none;
1025 bs->entropy.fse.matchlength_repeatMode = FSE_repeat_none;
1026 bs->entropy.fse.litlength_repeatMode = FSE_repeat_none;
1048 size_t const windowSize =
MAX(1, (
size_t)
MIN(((
U64)1 <<
params.cParams.windowLog), pledgedSrcSize));
1049 size_t const blockSize =
MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
1050 DEBUGLOG(4,
"ZSTD_continueCCtx: re-use context in place");
1060 DEBUGLOG(4,
"pledged content size : %u ; flag : %u",
1064 if (
params.ldmParams.enableLdm)
1078 const ZSTD_compressionParameters* cParams,
1081 size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((
size_t)1 << cParams->chainLog);
1082 size_t const hSize = ((
size_t)1) << cParams->hashLog;
1083 U32 const hashLog3 = (forCCtx && cParams->searchLength==3) ?
MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0;
1084 size_t const h3Size = ((
size_t)1) << hashLog3;
1085 size_t const tableSpace = (chainSize + hSize + h3Size) *
sizeof(
U32);
1097 if (forCCtx && ((cParams->strategy == ZSTD_btopt) | (cParams->strategy == ZSTD_btultra))) {
1098 DEBUGLOG(4,
"reserving optimal parser space");
1125 #define ZSTD_WORKSPACETOOLARGE_FACTOR 3 1126 #define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 1140 DEBUGLOG(4,
"ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u",
1141 (
U32)pledgedSrcSize,
params.cParams.windowLog);
1148 zbuff, pledgedSrcSize)) {
1149 DEBUGLOG(4,
"ZSTD_equivalentParams()==1 -> continue mode (wLog1=%u, blockSize1=%zu)",
1155 DEBUGLOG(4,
"ZSTD_equivalentParams()==0 -> reset CCtx");
1157 if (
params.ldmParams.enableLdm) {
1165 {
size_t const windowSize =
MAX(1, (
size_t)
MIN(((
U64)1 <<
params.cParams.windowLog), pledgedSrcSize));
1166 size_t const blockSize =
MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
1167 U32 const divider = (
params.cParams.searchLength==3) ? 3 : 4;
1168 size_t const maxNbSeq = blockSize / divider;
1171 size_t const buffInSize = (zbuff==
ZSTDb_buffered) ? windowSize + blockSize : 0;
1179 size_t const bufferSpace = buffInSize + buffOutSize;
1181 size_t const ldmSeqSpace = maxNbLdmSeq *
sizeof(
rawSeq);
1183 size_t const neededSpace = entropySpace + blockStateSpace + ldmSpace +
1184 ldmSeqSpace + matchStateSize + tokenSpace +
1187 int const workSpaceTooSmall = zc->
workSpaceSize < neededSpace;
1192 DEBUGLOG(4,
"Need %zuKB workspace, including %zuKB for match state, and %zuKB for buffers",
1193 neededSpace>>10, matchStateSize>>10, bufferSpace>>10);
1194 DEBUGLOG(4,
"windowSize: %zu - blockSize: %zu", windowSize, blockSize);
1196 if (workSpaceTooSmall || workSpaceWasteful) {
1197 DEBUGLOG(4,
"Need to resize workSpaceSize from %zuKB to %zuKB",
1229 DEBUGLOG(4,
"pledged content size : %u ; flag : %u",
1243 if (
params.ldmParams.enableLdm) {
1244 size_t const ldmHSize = ((
size_t)1) <<
params.ldmParams.hashLog;
1274 if (
params.ldmParams.enableLdm) {
1275 size_t const ldmBucketSize =
1277 params.ldmParams.bucketSizeLog);
1326 return ( pledgedSrcSize <= cutoff
1343 unsigned const windowLog =
params.cParams.windowLog;
1348 params.cParams.windowLog = windowLog;
1358 if (cdictLen == 0) {
1360 DEBUGLOG(4,
"skipping attaching empty dictionary");
1362 DEBUGLOG(4,
"attaching dictionary into context");
1392 DEBUGLOG(4,
"copying dictionary into context");
1394 {
unsigned const windowLog =
params.cParams.windowLog;
1397 params.cParams = *cdict_cParams;
1398 params.cParams.windowLog = windowLog;
1407 {
size_t const chainSize = (cdict_cParams->strategy == ZSTD_fast) ? 0 : ((
size_t)1 << cdict_cParams->chainLog);
1408 size_t const hSize = (
size_t)1 << cdict_cParams->hashLog;
1409 size_t const tableSpace = (chainSize + hSize) *
sizeof(
U32);
1450 DEBUGLOG(4,
"ZSTD_resetCCtx_usingCDict (pledgedSrcSize=%u)", (
U32)pledgedSrcSize);
1454 cctx, cdict,
params, pledgedSrcSize, zbuff);
1457 cctx, cdict,
params, pledgedSrcSize, zbuff);
1470 ZSTD_frameParameters fParams,
1474 DEBUGLOG(5,
"ZSTD_copyCCtx_internal");
1481 params.fParams = fParams;
1492 {
size_t const chainSize = (srcCCtx->
appliedParams.cParams.strategy == ZSTD_fast) ? 0 : ((
size_t)1 << srcCCtx->
appliedParams.cParams.chainLog);
1495 size_t const tableSpace = (chainSize + hSize + h3Size) *
sizeof(
U32);
1525 ZSTD_frameParameters fParams = { 1 , 0 , 0 };
1532 fParams, pledgedSrcSize,
1537 #define ZSTD_ROWSIZE 16 1552 for (rowNb=0 ; rowNb < nbRows ; rowNb++) {
1557 table[cellNb] += adder;
1559 if (
table[cellNb] < reducerValue)
table[cellNb] = 0;
1560 else table[cellNb] -= reducerValue;
1607 U32 const cBlockHeader24 = lastBlock + (((
U32)
bt_raw)<<1) + (
U32)(srcSize << 3);
1617 U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
1619 if (srcSize + flSize > dstCapacity)
return ERROR(dstSize_tooSmall);
1637 return srcSize + flSize;
1643 U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
1662 ostart[flSize] = *(
const BYTE*)
src;
1673 U32 const minlog = (strat==ZSTD_btultra) ? 7 : 6;
1674 return (srcSize >> minlog) + 2;
1679 ZSTD_strategy strategy,
int disableLiteralCompression,
1680 void*
dst,
size_t dstCapacity,
1681 const void*
src,
size_t srcSize,
1682 U32* workspace,
const int bmi2)
1684 size_t const minGain =
ZSTD_minGain(srcSize, strategy);
1685 size_t const lhSize = 3 + (srcSize >= 1
KB) + (srcSize >= 16
KB);
1687 U32 singleStream = srcSize < 256;
1691 DEBUGLOG(5,
"ZSTD_compressLiterals (disableLiteralCompression=%i)",
1692 disableLiteralCompression);
1695 memcpy(nextHuf, prevHuf,
sizeof(*prevHuf));
1697 if (disableLiteralCompression)
1701 # define COMPRESS_LITERALS_SIZE_MIN 63 1706 if (dstCapacity < lhSize+1)
return ERROR(dstSize_tooSmall);
1708 int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0;
1709 if (
repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
1714 if (
repeat != HUF_repeat_none) {
1720 if ((cLitSize==0) | (cLitSize >= srcSize - minGain) |
ERR_isError(cLitSize)) {
1721 memcpy(nextHuf, prevHuf,
sizeof(*prevHuf));
1725 memcpy(nextHuf, prevHuf,
sizeof(*prevHuf));
1738 {
U32 const lhc = hType + ((!singleStream) << 2) + ((
U32)srcSize<<4) + ((
U32)cLitSize<<14);
1743 {
U32 const lhc = hType + (2 << 2) + ((
U32)srcSize<<4) + ((
U32)cLitSize<<18);
1748 {
U32 const lhc = hType + (3 << 2) + ((
U32)srcSize<<4) + ((
U32)cLitSize<<22);
1750 ostart[4] = (
BYTE)(cLitSize >> 10);
1756 return lhSize+cLitSize;
1763 BYTE*
const llCodeTable = seqStorePtr->
llCode;
1764 BYTE*
const ofCodeTable = seqStorePtr->
ofCode;
1765 BYTE*
const mlCodeTable = seqStorePtr->
mlCode;
1768 assert(nbSeq <= seqStorePtr->maxNbSeq);
1769 for (
u=0;
u<nbSeq;
u++) {
1789 0, 2048, 1792, 1642, 1536, 1453, 1386, 1329, 1280, 1236, 1197, 1162,
1790 1130, 1100, 1073, 1047, 1024, 1001, 980, 960, 941, 923, 906, 889,
1791 874, 859, 844, 830, 817, 804, 791, 779, 768, 756, 745, 734,
1792 724, 714, 704, 694, 685, 676, 667, 658, 650, 642, 633, 626,
1793 618, 610, 603, 595, 588, 581, 574, 567, 561, 554, 548, 542,
1794 535, 529, 523, 517, 512, 506, 500, 495, 489, 484, 478, 473,
1795 468, 463, 458, 453, 448, 443, 438, 434, 429, 424, 420, 415,
1796 411, 407, 402, 398, 394, 390, 386, 382, 377, 373, 370, 366,
1797 362, 358, 354, 350, 347, 343, 339, 336, 332, 329, 325, 322,
1798 318, 315, 311, 308, 305, 302, 298, 295, 292, 289, 286, 282,
1799 279, 276, 273, 270, 267, 264, 261, 258, 256, 253, 250, 247,
1800 244, 241, 239, 236, 233, 230, 228, 225, 222, 220, 217, 215,
1801 212, 209, 207, 204, 202, 199, 197, 194, 192, 190, 187, 185,
1802 182, 180, 178, 175, 173, 171, 168, 166, 164, 162, 159, 157,
1803 155, 153, 151, 149, 146, 144, 142, 140, 138, 136, 134, 132,
1804 130, 128, 126, 123, 121, 119, 117, 115, 114, 112, 110, 108,
1805 106, 104, 102, 100, 98, 96, 94, 93, 91, 89, 87, 85,
1806 83, 82, 80, 78, 76, 74, 73, 71, 69, 67, 66, 64,
1807 62, 61, 59, 57, 55, 54, 52, 50, 49, 47, 46, 44,
1808 42, 41, 39, 37, 36, 34, 33, 31, 30, 28, 26, 25,
1809 23, 22, 20, 19, 17, 16, 14, 13, 11, 10, 8, 7,
1822 for (
s = 0;
s <=
max; ++
s) {
1839 unsigned const*
count,
unsigned const max)
1841 unsigned const shift = 8 - accuracyLog;
1844 assert(accuracyLog <= 8);
1845 for (
s = 0;
s <=
max; ++
s) {
1846 unsigned const normAcc =
norm[
s] != -1 ?
norm[
s] : 1;
1847 unsigned const norm256 = normAcc <<
shift;
1860 return maxSymbolValue;
1870 unsigned const*
count,
1873 unsigned const kAccuracyLog = 8;
1879 DEBUGLOG(5,
"Repeat FSE_CTable has maxSymbolValue %u < %u",
1881 return ERROR(GENERIC);
1883 for (
s = 0;
s <=
max; ++
s) {
1884 unsigned const tableLog =
cstate.stateLog;
1885 unsigned const badCost = (tableLog + 1) << kAccuracyLog;
1886 unsigned const bitCost = FSE_bitCost(
cstate.symbolTT, tableLog,
s, kAccuracyLog);
1889 if (bitCost >= badCost) {
1890 DEBUGLOG(5,
"Repeat FSE_CTable has Prob[%u] == 0",
s);
1891 return ERROR(GENERIC);
1893 cost +=
count[
s] * bitCost;
1895 return cost >> kAccuracyLog;
1903 size_t const nbSeq,
unsigned const FSELog)
1905 BYTE wksp[FSE_NCOUNTBOUND];
1920 FSE_repeat* repeatMode,
unsigned const*
count,
unsigned const max,
1921 size_t const mostFrequent,
size_t nbSeq,
unsigned const FSELog,
1923 short const* defaultNorm,
U32 defaultNormLog,
1925 ZSTD_strategy
const strategy)
1928 if (mostFrequent == nbSeq) {
1929 *repeatMode = FSE_repeat_none;
1930 if (isDefaultAllowed && nbSeq <= 2) {
1941 if (strategy < ZSTD_lazy) {
1942 if (isDefaultAllowed) {
1943 size_t const staticFse_nbSeq_max = 1000;
1944 size_t const mult = 10 - strategy;
1945 size_t const baseLog = 3;
1946 size_t const dynamicFse_nbSeq_min = (((
size_t)1 << defaultNormLog) * mult) >> baseLog;
1947 assert(defaultNormLog >= 5 && defaultNormLog <= 6);
1948 assert(mult <= 9 && mult >= 7);
1949 if ( (*repeatMode == FSE_repeat_valid)
1950 && (nbSeq < staticFse_nbSeq_max) ) {
1951 DEBUGLOG(5,
"Selected set_repeat");
1954 if ( (nbSeq < dynamicFse_nbSeq_min)
1955 || (mostFrequent < (nbSeq >> (defaultNormLog-1))) ) {
1963 *repeatMode = FSE_repeat_none;
1973 if (isDefaultAllowed) {
1979 DEBUGLOG(5,
"Estimated bit costs: basic=%u\trepeat=%u\tcompressed=%u",
1980 (
U32)basicCost, (
U32)repeatCost, (
U32)compressedCost);
1981 if (basicCost <= repeatCost && basicCost <= compressedCost) {
1983 assert(isDefaultAllowed);
1984 *repeatMode = FSE_repeat_none;
1987 if (repeatCost <= compressedCost) {
1988 DEBUGLOG(5,
"Selected set_repeat");
1992 assert(compressedCost < basicCost && compressedCost < repeatCost);
1994 DEBUGLOG(5,
"Selected set_compressed");
1995 *repeatMode = FSE_repeat_check;
2003 const BYTE* codeTable,
size_t nbSeq,
2004 const S16* defaultNorm,
U32 defaultNormLog,
U32 defaultMax,
2005 const FSE_CTable* prevCTable,
size_t prevCTableSize,
2006 void* workspace,
size_t workspaceSize)
2009 const BYTE*
const oend =
op + dstCapacity;
2017 memcpy(nextCTable, prevCTable, prevCTableSize);
2024 size_t nbSeq_1 = nbSeq;
2026 if (
count[codeTable[nbSeq-1]] > 1) {
2027 count[codeTable[nbSeq-1]]--;
2044 void*
dst,
size_t dstCapacity,
2051 FSE_CState_t stateMatchLength;
2052 FSE_CState_t stateOffsetBits;
2053 FSE_CState_t stateLitLength;
2058 FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]);
2059 FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]);
2060 FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]);
2066 U32 const ofBits = ofCodeTable[nbSeq-1];
2073 ofBits - extraBits);
2080 for (
n=nbSeq-2 ;
n<nbSeq ;
n--) {
2081 BYTE const llCode = llCodeTable[
n];
2082 BYTE const ofCode = ofCodeTable[
n];
2083 BYTE const mlCode = mlCodeTable[
n];
2085 U32 const ofBits = ofCode;
2087 DEBUGLOG(6,
"encoding: litlen:%2u - matchlen:%2u - offCode:%7u",
2093 FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode);
2094 FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode);
2096 FSE_encodeSymbol(&blockStream, &stateLitLength, llCode);
2110 ofBits - extraBits);
2117 DEBUGLOG(6,
"ZSTD_encodeSequences: flushing ML state with %u bits", stateMatchLength.stateLog);
2118 FSE_flushCState(&blockStream, &stateMatchLength);
2119 DEBUGLOG(6,
"ZSTD_encodeSequences: flushing Off state with %u bits", stateOffsetBits.stateLog);
2120 FSE_flushCState(&blockStream, &stateOffsetBits);
2121 DEBUGLOG(6,
"ZSTD_encodeSequences: flushing LL state with %u bits", stateLitLength.stateLog);
2122 FSE_flushCState(&blockStream, &stateLitLength);
2125 if (streamSize==0)
return ERROR(dstSize_tooSmall);
2132 void*
dst,
size_t dstCapacity,
2139 CTable_MatchLength, mlCodeTable,
2140 CTable_OffsetBits, ofCodeTable,
2141 CTable_LitLength, llCodeTable,
2149 ZSTD_encodeSequences_bmi2(
2150 void*
dst,
size_t dstCapacity,
2157 CTable_MatchLength, mlCodeTable,
2158 CTable_OffsetBits, ofCodeTable,
2159 CTable_LitLength, llCodeTable,
2166 void*
dst,
size_t dstCapacity,
2174 return ZSTD_encodeSequences_bmi2(
dst, dstCapacity,
2175 CTable_MatchLength, mlCodeTable,
2176 CTable_OffsetBits, ofCodeTable,
2177 CTable_LitLength, llCodeTable,
2183 CTable_MatchLength, mlCodeTable,
2184 CTable_OffsetBits, ofCodeTable,
2185 CTable_LitLength, llCodeTable,
2192 ZSTD_CCtx_params
const* cctxParams,
2193 void*
dst,
size_t dstCapacity,
U32* workspace,
2197 ZSTD_strategy
const strategy = cctxParams->cParams.strategy;
2202 U32 LLtype, Offtype, MLtype;
2204 const BYTE*
const ofCodeTable = seqStorePtr->
ofCode;
2205 const BYTE*
const llCodeTable = seqStorePtr->
llCode;
2206 const BYTE*
const mlCodeTable = seqStorePtr->
mlCode;
2208 BYTE*
const oend = ostart + dstCapacity;
2218 size_t const litSize = seqStorePtr->
lit - literals;
2219 int const disableLiteralCompression = (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0);
2221 &prevEntropy->
huf, &nextEntropy->
huf,
2222 cctxParams->cParams.strategy, disableLiteralCompression,
2228 assert(cSize <= dstCapacity);
2233 if ((oend-
op) < 3 + 1 )
return ERROR(dstSize_tooSmall);
2242 memcpy(&nextEntropy->
fse, &prevEntropy->
fse,
sizeof(prevEntropy->
fse));
2256 LLtype =
ZSTD_selectEncodingType(&nextEntropy->
fse.
litlength_repeatMode,
count,
max, mostFrequent, nbSeq,
LLFSELog, prevEntropy->
fse.
litlengthCTable,
LL_defaultNorm,
LL_defaultNormLog,
ZSTD_defaultAllowed, strategy);
2258 assert(!(LLtype < set_compressed && nextEntropy->fse.litlength_repeatMode != FSE_repeat_none));
2275 Offtype =
ZSTD_selectEncodingType(&nextEntropy->
fse.
offcode_repeatMode,
count,
max, mostFrequent, nbSeq,
OffFSELog, prevEntropy->
fse.
offcodeCTable,
OF_defaultNorm,
OF_defaultNormLog, defaultPolicy, strategy);
2276 assert(!(Offtype < set_compressed && nextEntropy->fse.offcode_repeatMode != FSE_repeat_none));
2291 MLtype =
ZSTD_selectEncodingType(&nextEntropy->
fse.
matchlength_repeatMode,
count,
max, mostFrequent, nbSeq,
MLFSELog, prevEntropy->
fse.
matchlengthCTable,
ML_defaultNorm,
ML_defaultNormLog,
ZSTD_defaultAllowed, strategy);
2292 assert(!(MLtype < set_compressed && nextEntropy->fse.matchlength_repeatMode != FSE_repeat_none));
2303 *seqHead = (
BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
2307 CTable_MatchLength, mlCodeTable,
2308 CTable_OffsetBits, ofCodeTable,
2309 CTable_LitLength, llCodeTable,
2313 op += bitstreamSize;
2322 if (lastNCount && (
op - lastNCount) < 4) {
2325 DEBUGLOG(5,
"Avoiding bug in zstd decoder in versions <= 1.3.4 by " 2326 "emitting an uncompressed block.");
2337 const ZSTD_CCtx_params* cctxParams,
2338 void*
dst,
size_t dstCapacity,
2339 size_t srcSize,
U32* workspace,
int bmi2)
2342 seqStorePtr, prevEntropy, nextEntropy, cctxParams,
dst, dstCapacity,
2344 if (cSize == 0)
return 0;
2348 if ((cSize ==
ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity))
2353 {
size_t const maxCSize = srcSize -
ZSTD_minGain(srcSize, cctxParams->cParams.strategy);
2354 if (cSize >= maxCSize)
return 0;
2399 selectedCompressor = blockCompressor[(
int)dictMode][(
U32)strat];
2401 return selectedCompressor;
2405 const BYTE* anchor,
size_t lastLLSize)
2407 memcpy(seqStorePtr->
lit, anchor, lastLLSize);
2408 seqStorePtr->
lit += lastLLSize;
2419 void*
dst,
size_t dstCapacity,
2420 const void*
src,
size_t srcSize)
2424 DEBUGLOG(5,
"ZSTD_compressBlock_internal (dstCapacity=%zu, dictLimit=%u, nextToUpdate=%u)",
2426 assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
2489 {
const BYTE*
const lastLiterals = (
const BYTE*)
src + srcSize - lastLLSize;
2526 void*
dst,
size_t dstCapacity,
2527 const void*
src,
size_t srcSize,
2531 size_t remaining = srcSize;
2538 DEBUGLOG(5,
"ZSTD_compress_frameChunk (blockSize=%u)", (
U32)blockSize);
2544 U32 const lastBlock = lastFrameChunk & (blockSize >= remaining);
2547 return ERROR(dstSize_tooSmall);
2548 if (remaining < blockSize) blockSize = remaining;
2580 assert(remaining >= blockSize);
2581 remaining -= blockSize;
2583 assert(dstCapacity >= cSize);
2584 dstCapacity -= cSize;
2585 DEBUGLOG(5,
"ZSTD_compress_frameChunk: adding a block of size %u",
2595 ZSTD_CCtx_params
params,
U64 pledgedSrcSize,
U32 dictID)
2597 U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536);
2598 U32 const dictIDSizeCode =
params.fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength;
2599 U32 const checksumFlag =
params.fParams.checksumFlag>0;
2600 U32 const windowSize = (
U32)1 <<
params.cParams.windowLog;
2601 U32 const singleSegment =
params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize);
2603 U32 const fcsCode =
params.fParams.contentSizeFlag ?
2604 (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFF
U) : 0;
2605 BYTE const frameHeaderDecriptionByte = (
BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) );
2609 if (dstCapacity < ZSTD_frameHeaderSize_max)
return ERROR(dstSize_tooSmall);
2610 DEBUGLOG(4,
"ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u",
2611 !
params.fParams.noDictIDFlag, dictID, dictIDSizeCode);
2613 if (
params.format == ZSTD_f_zstd1) {
2617 op[
pos++] = frameHeaderDecriptionByte;
2618 if (!singleSegment)
op[
pos++] = windowLogByte;
2619 switch(dictIDSizeCode)
2630 case 0 :
if (singleSegment)
op[
pos++] = (
BYTE)(pledgedSrcSize);
break;
2655 return ERROR(stage_wrong);
2657 return ERROR(parameter_unsupported);
2667 void*
dst,
size_t dstCapacity,
2668 const void*
src,
size_t srcSize,
2669 U32 frame,
U32 lastFrameChunk)
2674 DEBUGLOG(5,
"ZSTD_compressContinue_internal, stage: %u, srcSize: %u",
2682 dstCapacity -= fhSize;
2683 dst = (
char*)
dst + fhSize;
2687 if (!srcSize)
return fhSize;
2713 {
size_t const cSize = frame ?
2723 DEBUGLOG(4,
"error : pledgedSrcSize = %u, while realSrcSize >= %u",
2725 return ERROR(srcSize_wrong);
2728 return cSize + fhSize;
2733 void*
dst,
size_t dstCapacity,
2734 const void*
src,
size_t srcSize)
2736 DEBUGLOG(5,
"ZSTD_compressContinue (srcSize=%u)", (
U32)srcSize);
2743 ZSTD_compressionParameters
const cParams = cctx->
appliedParams.cParams;
2745 return MIN (ZSTD_BLOCKSIZE_MAX, (
U32)1 << cParams.windowLog);
2751 if (srcSize > blockSizeMax)
return ERROR(srcSize_wrong);
2760 ZSTD_CCtx_params
const*
params,
2761 const void*
src,
size_t srcSize,
2765 const BYTE*
const iend =
ip + srcSize;
2775 switch(
params->cParams.strategy)
2811 static size_t ZSTD_checkDictNCount(
short* normalizedCounter,
unsigned dictMaxSymbolValue,
unsigned maxSymbolValue) {
2813 if (dictMaxSymbolValue < maxSymbolValue)
return ERROR(dictionary_corrupted);
2814 for (
s = 0;
s <= maxSymbolValue; ++
s) {
2815 if (normalizedCounter[
s] == 0)
return ERROR(dictionary_corrupted);
2832 ZSTD_CCtx_params
const*
params,
2833 const void* dict,
size_t dictSize,
2837 const BYTE* dictPtr = (
const BYTE*)dict;
2838 const BYTE*
const dictEnd = dictPtr + dictSize;
2839 short offcodeNCount[
MaxOff+1];
2840 unsigned offcodeMaxValue =
MaxOff;
2851 {
unsigned maxSymbolValue = 255;
2852 size_t const hufHeaderSize =
HUF_readCTable((HUF_CElt*)
bs->entropy.huf.CTable, &maxSymbolValue, dictPtr, dictEnd-dictPtr);
2854 if (maxSymbolValue < 255)
return ERROR(dictionary_corrupted);
2855 dictPtr += hufHeaderSize;
2858 {
unsigned offcodeLog;
2859 size_t const offcodeHeaderSize =
FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
2865 dictionary_corrupted);
2866 dictPtr += offcodeHeaderSize;
2869 {
short matchlengthNCount[
MaxML+1];
2870 unsigned matchlengthMaxValue =
MaxML, matchlengthLog;
2871 size_t const matchlengthHeaderSize =
FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
2872 if (
FSE_isError(matchlengthHeaderSize))
return ERROR(dictionary_corrupted);
2873 if (matchlengthLog >
MLFSELog)
return ERROR(dictionary_corrupted);
2877 dictionary_corrupted);
2878 dictPtr += matchlengthHeaderSize;
2881 {
short litlengthNCount[
MaxLL+1];
2882 unsigned litlengthMaxValue =
MaxLL, litlengthLog;
2883 size_t const litlengthHeaderSize =
FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
2885 if (litlengthLog >
LLFSELog)
return ERROR(dictionary_corrupted);
2889 dictionary_corrupted);
2890 dictPtr += litlengthHeaderSize;
2893 if (dictPtr+12 > dictEnd)
return ERROR(dictionary_corrupted);
2899 {
size_t const dictContentSize = (
size_t)(dictEnd - dictPtr);
2901 if (dictContentSize <= ((
U32)-1) - 128
KB) {
2902 U32 const maxOffset = (
U32)dictContentSize + 128
KB;
2909 for (
u=0;
u<3;
u++) {
2910 if (
bs->rep[
u] == 0)
return ERROR(dictionary_corrupted);
2911 if (
bs->rep[
u] > dictContentSize)
return ERROR(dictionary_corrupted);
2914 bs->entropy.huf.repeatMode = HUF_repeat_valid;
2915 bs->entropy.fse.offcode_repeatMode = FSE_repeat_valid;
2916 bs->entropy.fse.matchlength_repeatMode = FSE_repeat_valid;
2917 bs->entropy.fse.litlength_repeatMode = FSE_repeat_valid;
2928 const ZSTD_CCtx_params*
params,
2929 const void* dict,
size_t dictSize,
2930 ZSTD_dictContentType_e dictContentType,
2934 DEBUGLOG(4,
"ZSTD_compress_insertDictionary (dictSize=%u)", (
U32)dictSize);
2935 if ((dict==
NULL) || (dictSize<=8))
return 0;
2940 if (dictContentType == ZSTD_dct_rawContent)
2944 if (dictContentType == ZSTD_dct_auto) {
2945 DEBUGLOG(4,
"raw content dictionary detected");
2948 if (dictContentType == ZSTD_dct_fullDict)
2949 return ERROR(dictionary_wrong);
2960 const void* dict,
size_t dictSize,
2961 ZSTD_dictContentType_e dictContentType,
2964 ZSTD_CCtx_params
params,
U64 pledgedSrcSize,
2967 DEBUGLOG(4,
"ZSTD_compressBegin_internal: wlog=%u",
params.cParams.windowLog);
2970 assert(!((dict) && (cdict)));
2990 const void* dict,
size_t dictSize,
2991 ZSTD_dictContentType_e dictContentType,
2995 unsigned long long pledgedSrcSize)
2997 DEBUGLOG(4,
"ZSTD_compressBegin_advanced_internal: wlog=%u",
params.cParams.windowLog);
3001 dict, dictSize, dictContentType, dtlm,
3010 const void* dict,
size_t dictSize,
3011 ZSTD_parameters
params,
unsigned long long pledgedSrcSize)
3013 ZSTD_CCtx_params
const cctxParams =
3018 cctxParams, pledgedSrcSize);
3024 ZSTD_CCtx_params
const cctxParams =
3026 DEBUGLOG(4,
"ZSTD_compressBegin_usingDict (dictSize=%u)", (
U32)dictSize);
3053 dstCapacity -= fhSize;
3061 if (dstCapacity<4)
return ERROR(dstSize_tooSmall);
3069 if (dstCapacity<4)
return ERROR(dstSize_tooSmall);
3080 void*
dst,
size_t dstCapacity,
3081 const void*
src,
size_t srcSize)
3085 dst, dstCapacity,
src, srcSize,
3093 DEBUGLOG(4,
"end of frame : controlling src size");
3095 DEBUGLOG(4,
"error : pledgedSrcSize = %u, while realSrcSize = %u",
3097 return ERROR(srcSize_wrong);
3099 return cSize + endResult;
3104 void*
dst,
size_t dstCapacity,
3105 const void*
src,
size_t srcSize,
3106 const void* dict,
size_t dictSize,
3109 ZSTD_CCtx_params
const cctxParams =
3111 DEBUGLOG(4,
"ZSTD_compress_internal");
3120 void*
dst,
size_t dstCapacity,
3121 const void*
src,
size_t srcSize,
3122 const void* dict,
size_t dictSize,
3125 DEBUGLOG(4,
"ZSTD_compress_advanced");
3137 void*
dst,
size_t dstCapacity,
3138 const void*
src,
size_t srcSize,
3139 const void* dict,
size_t dictSize,
3142 DEBUGLOG(4,
"ZSTD_compress_advanced_internal (srcSize:%u)", (
U32)srcSize);
3150 void*
dst,
size_t dstCapacity,
3151 const void*
src,
size_t srcSize,
3152 const void* dict,
size_t dictSize,
3153 int compressionLevel)
3155 ZSTD_parameters
const params =
ZSTD_getParams(compressionLevel, srcSize + (!srcSize), dict ? dictSize : 0);
3162 void*
dst,
size_t dstCapacity,
3163 const void*
src,
size_t srcSize,
3164 int compressionLevel)
3166 DEBUGLOG(4,
"ZSTD_compressCCtx (srcSize=%u)", (
U32)srcSize);
3172 const void*
src,
size_t srcSize,
3173 int compressionLevel)
3189 size_t dictSize, ZSTD_compressionParameters cParams,
3190 ZSTD_dictLoadMethod_e dictLoadMethod)
3194 + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
3199 ZSTD_compressionParameters
const cParams =
ZSTD_getCParams(compressionLevel, 0, dictSize);
3205 if (cdict==
NULL)
return 0;
3206 DEBUGLOG(5,
"sizeof(*cdict) : %u", (
U32)
sizeof(*cdict));
3212 const void* dictBuffer,
size_t dictSize,
3213 ZSTD_dictLoadMethod_e dictLoadMethod,
3214 ZSTD_dictContentType_e dictContentType,
3215 ZSTD_compressionParameters cParams)
3217 DEBUGLOG(3,
"ZSTD_initCDict_internal (dictContentType:%u)", (
U32)dictContentType);
3220 if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) {
3227 if (!internalBuffer)
return ERROR(memory_allocation);
3228 memcpy(internalBuffer, dictBuffer, dictSize);
3244 { ZSTD_CCtx_params
params;
3247 params.fParams.contentSizeFlag = 1;
3248 params.cParams = cParams;
3263 ZSTD_dictLoadMethod_e dictLoadMethod,
3264 ZSTD_dictContentType_e dictContentType,
3265 ZSTD_compressionParameters cParams, ZSTD_customMem customMem)
3267 DEBUGLOG(3,
"ZSTD_createCDict_advanced, mode %u", (
U32)dictContentType);
3268 if (!customMem.customAlloc ^ !customMem.customFree)
return NULL;
3272 void*
const workspace =
ZSTD_malloc(workspaceSize, customMem);
3274 if (!cdict || !workspace) {
3283 dictBuffer, dictSize,
3284 dictLoadMethod, dictContentType,
3296 ZSTD_compressionParameters cParams =
ZSTD_getCParams(compressionLevel, 0, dictSize);
3298 ZSTD_dlm_byCopy, ZSTD_dct_auto,
3299 cParams, ZSTD_defaultCMem);
3304 ZSTD_compressionParameters cParams =
ZSTD_getCParams(compressionLevel, 0, dictSize);
3306 ZSTD_dlm_byRef, ZSTD_dct_auto,
3307 cParams, ZSTD_defaultCMem);
3312 if (cdict==
NULL)
return 0;
3313 { ZSTD_customMem
const cMem = cdict->
customMem;
3335 void* workspace,
size_t workspaceSize,
3336 const void* dict,
size_t dictSize,
3337 ZSTD_dictLoadMethod_e dictLoadMethod,
3338 ZSTD_dictContentType_e dictContentType,
3339 ZSTD_compressionParameters cParams)
3342 size_t const neededSize =
sizeof(
ZSTD_CDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize)
3346 if ((
size_t)workspace & 7)
return NULL;
3347 DEBUGLOG(4,
"(workspaceSize < neededSize) : (%u < %u) => %u",
3348 (
U32)workspaceSize, (
U32)neededSize, (
U32)(workspaceSize < neededSize));
3349 if (workspaceSize < neededSize)
return NULL;
3351 if (dictLoadMethod == ZSTD_dlm_byCopy) {
3352 memcpy(cdict+1, dict, dictSize);
3363 ZSTD_dlm_byRef, dictContentType,
3380 ZSTD_frameParameters
const fParams,
unsigned long long const pledgedSrcSize)
3382 DEBUGLOG(4,
"ZSTD_compressBegin_usingCDict_advanced");
3383 if (cdict==
NULL)
return ERROR(dictionary_wrong);
3391 U32 const limitedSrcSize = (
U32)
MIN(pledgedSrcSize, 1
U << 19);
3392 U32 const limitedSrcLog = limitedSrcSize > 1 ?
ZSTD_highbit32(limitedSrcSize - 1) + 1 : 1;
3393 params.cParams.windowLog =
MAX(
params.cParams.windowLog, limitedSrcLog);
3395 params.fParams = fParams;
3409 ZSTD_frameParameters
const fParams = { 0 , 0 , 0 };
3410 DEBUGLOG(4,
"ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag);
3415 void*
dst,
size_t dstCapacity,
3416 const void*
src,
size_t srcSize,
3417 const ZSTD_CDict* cdict, ZSTD_frameParameters fParams)
3429 void*
dst,
size_t dstCapacity,
3430 const void*
src,
size_t srcSize,
3433 ZSTD_frameParameters
const fParams = { 1 , 0 , 0 };
3476 const void*
const dict,
size_t const dictSize, ZSTD_dictContentType_e
const dictContentType,
3478 ZSTD_CCtx_params
params,
unsigned long long const pledgedSrcSize)
3480 DEBUGLOG(4,
"ZSTD_resetCStream_internal");
3485 assert(!((dict) && (cdict)));
3508 DEBUGLOG(4,
"ZSTD_resetCStream: pledgedSrcSize = %u", (
U32)pledgedSrcSize);
3510 params.fParams.contentSizeFlag = 1;
3519 const void* dict,
size_t dictSize,
const ZSTD_CDict* cdict,
3520 ZSTD_CCtx_params
params,
unsigned long long pledgedSrcSize)
3522 DEBUGLOG(4,
"ZSTD_initCStream_internal");
3525 assert(!((dict) && (cdict)));
3527 if (dict && dictSize >= 8) {
3528 DEBUGLOG(4,
"loading dictionary of size %u", (
U32)dictSize);
3531 return ERROR(memory_allocation);
3535 ZSTD_dlm_byCopy, ZSTD_dct_auto,
3555 ZSTD_frameParameters fParams,
3556 unsigned long long pledgedSrcSize)
3558 DEBUGLOG(4,
"ZSTD_initCStream_usingCDict_advanced");
3559 if (!cdict)
return ERROR(dictionary_wrong);
3562 params.fParams = fParams;
3572 ZSTD_frameParameters
const fParams = { 0 , 0 , 0 };
3573 DEBUGLOG(4,
"ZSTD_initCStream_usingCDict");
3583 const void* dict,
size_t dictSize,
3584 ZSTD_parameters
params,
unsigned long long pledgedSrcSize)
3586 DEBUGLOG(4,
"ZSTD_initCStream_advanced: pledgedSrcSize=%u, flag=%u",
3587 (
U32)pledgedSrcSize,
params.fParams.contentSizeFlag);
3616 const void*
src,
size_t srcSize)
3618 size_t const length =
MIN(dstCapacity, srcSize);
3630 ZSTD_EndDirective
const flushMode)
3632 const char*
const istart = (
const char*)
input->src;
3633 const char*
const iend = istart +
input->size;
3634 const char*
ip = istart +
input->pos;
3635 char*
const ostart = (
char*)
output->dst;
3636 char*
const oend = ostart +
output->size;
3638 U32 someMoreWork = 1;
3641 DEBUGLOG(5,
"ZSTD_compressStream_generic, flush=%u", (
U32)flushMode);
3649 while (someMoreWork) {
3654 return ERROR(init_missing);
3657 if ( (flushMode == ZSTD_e_end)
3669 someMoreWork = 0;
break;
3678 if ( (flushMode == ZSTD_e_continue)
3681 someMoreWork = 0;
break;
3683 if ( (flushMode == ZSTD_e_flush)
3686 someMoreWork = 0;
break;
3690 DEBUGLOG(5,
"stream compression stage (flushMode==%u)", flushMode);
3694 size_t oSize = oend-
op;
3695 unsigned const lastBlock = (flushMode == ZSTD_e_end) && (
ip==iend);
3711 DEBUGLOG(5,
"inBuffTarget:%u / inBuffSize:%u",
3719 DEBUGLOG(5,
"Frame completed directly in outBuffer");
3735 DEBUGLOG(5,
"toFlush: %u into %u ==> flushed: %u",
3739 if (toFlush!=flushed) {
3747 DEBUGLOG(5,
"Frame completed on flush");
3765 if (hintInSize==0) hintInSize = zcs->
blockSize;
3783 ZSTD_EndDirective endOp)
3785 DEBUGLOG(5,
"ZSTD_compress_generic, endOp=%u ", (
U32)endOp);
3797 DEBUGLOG(4,
"ZSTD_compress_generic : transparent init stage");
3803 #ifdef ZSTD_MULTITHREAD 3807 if (
params.nbWorkers > 0) {
3809 if (cctx->mtctx ==
NULL) {
3810 DEBUGLOG(4,
"ZSTD_compress_generic: creating new mtctx for nbWorkers=%u",
3812 cctx->mtctx = ZSTDMT_createCCtx_advanced(
params.nbWorkers, cctx->
customMem);
3813 if (cctx->mtctx ==
NULL)
return ERROR(memory_allocation);
3816 DEBUGLOG(4,
"call ZSTDMT_initCStream_internal as nbWorkers=%u",
params.nbWorkers);
3817 CHECK_F( ZSTDMT_initCStream_internal(
3819 prefixDict.
dict, prefixDict.
dictSize, ZSTD_dct_rawContent,
3834 #ifdef ZSTD_MULTITHREAD 3837 ZSTDMT_updateCParams_whileCompressing(cctx->mtctx, &cctx->
requestedParams);
3840 {
size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx,
output,
input, endOp);
3842 || (endOp == ZSTD_e_end && flushMin == 0) ) {
3845 DEBUGLOG(5,
"completed ZSTD_compress_generic delegating to ZSTDMT_compressStream_generic");
3850 DEBUGLOG(5,
"completed ZSTD_compress_generic");
3856 void*
dst,
size_t dstCapacity,
size_t* dstPos,
3857 const void*
src,
size_t srcSize,
size_t* srcPos,
3858 ZSTD_EndDirective endOp)
3865 *srcPos =
input.pos;
3891 DEBUGLOG(4,
"ZSTD_endStream : remaining to flush : %u", (
U32)toFlush);
3899 #define ZSTD_MAX_CLEVEL 22 3906 { 19, 12, 13, 1, 6, 1, ZSTD_fast },
3907 { 19, 13, 14, 1, 7, 0, ZSTD_fast },
3908 { 19, 15, 16, 1, 6, 0, ZSTD_fast },
3909 { 20, 16, 17, 1, 5, 1, ZSTD_dfast },
3910 { 20, 18, 18, 1, 5, 1, ZSTD_dfast },
3911 { 20, 18, 18, 2, 5, 2, ZSTD_greedy },
3912 { 21, 18, 19, 2, 5, 4, ZSTD_lazy },
3913 { 21, 18, 19, 3, 5, 8, ZSTD_lazy2 },
3914 { 21, 19, 19, 3, 5, 16, ZSTD_lazy2 },
3915 { 21, 19, 20, 4, 5, 16, ZSTD_lazy2 },
3916 { 21, 20, 21, 4, 5, 16, ZSTD_lazy2 },
3917 { 21, 21, 22, 4, 5, 16, ZSTD_lazy2 },
3918 { 22, 20, 22, 5, 5, 16, ZSTD_lazy2 },
3919 { 22, 21, 22, 4, 5, 32, ZSTD_btlazy2 },
3920 { 22, 21, 22, 5, 5, 32, ZSTD_btlazy2 },
3921 { 22, 22, 22, 6, 5, 32, ZSTD_btlazy2 },
3922 { 22, 21, 22, 4, 5, 48, ZSTD_btopt },
3923 { 23, 22, 22, 4, 4, 64, ZSTD_btopt },
3924 { 23, 23, 22, 6, 3,256, ZSTD_btopt },
3925 { 23, 24, 22, 7, 3,256, ZSTD_btultra },
3926 { 25, 25, 23, 7, 3,256, ZSTD_btultra },
3927 { 26, 26, 24, 7, 3,512, ZSTD_btultra },
3928 { 27, 27, 25, 9, 3,999, ZSTD_btultra },
3932 { 18, 12, 13, 1, 5, 1, ZSTD_fast },
3933 { 18, 13, 14, 1, 6, 0, ZSTD_fast },
3934 { 18, 14, 14, 1, 5, 1, ZSTD_dfast },
3935 { 18, 16, 16, 1, 4, 1, ZSTD_dfast },
3936 { 18, 16, 17, 2, 5, 2, ZSTD_greedy },
3937 { 18, 18, 18, 3, 5, 2, ZSTD_greedy },
3938 { 18, 18, 19, 3, 5, 4, ZSTD_lazy },
3939 { 18, 18, 19, 4, 4, 4, ZSTD_lazy },
3940 { 18, 18, 19, 4, 4, 8, ZSTD_lazy2 },
3941 { 18, 18, 19, 5, 4, 8, ZSTD_lazy2 },
3942 { 18, 18, 19, 6, 4, 8, ZSTD_lazy2 },
3943 { 18, 18, 19, 5, 4, 16, ZSTD_btlazy2 },
3944 { 18, 19, 19, 6, 4, 16, ZSTD_btlazy2 },
3945 { 18, 19, 19, 8, 4, 16, ZSTD_btlazy2 },
3946 { 18, 18, 19, 4, 4, 24, ZSTD_btopt },
3947 { 18, 18, 19, 4, 3, 24, ZSTD_btopt },
3948 { 18, 19, 19, 6, 3, 64, ZSTD_btopt },
3949 { 18, 19, 19, 8, 3,128, ZSTD_btopt },
3950 { 18, 19, 19, 10, 3,256, ZSTD_btopt },
3951 { 18, 19, 19, 10, 3,256, ZSTD_btultra },
3952 { 18, 19, 19, 11, 3,512, ZSTD_btultra },
3953 { 18, 19, 19, 12, 3,512, ZSTD_btultra },
3954 { 18, 19, 19, 13, 3,999, ZSTD_btultra },
3958 { 17, 12, 12, 1, 5, 1, ZSTD_fast },
3959 { 17, 12, 13, 1, 6, 0, ZSTD_fast },
3960 { 17, 13, 15, 1, 5, 0, ZSTD_fast },
3961 { 17, 15, 16, 2, 5, 1, ZSTD_dfast },
3962 { 17, 17, 17, 2, 4, 1, ZSTD_dfast },
3963 { 17, 16, 17, 3, 4, 2, ZSTD_greedy },
3964 { 17, 17, 17, 3, 4, 4, ZSTD_lazy },
3965 { 17, 17, 17, 3, 4, 8, ZSTD_lazy2 },
3966 { 17, 17, 17, 4, 4, 8, ZSTD_lazy2 },
3967 { 17, 17, 17, 5, 4, 8, ZSTD_lazy2 },
3968 { 17, 17, 17, 6, 4, 8, ZSTD_lazy2 },
3969 { 17, 17, 17, 7, 4, 8, ZSTD_lazy2 },
3970 { 17, 18, 17, 6, 4, 16, ZSTD_btlazy2 },
3971 { 17, 18, 17, 8, 4, 16, ZSTD_btlazy2 },
3972 { 17, 18, 17, 4, 4, 32, ZSTD_btopt },
3973 { 17, 18, 17, 6, 3, 64, ZSTD_btopt },
3974 { 17, 18, 17, 7, 3,128, ZSTD_btopt },
3975 { 17, 18, 17, 7, 3,256, ZSTD_btopt },
3976 { 17, 18, 17, 8, 3,256, ZSTD_btopt },
3977 { 17, 18, 17, 8, 3,256, ZSTD_btultra },
3978 { 17, 18, 17, 9, 3,256, ZSTD_btultra },
3979 { 17, 18, 17, 10, 3,256, ZSTD_btultra },
3980 { 17, 18, 17, 11, 3,512, ZSTD_btultra },
3984 { 14, 12, 13, 1, 5, 1, ZSTD_fast },
3985 { 14, 14, 15, 1, 5, 0, ZSTD_fast },
3986 { 14, 14, 15, 1, 4, 0, ZSTD_fast },
3987 { 14, 14, 14, 2, 4, 1, ZSTD_dfast },
3988 { 14, 14, 14, 4, 4, 2, ZSTD_greedy },
3989 { 14, 14, 14, 3, 4, 4, ZSTD_lazy },
3990 { 14, 14, 14, 4, 4, 8, ZSTD_lazy2 },
3991 { 14, 14, 14, 6, 4, 8, ZSTD_lazy2 },
3992 { 14, 14, 14, 8, 4, 8, ZSTD_lazy2 },
3993 { 14, 15, 14, 5, 4, 8, ZSTD_btlazy2 },
3994 { 14, 15, 14, 9, 4, 8, ZSTD_btlazy2 },
3995 { 14, 15, 14, 3, 4, 12, ZSTD_btopt },
3996 { 14, 15, 14, 6, 3, 16, ZSTD_btopt },
3997 { 14, 15, 14, 6, 3, 24, ZSTD_btopt },
3998 { 14, 15, 15, 6, 3, 48, ZSTD_btopt },
3999 { 14, 15, 15, 6, 3, 64, ZSTD_btopt },
4000 { 14, 15, 15, 6, 3, 96, ZSTD_btopt },
4001 { 14, 15, 15, 6, 3,128, ZSTD_btopt },
4002 { 14, 15, 15, 8, 3,256, ZSTD_btopt },
4003 { 14, 15, 15, 6, 3,256, ZSTD_btultra },
4004 { 14, 15, 15, 8, 3,256, ZSTD_btultra },
4005 { 14, 15, 15, 9, 3,256, ZSTD_btultra },
4006 { 14, 15, 15, 10, 3,512, ZSTD_btultra },
4013 ZSTD_compressionParameters
ZSTD_getCParams(
int compressionLevel,
unsigned long long srcSizeHint,
size_t dictSize)
4015 size_t const addedSize = srcSizeHint ? 0 : 500;
4016 U64 const rSize = srcSizeHint+dictSize ? srcSizeHint+dictSize+addedSize : (
U64)-1;
4017 U32 const tableID = (rSize <= 256
KB) + (rSize <= 128
KB) + (rSize <= 16
KB);
4018 int row = compressionLevel;
4019 DEBUGLOG(5,
"ZSTD_getCParams (cLevel=%i)", compressionLevel);
4021 if (compressionLevel < 0)
row = 0;
4024 if (compressionLevel < 0)
cp.targetLength = (
unsigned)(-compressionLevel);
4032 ZSTD_parameters
ZSTD_getParams(
int compressionLevel,
unsigned long long srcSizeHint,
size_t dictSize) {
4034 ZSTD_compressionParameters
const cParams =
ZSTD_getCParams(compressionLevel, srcSizeHint, dictSize);
4035 DEBUGLOG(5,
"ZSTD_getParams (cLevel=%i)", compressionLevel);
4037 params.cParams = cParams;
4038 params.fParams.contentSizeFlag = 1;
FORCE_INLINE_TEMPLATE void ZSTD_reduceTable_internal(U32 *const table, U32 const size, U32 const reducerValue, int const preserveMark)
GLsizei GLenum const GLvoid GLsizei GLenum GLbyte GLbyte GLbyte GLdouble GLdouble GLdouble GLfloat GLfloat GLfloat GLint GLint GLint GLshort GLshort GLshort GLubyte GLubyte GLubyte GLuint GLuint GLuint GLushort GLushort GLushort GLbyte GLbyte GLbyte GLbyte GLdouble GLdouble GLdouble GLdouble GLfloat GLfloat GLfloat GLfloat GLint GLint GLint GLint GLshort GLshort GLshort GLshort GLubyte GLubyte GLubyte GLubyte GLuint GLuint GLuint GLuint GLushort GLushort GLushort GLushort GLboolean const GLdouble const GLfloat const GLint const GLshort const GLbyte const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLdouble const GLfloat const GLfloat const GLint const GLint const GLshort const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort const GLdouble const GLfloat const GLint const GLshort GLenum GLenum GLenum GLfloat GLenum GLint GLenum GLenum GLenum GLfloat GLenum GLenum GLint GLenum GLfloat GLenum GLint GLint GLushort GLenum GLenum GLfloat GLenum GLenum GLint GLfloat const GLubyte GLenum GLenum GLenum const GLfloat GLenum GLenum const GLint GLenum GLint GLint GLsizei GLsizei GLint GLenum GLenum const GLvoid GLenum GLenum const GLfloat GLenum GLenum const GLint GLenum GLenum const GLdouble GLenum GLenum const GLfloat GLenum GLenum const GLint GLsizei GLuint GLfloat GLuint GLbitfield GLfloat GLint GLuint GLboolean GLenum GLfloat GLenum GLbitfield GLenum GLfloat GLfloat GLint GLint const GLfloat GLenum GLfloat GLfloat GLint GLint GLfloat GLfloat GLint GLint const GLfloat GLint GLfloat GLfloat GLint GLfloat GLfloat GLint GLfloat GLfloat const GLdouble * u
size_t ZSTD_initCStream_srcSize(ZSTD_CStream *zcs, int compressionLevel, unsigned long long pss)
#define WILDCOPY_OVERLENGTH
int workSpaceOversizedDuration
FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
unsigned long long consumedSrcSize
MEM_STATIC void MEM_writeLE16(void *memPtr, U16 val)
rawSeqStore_t externSeqStore
MEM_STATIC size_t ZSTD_compressSequences(seqStore_t *seqStorePtr, const ZSTD_entropyCTables_t *prevEntropy, ZSTD_entropyCTables_t *nextEntropy, const ZSTD_CCtx_params *cctxParams, void *dst, size_t dstCapacity, size_t srcSize, U32 *workspace, int bmi2)
size_t HUF_compress1X_repeat(void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void *workSpace, size_t wkspSize, HUF_CElt *hufTable, HUF_repeat *repeat, int preferRepeat, int bmi2)
size_t FSE_buildCTable_wksp(FSE_CTable *ct, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workSpace, size_t wkspSize)
static size_t ZSTD_continueCCtx(ZSTD_CCtx *cctx, ZSTD_CCtx_params params, U64 pledgedSrcSize)
static U32 ZSTD_equivalentParams(ZSTD_CCtx_params params1, ZSTD_CCtx_params params2, size_t buffSize1, size_t maxNbSeq1, size_t maxNbLit1, ZSTD_buffered_policy_e buffPol2, U64 pledgedSrcSize)
size_t ZSTD_referenceExternalSequences(ZSTD_CCtx *cctx, rawSeq *seq, size_t nbSeq)
FSE_PUBLIC_API size_t FSE_normalizeCount(short *normalizedCounter, unsigned tableLog, const unsigned *count, size_t srcSize, unsigned maxSymbolValue)
static int ZSTD_shouldAttachDict(const ZSTD_CDict *cdict, ZSTD_CCtx_params params, U64 pledgedSrcSize)
size_t ZSTD_writeLastEmptyBlock(void *dst, size_t dstCapacity)
size_t ZSTD_initCStream_usingDict(ZSTD_CStream *zcs, const void *dict, size_t dictSize, int compressionLevel)
ZSTD_CCtx * ZSTD_createCCtx(void)
ZSTD_compressedBlockState_t cBlockState
static UCHAR ULONG UCHAR ULONG UCHAR * output
static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t *bs, ZSTD_matchState_t *ms, ZSTD_CCtx_params const *params, const void *dict, size_t dictSize, ZSTD_dictTableLoadMethod_e dtlm, void *workspace)
size_t ZSTD_compressBlock_lazy_dictMatchState(ZSTD_matchState_t *ms, seqStore_t *seqStore, U32 rep[ZSTD_REP_NUM], void const *src, size_t srcSize)
struct png_info_def **typedef void(__cdecl typeof(png_destroy_read_struct))(struct png_struct_def **
static size_t ZSTD_compressBlock_internal(ZSTD_CCtx *zc, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
unsigned long long pledgedSrcSizePlusOne
size_t ZSTD_flushStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output)
size_t ZSTD_compressBlock_btultra(ZSTD_matchState_t *ms, seqStore_t *seqStore, U32 rep[ZSTD_REP_NUM], const void *src, size_t srcSize)
void ZSTD_fillHashTable(ZSTD_matchState_t *ms, void const *end, ZSTD_dictTableLoadMethod_e dtlm)
void ZSTD_ldm_adjustParameters(ldmParams_t *params, ZSTD_compressionParameters const *cParams)
#define ZSTD_CLEVEL_DEFAULT
#define ZSTD_CONTENTSIZE_UNKNOWN
_Tp _STLP_CALL norm(const complex< _Tp > &__z)
ZSTD_CCtx_params * ZSTD_createCCtxParams(void)
#define HUF_WORKSPACE_SIZE_U32
static size_t ZSTD_compressLiterals(ZSTD_hufCTables_t const *prevHuf, ZSTD_hufCTables_t *nextHuf, ZSTD_strategy strategy, int disableLiteralCompression, void *dst, size_t dstCapacity, const void *src, size_t srcSize, U32 *workspace, const int bmi2)
#define TARGET_ATTRIBUTE(target)
size_t ZSTD_sizeof_CStream(const ZSTD_CStream *zcs)
static void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t *bs)
ZSTD_match_t * matchTable
size_t ZSTD_estimateCCtxSize(int compressionLevel)
size_t FSE_readNCount(short *normalizedCounter, unsigned *maxSVPtr, unsigned *tableLogPtr, const void *headerBuffer, size_t hbSize)
size_t ZSTD_compressBlock_fast_dictMatchState(ZSTD_matchState_t *ms, seqStore_t *seqStore, U32 rep[ZSTD_REP_NUM], void const *src, size_t srcSize)
void ZSTD_free(void *ptr, ZSTD_customMem customMem)
GLuint GLuint GLsizei count
static const U32 LL_bits[MaxLL+1]
MEM_STATIC U32 MEM_readLE32(const void *memPtr)
U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t *ms, const BYTE *ip)
size_t ZSTD_compressBlock(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
size_t ZSTD_compressContinue(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize)
size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel)
MEM_STATIC U32 ZSTD_window_hasExtDict(ZSTD_window_t const window)
#define ZSTD_WINDOWLOG_ABSOLUTEMIN
MEM_STATIC void ZSTD_window_clear(ZSTD_window_t *window)
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream *zcs, const ZSTD_CDict *cdict)
static size_t ZSTD_compressRleLiteralsBlock(void *dst, size_t dstCapacity, const void *src, size_t srcSize)
ZSTD_matchState_t matchState
static const U32 ML_defaultNormLog
MEM_STATIC unsigned MEM_32bits(void)
static size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat)
size_t ZSTD_compressStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output, ZSTD_inBuffer *input)
size_t ZSTD_compressBlock_lazy2_dictMatchState(ZSTD_matchState_t *ms, seqStore_t *seqStore, U32 rep[ZSTD_REP_NUM], void const *src, size_t srcSize)
MEM_STATIC void MEM_writeLE64(void *memPtr, U64 val64)
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx *cctx, const void *dict, size_t dictSize, int compressionLevel)
size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params *params)
size_t ZSTD_endStream(ZSTD_CStream *zcs, ZSTD_outBuffer *output)
#define ZSTD_LDM_DEFAULT_WINDOW_LOG
size_t ZSTD_compressBlock_doubleFast_extDict(ZSTD_matchState_t *ms, seqStore_t *seqStore, U32 rep[ZSTD_REP_NUM], void const *src, size_t srcSize)
size_t ZSTD_compress_advanced(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const void *dict, size_t dictSize, ZSTD_parameters params)
size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx *cctx, const void *prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType)
#define ZSTD_WORKSPACETOOLARGE_FACTOR
ZSTD_CDict * ZSTD_createCDict_byReference(const void *dict, size_t dictSize, int compressionLevel)
size_t ZSTD_compress_usingCDict(ZSTD_CCtx *cctx, void *dst, size_t dstCapacity, const void *src, size_t srcSize, const ZSTD_CDict *cdict)
MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void)
static size_t ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx *cctx, const ZSTD_CDict *cdict, ZSTD_CCtx_params params, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff)
U64 ZSTD_ldm_getHashPower(U32 minMatchLength)
size_t outBuffFlushedSize
size_t ZSTD_CStreamOutSize(void)
static U32 ZSTD_sufficientBuff(size_t bufferSize1, size_t maxNbSeq1, size_t maxNbLit1, ZSTD_buffered_policy_e buffPol2, ZSTD_compressionParameters cParams2, U64 pledgedSrcSize)
ZSTD_prefixDict prefixDict
MEM_STATIC size_t BIT_initCStream(BIT_CStream_t *bitC, void *dstBuffer, size