ReactOS  0.4.15-dev-5461-g062a8f2
zstd_ddict.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
3  * All rights reserved.
4  *
5  * This source code is licensed under both the BSD-style license (found in the
6  * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7  * in the COPYING file in the root directory of this source tree).
8  * You may select, at your option, one of the above-listed licenses.
9  */
10 
11 /* zstd_ddict.c :
12  * concentrates all logic that needs to know the internals of ZSTD_DDict object */
13 
14 /*-*******************************************************
15 * Dependencies
16 *********************************************************/
17 #include <string.h> /* memcpy, memmove, memset */
18 #include "cpu.h" /* bmi2 */
19 #include "mem.h" /* low level memory routines */
20 #define FSE_STATIC_LINKING_ONLY
21 #include "fse.h"
22 #define HUF_STATIC_LINKING_ONLY
23 #include "huf.h"
25 #include "zstd_ddict.h"
26 
27 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
28 # include "../legacy/zstd_legacy.h"
29 #endif
30 
31 
32 
33 /*-*******************************************************
34 * Types
35 *********************************************************/
36 struct ZSTD_DDict_s {
37  void* dictBuffer;
38  const void* dictContent;
39  size_t dictSize;
43  ZSTD_customMem cMem;
44 }; /* typedef'd to ZSTD_DDict within "zstd.h" */
45 
46 const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict)
47 {
48  assert(ddict != NULL);
49  return ddict->dictContent;
50 }
51 
52 size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict)
53 {
54  assert(ddict != NULL);
55  return ddict->dictSize;
56 }
57 
59 {
60  DEBUGLOG(4, "ZSTD_copyDDictParameters");
61  assert(dctx != NULL);
62  assert(ddict != NULL);
63  dctx->dictID = ddict->dictID;
64  dctx->prefixStart = ddict->dictContent;
65  dctx->virtualStart = ddict->dictContent;
66  dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize;
67  dctx->previousDstEnd = dctx->dictEnd;
68 #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
69  dctx->dictContentBeginForFuzzing = dctx->prefixStart;
70  dctx->dictContentEndForFuzzing = dctx->previousDstEnd;
71 #endif
72  if (ddict->entropyPresent) {
73  dctx->litEntropy = 1;
74  dctx->fseEntropy = 1;
75  dctx->LLTptr = ddict->entropy.LLTable;
76  dctx->MLTptr = ddict->entropy.MLTable;
77  dctx->OFTptr = ddict->entropy.OFTable;
78  dctx->HUFptr = ddict->entropy.hufTable;
79  dctx->entropy.rep[0] = ddict->entropy.rep[0];
80  dctx->entropy.rep[1] = ddict->entropy.rep[1];
81  dctx->entropy.rep[2] = ddict->entropy.rep[2];
82  } else {
83  dctx->litEntropy = 0;
84  dctx->fseEntropy = 0;
85  }
86 }
87 
88 
89 static size_t
91  ZSTD_dictContentType_e dictContentType)
92 {
93  ddict->dictID = 0;
94  ddict->entropyPresent = 0;
95  if (dictContentType == ZSTD_dct_rawContent) return 0;
96 
97  if (ddict->dictSize < 8) {
98  if (dictContentType == ZSTD_dct_fullDict)
99  return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
100  return 0; /* pure content mode */
101  }
102  { U32 const magic = MEM_readLE32(ddict->dictContent);
103  if (magic != ZSTD_MAGIC_DICTIONARY) {
104  if (dictContentType == ZSTD_dct_fullDict)
105  return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
106  return 0; /* pure content mode */
107  }
108  }
109  ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE);
110 
111  /* load entropy tables */
113  &ddict->entropy, ddict->dictContent, ddict->dictSize)),
114  dictionary_corrupted, "");
115  ddict->entropyPresent = 1;
116  return 0;
117 }
118 
119 
120 static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
121  const void* dict, size_t dictSize,
122  ZSTD_dictLoadMethod_e dictLoadMethod,
123  ZSTD_dictContentType_e dictContentType)
124 {
125  if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {
126  ddict->dictBuffer = NULL;
127  ddict->dictContent = dict;
128  if (!dict) dictSize = 0;
129  } else {
130  void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem);
131  ddict->dictBuffer = internalBuffer;
132  ddict->dictContent = internalBuffer;
133  if (!internalBuffer) return ERROR(memory_allocation);
134  memcpy(internalBuffer, dict, dictSize);
135  }
136  ddict->dictSize = dictSize;
137  ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
138 
139  /* parse dictionary content */
140  FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , "");
141 
142  return 0;
143 }
144 
145 ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
146  ZSTD_dictLoadMethod_e dictLoadMethod,
147  ZSTD_dictContentType_e dictContentType,
148  ZSTD_customMem customMem)
149 {
150  if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
151 
152  { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
153  if (ddict == NULL) return NULL;
154  ddict->cMem = customMem;
155  { size_t const initResult = ZSTD_initDDict_internal(ddict,
156  dict, dictSize,
157  dictLoadMethod, dictContentType);
158  if (ZSTD_isError(initResult)) {
159  ZSTD_freeDDict(ddict);
160  return NULL;
161  } }
162  return ddict;
163  }
164 }
165 
170 ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
171 {
172  ZSTD_customMem const allocator = { NULL, NULL, NULL };
173  return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator);
174 }
175 
180 ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
181 {
182  ZSTD_customMem const allocator = { NULL, NULL, NULL };
183  return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator);
184 }
185 
186 
188  void* sBuffer, size_t sBufferSize,
189  const void* dict, size_t dictSize,
190  ZSTD_dictLoadMethod_e dictLoadMethod,
191  ZSTD_dictContentType_e dictContentType)
192 {
193  size_t const neededSpace = sizeof(ZSTD_DDict)
194  + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
195  ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer;
196  assert(sBuffer != NULL);
197  assert(dict != NULL);
198  if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */
199  if (sBufferSize < neededSpace) return NULL;
200  if (dictLoadMethod == ZSTD_dlm_byCopy) {
201  memcpy(ddict+1, dict, dictSize); /* local copy */
202  dict = ddict+1;
203  }
205  dict, dictSize,
206  ZSTD_dlm_byRef, dictContentType) ))
207  return NULL;
208  return ddict;
209 }
210 
211 
213 {
214  if (ddict==NULL) return 0; /* support free on NULL */
215  { ZSTD_customMem const cMem = ddict->cMem;
216  ZSTD_free(ddict->dictBuffer, cMem);
217  ZSTD_free(ddict, cMem);
218  return 0;
219  }
220 }
221 
225 size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod)
226 {
227  return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
228 }
229 
230 size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
231 {
232  if (ddict==NULL) return 0; /* support sizeof on NULL */
233  return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ;
234 }
235 
240 unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
241 {
242  if (ddict==NULL) return 0;
243  return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
244 }
const void * previousDstEnd
struct ZSTD_DDict_s ZSTD_DDict
Definition: zstd.h:845
const void * dictContent
Definition: zstd_ddict.c:38
#define ERROR(name)
Definition: error_private.h:53
void ZSTD_free(void *ptr, ZSTD_customMem customMem)
Definition: zstd_common.c:70
MEM_STATIC U32 MEM_readLE32(const void *memPtr)
Definition: mem.h:346
const ZSTD_DDict * ZSTD_initStaticDDict(void *sBuffer, size_t sBufferSize, const void *dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType)
Definition: zstd_ddict.c:187
ZSTD_entropyDTables_t entropy
Definition: zstd_ddict.c:40
const ZSTD_seqSymbol * OFTptr
ZSTD_DDict * ZSTD_createDDict_byReference(const void *dictBuffer, size_t dictSize)
Definition: zstd_ddict.c:180
u32_t magic(void)
ZSTD_DDict * ZSTD_createDDict_advanced(const void *dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, ZSTD_customMem customMem)
Definition: zstd_ddict.c:145
#define assert(x)
Definition: debug.h:53
void * dictBuffer
Definition: zstd_ddict.c:37
ZSTD_DDict * ZSTD_createDDict(const void *dict, size_t dictSize)
Definition: zstd_ddict.c:170
ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]
if(dx==0 &&dy==0)
Definition: linetemp.h:174
size_t ZSTD_sizeof_DDict(const ZSTD_DDict *ddict)
Definition: zstd_ddict.c:230
static size_t ZSTD_loadEntropy_intoDDict(ZSTD_DDict *ddict, ZSTD_dictContentType_e dictContentType)
Definition: zstd_ddict.c:90
U32 entropyPresent
Definition: zstd_ddict.c:42
#define DEBUGLOG(l,...)
Definition: debug.h:106
const void * virtualStart
#define ZSTD_isError
Definition: zstd_internal.h:46
const void * prefixStart
void ZSTD_copyDDictParameters(ZSTD_DCtx *dctx, const ZSTD_DDict *ddict)
Definition: zstd_ddict.c:58
#define HufLog
ZSTD_customMem cMem
Definition: zstd_ddict.c:43
const ZSTD_seqSymbol * LLTptr
const ZSTD_seqSymbol * MLTptr
#define RETURN_ERROR_IF(cond, err,...)
Definition: zstd_internal.h:91
ZSTD_entropyDTables_t entropy
ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]
size_t dictSize
Definition: zstd_ddict.c:39
HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]
size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t *entropy, const void *const dict, size_t const dictSize)
size_t ZSTD_DDict_dictSize(const ZSTD_DDict *ddict)
Definition: zstd_ddict.c:52
#define memcpy(s1, s2, n)
Definition: mkisofs.h:878
#define ZSTD_MAGIC_DICTIONARY
Definition: zstd.h:99
void * ZSTD_malloc(size_t size, ZSTD_customMem customMem)
Definition: zstd_common.c:56
unsigned char BYTE
Definition: xxhash.c:193
ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void *dict, size_t dictSize)
size_t ZSTD_freeDDict(ZSTD_DDict *ddict)
Definition: zstd_ddict.c:212
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict *ddict)
Definition: zstd_ddict.c:240
#define ZSTD_FRAMEIDSIZE
ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]
static size_t ZSTD_initDDict_internal(ZSTD_DDict *ddict, const void *dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType)
Definition: zstd_ddict.c:120
#define NULL
Definition: types.h:112
#define FORWARD_IF_ERROR(err,...)
const HUF_DTable * HUFptr
size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod)
Definition: zstd_ddict.c:225
const void * ZSTD_DDict_dictContent(const ZSTD_DDict *ddict)
Definition: zstd_ddict.c:46
unsigned int U32
Definition: xxhash.c:195