facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix 2 bugs in dictionary loading

Browse Source 
* Silently skip dictionaries less than 8 bytes, unless using `ZSTD_dct_fullDict`.
  This changes the compressor, which silently skips dictionaries <= 8 bytes.
* Allow repcodes that are equal to the dictionary content size, since it is in bounds.
This commit is contained in:
Nick Terrell 2019-11-01 15:33:39 -07:00
parent 75e7c0d107
commit 60205fec02
3 changed files with 11 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
lib/compress/zstd_compress.c
View File

@ -2771,7 +2771,7 @@ static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSym
/*! ZSTD_loadZstdDictionary() : /*! ZSTD_loadZstdDictionary() :
* @return : dictID, or an error code * @return : dictID, or an error code
* assumptions : magic number supposed already checked * assumptions : magic number supposed already checked
* dictSize supposed > 8 * dictSize supposed >= 8
*/ */
static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
ZSTD_matchState_t* ms, ZSTD_matchState_t* ms,
@ -2788,7 +2788,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
size_t dictID; size_t dictID;
ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog))); ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog)));
assert(dictSize > 8); assert(dictSize >= 8);
assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY); assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY);
dictPtr += 4; /* skip magic number */ dictPtr += 4; /* skip magic number */
@ -2890,7 +2890,10 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs,
void* workspace) void* workspace)
{ {
DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize); DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize);
if ((dict==NULL) || (dictSize<=8)) return 0; if ((dict==NULL) || (dictSize<8)) {
RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong);
return 0;
}
ZSTD_reset_compressedBlockState(bs); ZSTD_reset_compressedBlockState(bs);
@ -2942,7 +2945,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, *params, pledgedSrcSize, FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, *params, pledgedSrcSize,
ZSTDcrp_makeClean, zbuff) ); ZSTDcrp_makeClean, zbuff) );
{ size_t const dictID = cdict ? { size_t const dictID = cdict ?
ZSTD_compress_insertDictionary( ZSTD_compress_insertDictionary(
cctx->blockState.prevCBlock, &cctx->blockState.matchState, cctx->blockState.prevCBlock, &cctx->blockState.matchState,
&cctx->workspace, params, cdict->dictContent, cdict->dictContentSize, &cctx->workspace, params, cdict->dictContent, cdict->dictContentSize,
@ -3219,7 +3222,7 @@ static size_t ZSTD_initCDict_internal(
ZSTDirp_reset, ZSTDirp_reset,
ZSTD_resetTarget_CDict)); ZSTD_resetTarget_CDict));
/* (Maybe) load the dictionary /* (Maybe) load the dictionary
* Skips loading the dictionary if it is <= 8 bytes. * Skips loading the dictionary if it is < 8 bytes.
*/ */
{ ZSTD_CCtx_params params; { ZSTD_CCtx_params params;
memset(&params, 0, sizeof(params)); memset(&params, 0, sizeof(params));

4
lib/decompress/zstd_decompress.c
View File

@ -1096,7 +1096,7 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12)); size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12));
for (i=0; i<3; i++) { for (i=0; i<3; i++) {
U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4; U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4;
RETURN_ERROR_IF(rep==0 || rep >= dictContentSize, RETURN_ERROR_IF(rep==0 || rep > dictContentSize,
dictionary_corrupted); dictionary_corrupted);
entropy->rep[i] = rep; entropy->rep[i] = rep;
} } } }
@ -1265,7 +1265,7 @@ size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx,
{ {
RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong); RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
ZSTD_clearDict(dctx); ZSTD_clearDict(dctx);
if (dict && dictSize >= 8) { if (dict && dictSize != 0) {
dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem); dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem);
RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation); RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation);
dctx->ddict = dctx->ddictLocal; dctx->ddict = dctx->ddictLocal;

2
tests/fuzz/Makefile
View File

@ -74,7 +74,7 @@ FUZZ_TARGETS := \
dictionary_decompress \ dictionary_decompress \
zstd_frame_info \ zstd_frame_info \
simple_compress \ simple_compress \
dict_loader dictionary_loader
all: $(FUZZ_TARGETS) all: $(FUZZ_TARGETS)