facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/Cyan4973/dev070' into dev070 

# Conflicts:
#	.gitignore
#	lib/decompress/zstd_decompress.c
#	programs/zbufftest.c
dev
inikep 2016-06-01 09:16:11 +02:00
parent 00fa1ae0c0 202844ebd0
commit 5c2771710d
11 changed files with 138 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -31,7 +31,7 @@ projects/VS2015
.directory
_codelite/
_zstdbench/
zlib_wrapper/
.clang_complete
# CMake
projects/cmake/

1
lib/common/error_private.h
View File

@ -102,6 +102,7 @@ ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
case PREFIX(srcSize_wrong): return "Src size incorrect";
case PREFIX(corruption_detected): return "Corrupted block detected";
case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";

1
lib/common/error_public.h
View File

@ -54,6 +54,7 @@ typedef enum {
ZSTD_error_dstSize_tooSmall,
ZSTD_error_srcSize_wrong,
ZSTD_error_corruption_detected,
ZSTD_error_checksum_wrong,
ZSTD_error_tableLog_tooLarge,
ZSTD_error_maxSymbolValue_tooLarge,
ZSTD_error_maxSymbolValue_tooSmall,

2
lib/common/zstd_static.h
View File

@ -88,6 +88,7 @@ typedef struct {
typedef struct {
U32 contentSizeFlag; /* 1: content size will be in frame header (if known). */
U32 checksumFlag; /* 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
U32 noDictIDFlag; /* 1: no dict ID will be saved into frame header (if dictionary compression) */
} ZSTD_frameParameters;
@ -197,6 +198,7 @@ typedef struct {
U64 frameContentSize;
U32 windowLog;
U32 dictID;
U32 checksumFlag;
} ZSTD_frameParams;
#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */

1
lib/compress/zbuff_compress.c
View File

@ -180,6 +180,7 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel)
{
ZSTD_parameters params;
memset(&params, 0, sizeof(params));
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
params.fParams.contentSizeFlag = 0;
ZSTD_adjustCParams(&params.cParams, 0, dictSize);

94
lib/compress/zstd_compress.c
View File

@ -54,6 +54,8 @@
#include <stdlib.h> /* malloc */
#include <string.h> /* memset */
#include "mem.h"
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
#include "xxhash.h" /* XXH_reset, update, digest */
#include "fse_static.h"
#include "huf_static.h"
#include "zstd_internal.h"
@ -104,6 +106,7 @@ struct ZSTD_CCtx_s
void* workSpace;
size_t workSpaceSize;
size_t blockSize;
XXH64_state_t xxhState;
ZSTD_allocFunction customAlloc;
ZSTD_freeFunction customFree;
@ -266,6 +269,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
} }
if (reset) memset(zc->workSpace, 0, tableSpace ); /* reset only tables */
XXH64_reset(&zc->xxhState, 0);
zc->hashTable3 = (U32*)(zc->workSpace);
zc->hashTable = zc->hashTable3 + h3Size;
zc->chainTable = zc->hashTable + hSize;
@ -1938,7 +1942,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
/* catch up */
if (offset) {
U32 matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE));
U32 const matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE));
const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex;
const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart;
while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */
@ -2043,19 +2047,22 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa
static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
size_t blockSize = zc->blockSize;
size_t blockSize = cctx->blockSize;
size_t remaining = srcSize;
const BYTE* ip = (const BYTE*)src;
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
const U32 maxDist = 1 << zc->params.cParams.windowLog;
ZSTD_stats_t* stats = &zc->seqStore.stats;
const U32 maxDist = 1 << cctx->params.cParams.windowLog;
ZSTD_stats_t* stats = &cctx->seqStore.stats;
ZSTD_statsInit(stats);
if (cctx->params.fParams.checksumFlag)
XXH64_update(&cctx->xxhState, src, srcSize);
while (remaining) {
size_t cSize;
ZSTD_statsResetFreqs(stats);
@ -2063,14 +2070,14 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
if (remaining < blockSize) blockSize = remaining;
if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) {
if ((U32)(ip+blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) {
/* enforce maxDist */
U32 const newLowLimit = (U32)(ip+blockSize - zc->base) - maxDist;
if (zc->lowLimit < newLowLimit) zc->lowLimit = newLowLimit;
if (zc->dictLimit < zc->lowLimit) zc->dictLimit = zc->lowLimit;
U32 const newLowLimit = (U32)(ip+blockSize - cctx->base) - maxDist;
if (cctx->lowLimit < newLowLimit) cctx->lowLimit = newLowLimit;
if (cctx->dictLimit < cctx->lowLimit) cctx->dictLimit = cctx->lowLimit;
}
cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize);
cSize = ZSTD_compressBlock_internal(cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize);
if (ZSTD_isError(cSize)) return cSize;
if (cSize == 0) { /* block is not compressible */
@ -2090,7 +2097,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
op += cSize;
}
ZSTD_statsPrint(stats, zc->params.cParams.searchLength);
ZSTD_statsPrint(stats, cctx->params.cParams.searchLength);
return op-ostart;
}
@ -2104,7 +2111,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
BYTE const fAllocByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) /* windowLog : 4 KB - 128 MB */
| (fcsId << 6) );
U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */
BYTE const fCheckByte = (BYTE)(dictIDSizeCode&3);
BYTE const fCheckByte = (BYTE)((dictIDSizeCode&3) + (params.fParams.checksumFlag<<4));
size_t pos;
if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall);
@ -2261,40 +2268,45 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
{
/* note : magic number already checked */
size_t offcodeHeaderSize, matchlengthHeaderSize, litlengthHeaderSize, errorCode;
short offcodeNCount[MaxOff+1];
unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog;
short matchlengthNCount[MaxML+1];
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
short litlengthNCount[MaxLL+1];
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
size_t const dictSizeStart = dictSize;
size_t const hufHeaderSize = HUF_readCTable(zc->hufTable, 255, dict, dictSize);
{ size_t const hufHeaderSize = HUF_readCTable(zc->hufTable, 255, dict, dictSize);
if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted);
zc->flagStaticTables = 1;
dict = (const char*)dict + hufHeaderSize;
dictSize -= hufHeaderSize;
}
offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
{ short offcodeNCount[MaxOff+1];
unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog;
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
errorCode = FSE_buildCTable(zc->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSE_buildCTable(zc->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
dict = (const char*)dict + offcodeHeaderSize;
dictSize -= offcodeHeaderSize;
}
matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
{ short matchlengthNCount[MaxML+1];
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
errorCode = FSE_buildCTable(zc->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSE_buildCTable(zc->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
dict = (const char*)dict + matchlengthHeaderSize;
dictSize -= matchlengthHeaderSize;
}
litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
{ short litlengthNCount[MaxLL+1];
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
errorCode = FSE_buildCTable(zc->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSE_buildCTable(zc->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
dictSize -= litlengthHeaderSize;
}
return hufHeaderSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize;
return (dictSizeStart-dictSize);
}
/** ZSTD_compress_insertDictionary() :
@ -2366,30 +2378,34 @@ size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel)
/*! ZSTD_compressEnd() :
* Write frame epilogue.
* @return : nb of bytes written into dst (or an error code) */
size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t dstCapacity)
size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
{
BYTE* op = (BYTE*)dst;
size_t fhSize = 0;
/* not even init ! */
if (zc->stage==0) return ERROR(stage_wrong);
if (cctx->stage==0) return ERROR(stage_wrong);
/* special case : empty frame */
if (zc->stage==1) {
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, 0, 0);
if (cctx->stage==1) {
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, 0, 0);
if (ZSTD_isError(fhSize)) return fhSize;
dstCapacity -= fhSize;
op += fhSize;
zc->stage = 2;
cctx->stage = 2;
}
/* frame epilogue */
if (dstCapacity < 3) return ERROR(dstSize_tooSmall);
op[0] = (BYTE)(bt_end << 6);
op[1] = 0;
op[2] = 0;
{ U32 const checksum = cctx->params.fParams.checksumFlag ?
(U32)((XXH64_digest(&cctx->xxhState) >> 11) & ((1<<22)-1)) :
0;
op[0] = (BYTE)((bt_end<<6) + (checksum>>16));
op[1] = (BYTE)(checksum>>8);
op[2] = (BYTE)checksum;
}
zc->stage = 0; /* return to "created by not init" status */
cctx->stage = 0; /* return to "created but not init" status */
return 3+fhSize;
}

49
lib/decompress/zstd_decompress.c
View File

@ -57,6 +57,8 @@
#include <string.h> /* memcpy, memmove */
#include <stdio.h> /* debug only : printf */
#include "mem.h" /* low level memory routines */
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
#include "xxhash.h" /* XXH64_* */
#include "zstd_internal.h"
#include "fse_static.h"
#include "huf_static.h"
@ -117,6 +119,7 @@ struct ZSTD_DCtx_s
size_t expected;
size_t headerSize;
ZSTD_frameParams fParams;
XXH64_state_t xxhState;
ZSTD_allocFunction customAlloc;
ZSTD_freeFunction customFree;
blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
@ -146,12 +149,6 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
return 0;
}
ZSTD_DCtx* ZSTD_createDCtx(void)
{
ZSTD_customMem customMem = { NULL, NULL };
return ZSTD_createDCtx_advanced(customMem);
}
ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
{
ZSTD_DCtx* dctx;
@ -178,6 +175,12 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
return dctx;
}
ZSTD_DCtx* ZSTD_createDCtx(void)
{
ZSTD_customMem const customMem = { NULL, NULL };
return ZSTD_createDCtx_advanced(customMem);
}
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
{
@ -346,8 +349,10 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
BYTE const checkByte = ip[5];
size_t pos = ZSTD_frameHeaderSize_min;
U32 const dictIDSizeCode = checkByte&3;
fparamsPtr->windowLog = (allocByte & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
if ((allocByte & 0x30) != 0) return ERROR(frameParameter_unsupported); /* reserved bits */
if ((checkByte & 0xEC) != 0) return ERROR(frameParameter_unsupported); /* reserved bits */
fparamsPtr->windowLog = (allocByte & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
fparamsPtr->checksumFlag = checkByte & 0x10;
switch(dictIDSizeCode) /* fcsId */
{
default: /* impossible */
@ -376,6 +381,7 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t sr
size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, srcSize);
if ((MEM_32bits()) && (dctx->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits);
if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong);
if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0);
return result;
}
@ -964,6 +970,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
if (cBlockSize == 0) break; /* bt_end */
if (ZSTD_isError(decodedSize)) return decodedSize;
if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, op, decodedSize);
op += decodedSize;
ip += cBlockSize;
remainingSize -= cBlockSize;
@ -1035,6 +1042,9 @@ int ZSTD_isSkipFrame(ZSTD_DCtx* dctx)
return dctx->stage == ZSTDds_skipFrame;
}
/** ZSTD_decompressContinue() :
* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
* or an error code, which can be tested using ZSTD_isError() */
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
/* Sanity check */
@ -1076,6 +1086,13 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
if (ZSTD_isError(cBlockSize)) return cBlockSize;
if (bp.blockType == bt_end) {
if (dctx->fParams.checksumFlag) {
U64 const h64 = XXH64_digest(&dctx->xxhState);
U32 const h32 = (U32)(h64>>11) & ((1<<22)-1);
const BYTE* const ip = (const BYTE*)src;
U32 const check32 = ip[2] + (ip[1] << 8) + ((ip[0] & 0x3F) << 16);
if (check32 != h32) return ERROR(checksum_wrong);
}
dctx->expected = 0;
dctx->stage = ZSTDds_getFrameHeaderSize;
} else {
@ -1107,6 +1124,8 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
dctx->stage = ZSTDds_decodeBlockHeader;
dctx->expected = ZSTD_blockHeaderSize;
dctx->previousDstEnd = (char*)dst + rSize;
if (ZSTD_isError(rSize)) return rSize;
if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize);
return rSize;
}
case ZSTDds_decodeSkippableHeader:
@ -1134,18 +1153,19 @@ static void ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSi
dctx->previousDstEnd = (const char*)dict + dictSize;
}
static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t const dictSizeStart)
{
size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, litlengthHeaderSize;
size_t dictSize = dictSizeStart;
hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize);
{ size_t const hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize);
if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
dict = (const char*)dict + hSize;
dictSize -= hSize;
}
{ short offcodeNCount[MaxOff+1];
U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
@ -1155,7 +1175,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSiz
{ short matchlengthNCount[MaxML+1];
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
@ -1165,14 +1185,15 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSiz
{ short litlengthNCount[MaxLL+1];
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
dictSize -= litlengthHeaderSize;
}
dctx->flagRepeatTable = 1;
return hSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize;
return dictSizeStart - dictSize;
}
static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)

3
programs/.clang_complete
View File

@ -1,3 +0,0 @@
-I../lib/common
-I../lib/legacy
-I./legacy

1
programs/.gitignore vendored
View File

@ -40,6 +40,7 @@ grillResults.txt
_*
tmp*
*.zst
result
# fuzzer
afl

10
programs/fuzzer.c
View File

@ -216,7 +216,7 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++);
{ size_t const testSize = CNBuffSize / 3;
{ ZSTD_compressionParameters const cPar = ZSTD_getCParams(2, testSize, dictSize);
ZSTD_frameParameters const fPar = { 1 , 0 };
ZSTD_frameParameters const fPar = { 1 , 0 , 0 };
ZSTD_parameters p;
p.cParams = cPar; p.fParams = fPar;
CHECK( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) );
@ -276,7 +276,7 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : compress without dictID : ", testNb++);
{ ZSTD_frameParameters const fParams = { 0, 1 /*NoDictID*/ };
{ ZSTD_frameParameters const fParams = { 0 /*contentSize*/, 0 /*checksum*/, 1 /*NoDictID*/ };
ZSTD_compressionParameters const cParams = ZSTD_getCParams(3, CNBuffSize, dictSize);
ZSTD_parameters p;
p.cParams = cParams; p.fParams = fParams;
@ -639,12 +639,14 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
dictSize = FUZ_randomLength(&lseed, maxSampleLog); /* needed also for decompression */
dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize));
if (FUZ_rand(&lseed) & 15) {
if (FUZ_rand(&lseed) & 0xF) {
size_t const errorCode = ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel);
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode));
} else {
ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize);
ZSTD_frameParameters const fpar = { FUZ_rand(&lseed)&1, FUZ_rand(&lseed)&1 }; /* note : since dictionary is fake, dictIDflag has no impact */
ZSTD_frameParameters const fpar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
!(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/,
0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */
ZSTD_parameters p;
size_t errorCode;
p.cParams = cPar; p.fParams = fpar;

26
programs/zbufftest.c
View File

@ -97,7 +97,7 @@ static U32 FUZ_GetMilliStart(void)
static U32 FUZ_GetMilliSpan(U32 nTimeStart)
{
U32 nCurrent = FUZ_GetMilliStart();
U32 const nCurrent = FUZ_GetMilliStart();
U32 nSpan = nCurrent - nTimeStart;
if (nTimeStart > nCurrent)
nSpan += 0x100000 * 1000;
@ -152,7 +152,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
void* compressedBuffer = malloc(compressedBufferSize);
size_t const decodedBufferSize = CNBufferSize;
void* decodedBuffer = malloc(decodedBufferSize);
size_t result, cSize, readSize, readSkipSize, genSize;
size_t cSize, readSize, readSkipSize, genSize;
U32 testNb=0;
ZBUFF_CCtx* zc = ZBUFF_createCCtx_advanced(customMem);
ZBUFF_DCtx* zd = ZBUFF_createDCtx_advanced(customMem);
@ -173,13 +173,13 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1);
readSize = CNBufferSize;
genSize = compressedBufferSize;
result = ZBUFF_compressContinue(zc, ((char*)compressedBuffer)+cSize, &genSize, CNBuffer, &readSize);
if (ZBUFF_isError(result)) goto _output_error;
{ size_t const r = ZBUFF_compressContinue(zc, ((char*)compressedBuffer)+cSize, &genSize, CNBuffer, &readSize);
if (ZBUFF_isError(r)) goto _output_error; }
if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */
cSize += genSize;
genSize = compressedBufferSize - cSize;
result = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize);
if (result != 0) goto _output_error; /* error, or some data not flushed */
{ size_t const r = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize);
if (r != 0) goto _output_error; } /*< error, or some data not flushed */
cSize += genSize;
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
@ -188,13 +188,14 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
readSkipSize = cSize;
genSize = CNBufferSize;
result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSkipSize);
if (genSize != 0) goto _output_error; /* skippable frame */
{ size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSkipSize);
if (r != 0) goto _output_error; }
if (genSize != 0) goto _output_error; /* skippable frame len is 0 */
ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
readSize = cSize - readSkipSize;
genSize = CNBufferSize;
result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, ((char*)compressedBuffer)+readSkipSize, &readSize);
if (result != 0) goto _output_error; /* should reach end of frame == 0; otherwise, some data left, or an error */
{ size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, ((char*)compressedBuffer)+readSkipSize, &readSize);
if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */
if (readSize+readSkipSize != cSize) goto _output_error; /* should have read the entire frame */
DISPLAYLEVEL(4, "OK \n");
@ -613,9 +614,10 @@ int main(int argc, const char** argv)
if (testNb==0) {
result = basicUnitTests(0, ((double)proba) / 100, customNULL); /* constant seed for predictability */
if (!result)
if (!result) {
DISPLAYLEVEL(4, "Unit tests using customMem :\n")
result = basicUnitTests(0, ((double)proba) / 100, customMem); /* use custom memory allocation functions */
}
} }
if (!result)
result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);