facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove formatting-only changes

Browse Source
This commit is contained in:
Stella Lau 2017-08-23 10:24:19 -07:00
parent 11303778d0
commit 6f1a21c7e9
6 changed files with 28 additions and 327 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
lib/common/zstd_internal.h
View File

@ -304,7 +304,6 @@ struct ZSTD_CCtx_params_s {
unsigned overlapSizeLog; unsigned overlapSizeLog;
}; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */ }; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); void ZSTD_seqToCodes(const seqStore_t* seqStorePtr);

53
lib/compress/zstd_compress.c
View File

@ -215,6 +215,7 @@ size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned
} }
} }
#define ZSTD_CLEVEL_CUSTOM 999 #define ZSTD_CLEVEL_CUSTOM 999
static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx) static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx)
{ {
@ -3151,10 +3152,8 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
if (cctx->stage==ZSTDcs_created) return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */ if (cctx->stage==ZSTDcs_created) return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */
if (frame && (cctx->stage==ZSTDcs_init)) { if (frame && (cctx->stage==ZSTDcs_init)) {
fhSize = ZSTD_writeFrameHeader( fhSize = ZSTD_writeFrameHeader(dst, dstCapacity,cctx->appliedParams,
dst, dstCapacity, cctx->pledgedSrcSizePlusOne-1, cctx->dictID);
cctx->appliedParams,
cctx->pledgedSrcSizePlusOne-1, cctx->dictID);
if (ZSTD_isError(fhSize)) return fhSize; if (ZSTD_isError(fhSize)) return fhSize;
dstCapacity -= fhSize; dstCapacity -= fhSize;
dst = (char*)dst + fhSize; dst = (char*)dst + fhSize;
@ -3444,8 +3443,7 @@ size_t ZSTD_compressBegin_advanced_internal(
* @return : 0, or an error code */ * @return : 0, or an error code */
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_parameters params, ZSTD_parameters params, unsigned long long pledgedSrcSize)
unsigned long long pledgedSrcSize)
{ {
ZSTD_CCtx_params cctxParams = ZSTD_CCtx_params cctxParams =
ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params);
@ -3571,11 +3569,8 @@ size_t ZSTD_compress_advanced_internal(
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
} }
size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize,
size_t dstCapacity, const void* dict, size_t dictSize, int compressionLevel)
const void* src, size_t srcSize,
const void* dict, size_t dictSize,
int compressionLevel)
{ {
ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, dict ? dictSize : 0); ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, dict ? dictSize : 0);
params.fParams.contentSizeFlag = 1; params.fParams.contentSizeFlag = 1;
@ -3606,8 +3601,7 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, unsigned byReference) size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, unsigned byReference)
{ {
DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (U32)sizeof(ZSTD_CDict)); DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (U32)sizeof(ZSTD_CDict));
DEBUGLOG(5, "CCtx estimate : %u", DEBUGLOG(5, "CCtx estimate : %u", (U32)ZSTD_estimateCCtxSize_advanced(cParams));
(U32)ZSTD_estimateCCtxSize_advanced(cParams));
return sizeof(ZSTD_CDict) + ZSTD_estimateCCtxSize_advanced(cParams) return sizeof(ZSTD_CDict) + ZSTD_estimateCCtxSize_advanced(cParams)
+ (byReference ? 0 : dictSize); + (byReference ? 0 : dictSize);
} }
@ -3645,7 +3639,7 @@ static size_t ZSTD_initCDict_internal(
} }
cdict->dictContentSize = dictSize; cdict->dictContentSize = dictSize;
{ ZSTD_frameParameters const fParams = { 0 /*contentSizeFlag */, { ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */,
0 /* checksumFlag */, 0 /* noDictIDFlag */ }; /* dummy */ 0 /* checksumFlag */, 0 /* noDictIDFlag */ }; /* dummy */
/* TODO: correct? */ /* TODO: correct? */
ZSTD_CCtx_params cctxParams = cdict->refContext->requestedParams; ZSTD_CCtx_params cctxParams = cdict->refContext->requestedParams;
@ -3666,7 +3660,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
unsigned byReference, ZSTD_dictMode_e dictMode, unsigned byReference, ZSTD_dictMode_e dictMode,
ZSTD_compressionParameters cParams, ZSTD_customMem customMem) ZSTD_compressionParameters cParams, ZSTD_customMem customMem)
{ {
DEBUGLOG(5, "ZSTD_createCDict_advanced, mode %u", dictMode); DEBUGLOG(5, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode);
if (!customMem.customAlloc ^ !customMem.customFree) return NULL; if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
{ ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); { ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
@ -3677,8 +3671,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
return NULL; return NULL;
} }
cdict->refContext = cctx; cdict->refContext = cctx;
if (ZSTD_isError( ZSTD_initCDict_internal( if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
cdict,
dictBuffer, dictSize, dictBuffer, dictSize,
byReference, dictMode, byReference, dictMode,
cParams) )) { cParams) )) {
@ -3735,8 +3728,7 @@ ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize,
ZSTD_compressionParameters cParams) ZSTD_compressionParameters cParams)
{ {
size_t const cctxSize = ZSTD_estimateCCtxSize_advanced(cParams); size_t const cctxSize = ZSTD_estimateCCtxSize_advanced(cParams);
size_t const neededSize = sizeof(ZSTD_CDict) size_t const neededSize = sizeof(ZSTD_CDict) + (byReference ? 0 : dictSize)
+ (byReference ? 0 : dictSize)
+ cctxSize; + cctxSize;
ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace; ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace;
void* ptr; void* ptr;
@ -3755,9 +3747,10 @@ ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize,
} }
cdict->refContext = ZSTD_initStaticCCtx(ptr, cctxSize); cdict->refContext = ZSTD_initStaticCCtx(ptr, cctxSize);
if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dict, dictSize, if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
1 /* byReference */, dict, dictSize,
dictMode, cParams) )) 1 /* byReference */, dictMode,
cParams) ))
return NULL; return NULL;
return cdict; return cdict;
@ -3897,12 +3890,10 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
* Note : not static, but hidden (not exposed). Used by zstdmt_compress.c * Note : not static, but hidden (not exposed). Used by zstdmt_compress.c
* Assumption 1 : params are valid * Assumption 1 : params are valid
* Assumption 2 : either dict, or cdict, is defined, not both */ * Assumption 2 : either dict, or cdict, is defined, not both */
size_t ZSTD_initCStream_internal( size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
ZSTD_CStream* zcs, const void* dict, size_t dictSize,
const void* dict, size_t dictSize, const ZSTD_CDict* cdict,
const ZSTD_CDict* cdict, ZSTD_CCtx_params params, unsigned long long pledgedSrcSize)
ZSTD_CCtx_params params,
unsigned long long pledgedSrcSize)
{ {
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
assert(!((dict) && (cdict))); /* either dict or cdict, not both */ assert(!((dict) && (cdict))); /* either dict or cdict, not both */
@ -3931,10 +3922,10 @@ size_t ZSTD_initCStream_internal(
zcs->cdictLocal = NULL; zcs->cdictLocal = NULL;
zcs->cdict = cdict; zcs->cdict = cdict;
} }
zcs->requestedParams = params; zcs->requestedParams = params;
return ZSTD_resetCStream_internal( return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->cdict, params, pledgedSrcSize);
zcs, NULL, 0, zcs->cdict, params, pledgedSrcSize);
} }
/* ZSTD_initCStream_usingCDict_advanced() : /* ZSTD_initCStream_usingCDict_advanced() :
@ -4167,6 +4158,7 @@ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
const void* dict, size_t dictSize, const ZSTD_CDict* cdict, const void* dict, size_t dictSize, const ZSTD_CDict* cdict,
ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); ZSTD_CCtx_params params, unsigned long long pledgedSrcSize);
size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
ZSTD_outBuffer* output, ZSTD_outBuffer* output,
ZSTD_inBuffer* input, ZSTD_inBuffer* input,
@ -4182,7 +4174,6 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
const void* const prefix = cctx->prefix; const void* const prefix = cctx->prefix;
size_t const prefixSize = cctx->prefixSize; size_t const prefixSize = cctx->prefixSize;
ZSTD_CCtx_params params = cctx->requestedParams; ZSTD_CCtx_params params = cctx->requestedParams;
if (params.compressionLevel != ZSTD_CLEVEL_CUSTOM) if (params.compressionLevel != ZSTD_CLEVEL_CUSTOM)
params.cParams = ZSTD_getCParams(params.compressionLevel, params.cParams = ZSTD_getCParams(params.compressionLevel,
cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/); cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/);

4
lib/compress/zstdmt_compress.c
View File

@ -62,7 +62,7 @@ static unsigned long long GetCurrentClockTimeMicroseconds(void)
DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \ DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \
elapsedTime, #mutex); \ elapsedTime, #mutex); \
} } \ } } \
} else { pthread_mutex_lock(mutex); } \ } else pthread_mutex_lock(mutex); \
} }
#else #else
@ -743,7 +743,7 @@ size_t ZSTDMT_initCStream_internal(
ZSTD_freeCDict(zcs->cdictLocal); ZSTD_freeCDict(zcs->cdictLocal);
/* TODO: cctxParam version? Is this correct? */ /* TODO: cctxParam version? Is this correct? */
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
0 /* byRef */, ZSTD_dm_auto, /* note : a loadPrefix becomes an internal CDict */ 0 /* byRef */, ZSTD_dm_auto, /* note : a loadPrefix becomes an internal CDict */
requestedParams.cParams, zcs->cMem); requestedParams.cParams, zcs->cMem);
zcs->cdict = zcs->cdictLocal; zcs->cdict = zcs->cdictLocal;
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);

3
lib/zstd.h
View File

@ -987,7 +987,7 @@ typedef enum {
/* advanced parameters - may not remain available after API update */ /* advanced parameters - may not remain available after API update */
ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize, ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize,
* even when referencing into Dictionary content (default:0) */ * even when referencing into Dictionary content (default:0) */
} ZSTD_cParameter; } ZSTD_cParameter;
@ -1101,6 +1101,7 @@ size_t ZSTD_compress_generic_simpleArgs (
const void* src, size_t srcSize, size_t* srcPos, const void* src, size_t srcSize, size_t* srcPos,
ZSTD_EndDirective endOp); ZSTD_EndDirective endOp);
/** ZSTD_CCtx_params : /** ZSTD_CCtx_params :
* *
* - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure

1
programs/fileio.c
View File

@ -214,6 +214,7 @@ void FIO_setOverlapLog(unsigned overlapLog){
g_overlapLog = overlapLog; g_overlapLog = overlapLog;
} }
/*-************************************* /*-*************************************
* Functions * Functions
***************************************/ ***************************************/

293
tests/zstreamtest.c
View File

@ -1204,298 +1204,6 @@ _output_error:
goto _cleanup; goto _cleanup;
} }
/* Tests for ZSTD_compress_generic() API */
#if 0
static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests)
{
U32 const maxSrcLog = bigTests ? 24 : 22;
static const U32 maxSampleLog = 19;
size_t const srcBufferSize = (size_t)1<<maxSrcLog;
BYTE* cNoiseBuffer[5];
size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog);
BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize);
size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
BYTE* const cBuffer = (BYTE*)malloc (cBufferSize);
size_t const dstBufferSize = srcBufferSize;
BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize);
U32 result = 0;
U32 testNb = 0;
U32 coreSeed = seed;
ZSTD_CCtx* zc = ZSTD_createCCtx(); /* will be reset sometimes */
ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */
ZSTD_DStream* const zd_noise = ZSTD_createDStream();
clock_t const startClock = clock();
const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */
size_t dictSize = 0;
U32 oldTestLog = 0;
U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests;
U32 const nbThreadsMax = bigTests ? 5 : 1;
/* allocations */
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
!copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise ,
"Not enough memory, fuzzer tests cancelled");
/* Create initial samples */
RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) ); /* ensure at least one init */
/* catch up testNb */
for (testNb=1; testNb < startTest; testNb++)
FUZ_rand(&coreSeed);
/* test loop */
for ( ; (testNb <= nbTests) || (FUZ_GetClockSpan(startClock) < g_clockTime) ; testNb++ ) {
U32 lseed;
const BYTE* srcBuffer;
size_t totalTestSize, totalGenSize, cSize;
XXH64_state_t xxhState;
U64 crcOrig;
U32 resetAllowed = 1;
size_t maxTestSize;
/* init */
if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
FUZ_rand(&coreSeed);
lseed = coreSeed ^ prime32;
/* states full reset (deliberately not synchronized) */
/* some issues can only happen when reusing states */
if ((FUZ_rand(&lseed) & 0xFF) == 131) {
DISPLAYLEVEL(5, "Creating new context \n");
ZSTD_freeCCtx(zc);
zc = ZSTD_createCCtx();
CHECK(zc==NULL, "ZSTD_createCCtx allocation error");
resetAllowed=0;
}
if ((FUZ_rand(&lseed) & 0xFF) == 132) {
ZSTD_freeDStream(zd);
zd = ZSTD_createDStream();
CHECK(zd==NULL, "ZSTD_createDStream allocation error");
ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */
}
/* srcBuffer selection [0-4] */
{ U32 buffNb = FUZ_rand(&lseed) & 0x7F;
if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
else {
buffNb >>= 3;
if (buffNb & 7) {
const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
buffNb = tnb[buffNb >> 3];
} else {
const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
buffNb = tnb[buffNb >> 3];
} }
srcBuffer = cNoiseBuffer[buffNb];
}
/* compression init */
CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) ); /* cancel previous dict /*/
if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
&& oldTestLog /* at least one test happened */ && resetAllowed) {
maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1;
{ int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1;
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_compressionLevel, compressionLevel) );
}
} else {
U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const cLevelCandidate = (FUZ_rand(&lseed) %
(ZSTD_maxCLevel() -
(MAX(testLog, dictLog) / 3))) +
1;
U32 const cLevel = MIN(cLevelCandidate, cLevelMax);
maxTestSize = FUZ_rLogLength(&lseed, testLog);
oldTestLog = testLog;
/* random dictionary selection */
dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
dict = srcBuffer + dictStart;
if (!dictSize) dict=NULL;
}
{ U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize);
/* mess with compression parameters */
cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1;
cParams.searchLength += (FUZ_rand(&lseed) & 3) - 1;
cParams.targetLength = (U32)(cParams.targetLength * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128)));
cParams = ZSTD_adjustCParams(cParams, 0, 0);
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_windowLog, cParams.windowLog) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_hashLog, cParams.hashLog) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_chainLog, cParams.chainLog) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_searchLog, cParams.searchLog) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_minMatch, cParams.searchLength) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_targetLength, cParams.targetLength) );
/* unconditionally set, to be sync with decoder */
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_refDictContent, FUZ_rand(&lseed) & 1) );
if (FUZ_rand(&lseed) & 1) {
CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
if (dict && dictSize) {
/* test that compression parameters are rejected (correctly) after loading a non-NULL dictionary */
size_t const setError = ZSTD_CCtx_setParameter(zc, ZSTD_p_windowLog, cParams.windowLog-1) ;
CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_setParameter should have failed");
} } else {
CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) );
}
/* mess with frame parameters */
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_checksumFlag, FUZ_rand(&lseed) & 1) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_contentSizeFlag, FUZ_rand(&lseed) & 1) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
DISPLAYLEVEL(5, "pledgedSrcSize : %u \n", (U32)pledgedSrcSize);
/* multi-threading parameters */
{ U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
U32 const nbThreads = MIN(nbThreadsCandidate, nbThreadsMax);
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_nbThreads, nbThreads) );
if (nbThreads > 1) {
U32 const jobLog = FUZ_rand(&lseed) % (testLog+1);
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_overlapSizeLog, FUZ_rand(&lseed) % 10) );
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog)) );
} } } }
/* multi-segments compression test */
XXH64_reset(&xxhState, 0);
{ ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
for (cSize=0, totalTestSize=0 ; (totalTestSize < maxTestSize) ; ) {
/* compress random chunks into randomly sized dst buffers */
size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
ZSTD_EndDirective const flush = (FUZ_rand(&lseed) & 15) ? ZSTD_e_continue : ZSTD_e_flush;
ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
outBuff.size = outBuff.pos + dstBuffSize;
CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, flush) );
DISPLAYLEVEL(5, "compress consumed %u bytes (total : %u) \n",
(U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos));
XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
totalTestSize += inBuff.pos;
}
/* final frame epilogue */
{ size_t remainingToFlush = (size_t)(-1);
while (remainingToFlush) {
ZSTD_inBuffer inBuff = { NULL, 0, 0 };
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
outBuff.size = outBuff.pos + adjustedDstSize;
DISPLAYLEVEL(5, "End-flush into dst buffer of size %u \n", (U32)adjustedDstSize);
remainingToFlush = ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end);
CHECK(ZSTD_isError(remainingToFlush),
"ZSTD_compress_generic w/ ZSTD_e_end error : %s",
ZSTD_getErrorName(remainingToFlush) );
} }
crcOrig = XXH64_digest(&xxhState);
cSize = outBuff.pos;
DISPLAYLEVEL(5, "Frame completed : %u bytes \n", (U32)cSize);
}
/* multi - fragments decompression test */
if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
DISPLAYLEVEL(5, "resetting DCtx (dict:%08X) \n", (U32)(size_t)dict);
CHECK_Z( ZSTD_resetDStream(zd) );
} else {
DISPLAYLEVEL(5, "using dict of size %u \n", (U32)dictSize);
CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
}
{ size_t decompressionResult = 1;
ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
for (totalGenSize = 0 ; decompressionResult ; ) {
size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
inBuff.size = inBuff.pos + readCSrcSize;
outBuff.size = inBuff.pos + dstBuffSize;
DISPLAYLEVEL(5, "ZSTD_decompressStream input %u bytes (pos:%u/%u)\n",
(U32)readCSrcSize, (U32)inBuff.pos, (U32)cSize);
decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
DISPLAYLEVEL(5, "inBuff.pos = %u \n", (U32)readCSrcSize);
}
CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (U32)outBuff.pos, (U32)totalTestSize);
CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (U32)inBuff.pos, (U32)cSize);
{ U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
CHECK (crcDest!=crcOrig, "decompressed data corrupted");
} }
/*===== noisy/erroneous src decompression test =====*/
/* add some noise */
{ U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
} }
/* try decompression on noisy data */
CHECK_Z( ZSTD_initDStream(zd_noise) ); /* note : no dictionary */
{ ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
while (outBuff.pos < dstBufferSize) {
size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize);
outBuff.size = outBuff.pos + adjustedDstSize;
inBuff.size = inBuff.pos + adjustedCSrcSize;
{ size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (ZSTD_isError(decompressError)) break; /* error correctly detected */
/* Good so far, but no more progress possible */
if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
} } } }
DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
_cleanup:
ZSTD_freeCCtx(zc);
ZSTD_freeDStream(zd);
ZSTD_freeDStream(zd_noise);
free(cNoiseBuffer[0]);
free(cNoiseBuffer[1]);
free(cNoiseBuffer[2]);
free(cNoiseBuffer[3]);
free(cNoiseBuffer[4]);
free(copyBuffer);
free(cBuffer);
free(dstBuffer);
return result;
_output_error:
result = 1;
goto _cleanup;
}
#endif
/** If useOpaqueAPI, sets param in cctxParams. /** If useOpaqueAPI, sets param in cctxParams.
* Otherwise, sets the param in zc. */ * Otherwise, sets the param in zc. */
static size_t setCCtxParameter(ZSTD_CCtx* zc, ZSTD_CCtx_params* cctxParams, static size_t setCCtxParameter(ZSTD_CCtx* zc, ZSTD_CCtx_params* cctxParams,
@ -1509,6 +1217,7 @@ static size_t setCCtxParameter(ZSTD_CCtx* zc, ZSTD_CCtx_params* cctxParams,
} }
} }
/* Tests for ZSTD_compress_generic() API */
static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests, U32 const useOpaqueAPI) static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests, U32 const useOpaqueAPI)
{ {
U32 const maxSrcLog = bigTests ? 24 : 22; U32 const maxSrcLog = bigTests ? 24 : 22;