diff --git a/tests/fuzzer.c b/tests/fuzzer.c index c9461ee9..6586fc12 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -51,14 +51,14 @@ static const U32 nbTestsDefault = 30000; /*-************************************ * Display Macros **************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAY(...) fprintf(stdout, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } static U32 g_displayLevel = 2; #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ if ((FUZ_clockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stderr); } } + if (g_displayLevel>=4) fflush(stdout); } } static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6; static clock_t g_displayClock = 0; @@ -97,7 +97,161 @@ static unsigned FUZ_highbit32(U32 v32) /*============================================= -* Basic Unit tests +* Memory Tests +=============================================*/ +#if defined(__APPLE__) && defined(__MACH__) + +#include /* malloc_size */ + +typedef struct { + unsigned long long totalMalloc; + size_t peakMalloc; + unsigned nbMalloc; + unsigned nbFree; +} mallocCounter_t; + +static const mallocCounter_t INIT_MALLOC_COUNTER = { 0, 0, 0, 0 }; + +static void* FUZ_mallocDebug(void* counter, size_t size) +{ + mallocCounter_t* const mcPtr = (mallocCounter_t*)counter; + void* const ptr = malloc(size); + if (ptr==NULL) return NULL; + mcPtr->totalMalloc += size; + mcPtr->peakMalloc += size; + mcPtr->nbMalloc += 1; + return ptr; +} + +static void FUZ_freeDebug(void* counter, void* address) +{ + mallocCounter_t* const mcPtr = (mallocCounter_t*)counter; + free(address); + mcPtr->nbFree += 1; + mcPtr->peakMalloc -= malloc_size(address); /* OS-X specific */ +} + +static void FUZ_displayMallocStats(mallocCounter_t count) +{ + DISPLAYLEVEL(3, "peak:%u KB, nbMallocs:%u, total:%u KB \n", + (U32)(count.peakMalloc >> 10), + count.nbMalloc, + (U32)(count.totalMalloc >> 10)); +} + +static int FUZ_mallocTests(unsigned seed, double compressibility) +{ + size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */ + size_t const outSize = ZSTD_compressBound(inSize); + void* const inBuffer = malloc(inSize); + void* const outBuffer = malloc(outSize); + + /* test only played in verbose mode, as they are long */ + if (g_displayLevel<3) return 0; + + /* Create compressible noise */ + if (!inBuffer || !outBuffer) { + DISPLAY("Not enough memory, aborting\n"); + exit(1); + } + RDG_genBuffer(inBuffer, inSize, compressibility, 0. /*auto*/, seed); + + /* simple compression tests */ + { int compressionLevel; + mallocCounter_t malcount = INIT_MALLOC_COUNTER; + ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; + for (compressionLevel=1; compressionLevel<=5; compressionLevel++) { + ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); + ZSTD_compressCCtx(cctx, outBuffer, outSize, inBuffer, inSize, compressionLevel); + ZSTD_freeCCtx(cctx); + DISPLAYLEVEL(3, "compressCCtx level %i : ", compressionLevel); + FUZ_displayMallocStats(malcount); + malcount = INIT_MALLOC_COUNTER; + } } + + /* streaming compression tests */ + { int compressionLevel; + mallocCounter_t malcount = INIT_MALLOC_COUNTER; + ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; + for (compressionLevel=1; compressionLevel<=5; compressionLevel++) { + ZSTD_CCtx* const cstream = ZSTD_createCStream_advanced(cMem); + ZSTD_outBuffer out = { outBuffer, outSize, 0 }; + ZSTD_inBuffer in = { inBuffer, inSize, 0 }; + ZSTD_initCStream(cstream, compressionLevel); + ZSTD_compressStream(cstream, &out, &in); + ZSTD_endStream(cstream, &out); + ZSTD_freeCStream(cstream); + DISPLAYLEVEL(3, "compressStream level %i : ", compressionLevel); + FUZ_displayMallocStats(malcount); + malcount = INIT_MALLOC_COUNTER; + } } + + /* advanced API test */ + { int compressionLevel; + mallocCounter_t malcount = INIT_MALLOC_COUNTER; + ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; + for (compressionLevel=1; compressionLevel<=5; compressionLevel++) { + ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); + ZSTD_outBuffer out = { outBuffer, outSize, 0 }; + ZSTD_inBuffer in = { inBuffer, inSize, 0 }; + ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (U32)compressionLevel); + ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end); + ZSTD_freeCCtx(cctx); + DISPLAYLEVEL(3, "compress_generic,end level %i : ", compressionLevel); + FUZ_displayMallocStats(malcount); + malcount = INIT_MALLOC_COUNTER; + } } + + /* advanced MT API test */ + { int compressionLevel; + mallocCounter_t malcount = INIT_MALLOC_COUNTER; + ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; + for (compressionLevel=1; compressionLevel<=5; compressionLevel++) { + ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); + ZSTD_outBuffer out = { outBuffer, outSize, 0 }; + ZSTD_inBuffer in = { inBuffer, inSize, 0 }; + ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (U32)compressionLevel); + ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbThreads, 2); + ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end); + ZSTD_freeCCtx(cctx); + DISPLAYLEVEL(3, "compress_generic,-T2,end level %i : ", compressionLevel); + FUZ_displayMallocStats(malcount); + malcount = INIT_MALLOC_COUNTER; + } } + + /* advanced MT streaming API test */ + { int compressionLevel; + mallocCounter_t malcount = INIT_MALLOC_COUNTER; + ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; + for (compressionLevel=1; compressionLevel<=5; compressionLevel++) { + ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); + ZSTD_outBuffer out = { outBuffer, outSize, 0 }; + ZSTD_inBuffer in = { inBuffer, inSize, 0 }; + ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, (U32)compressionLevel); + ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbThreads, 2); + ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue); + ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end); + ZSTD_freeCCtx(cctx); + DISPLAYLEVEL(3, "compress_generic,-T2,continue level %i : ", compressionLevel); + FUZ_displayMallocStats(malcount); + malcount = INIT_MALLOC_COUNTER; + } } + + return 0; +} + +#else + +static int FUZ_mallocTests(unsigned seed, double compressibility) +{ + (void)seed; (void)compressibility; + return 0; +} + +#endif + +/*============================================= +* Unit tests =============================================*/ #define CHECK_V(var, fn) size_t const var = fn; if (ZSTD_isError(var)) goto _output_error @@ -108,7 +262,8 @@ static int basicUnitTests(U32 seed, double compressibility) { size_t const CNBuffSize = 5 MB; void* const CNBuffer = malloc(CNBuffSize); - void* const compressedBuffer = malloc(ZSTD_compressBound(CNBuffSize)); + size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize); + void* const compressedBuffer = malloc(compressedBufferSize); void* const decodedBuffer = malloc(CNBuffSize); ZSTD_DCtx* dctx = ZSTD_createDCtx(); int testResult = 0; @@ -123,6 +278,9 @@ static int basicUnitTests(U32 seed, double compressibility) } RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed); + /* memory tests */ + FUZ_mallocTests(seed, compressibility); + /* Basic tests */ DISPLAYLEVEL(4, "test%3i : ZSTD_getErrorName : ", testNb++); { const char* errorString = ZSTD_getErrorName(0); @@ -139,7 +297,7 @@ static int basicUnitTests(U32 seed, double compressibility) { ZSTD_CCtx* cctx = ZSTD_createCCtx(); if (cctx==NULL) goto _output_error; CHECKPLUS(r, ZSTD_compressCCtx(cctx, - compressedBuffer, ZSTD_compressBound(CNBuffSize), + compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, 1), cSize=r ); DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); @@ -226,7 +384,7 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "test%3i : simple compression test with static CCtx : ", testNb++); CHECKPLUS(r, ZSTD_compressCCtx(staticCCtx, - compressedBuffer, ZSTD_compressBound(CNBuffSize), + compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL), cSize=r ); DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", @@ -295,7 +453,7 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "test%3i : compress %u bytes with 2 threads : ", testNb++, (U32)CNBuffSize); CHECKPLUS(r, ZSTDMT_compressCCtx(mtctx, - compressedBuffer, ZSTD_compressBound(CNBuffSize), + compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, 1), cSize=r ); @@ -382,7 +540,7 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "test%3i : compress with flat dictionary : ", testNb++); cSize = 0; - CHECKPLUS(r, ZSTD_compressEnd(ctxOrig, compressedBuffer, ZSTD_compressBound(CNBuffSize), + CHECKPLUS(r, ZSTD_compressEnd(ctxOrig, compressedBuffer, compressedBufferSize, (const char*)CNBuffer + dictSize, CNBuffSize - dictSize), cSize += r); DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); @@ -398,7 +556,7 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "test%3i : compress with duplicated context : ", testNb++); { size_t const cSizeOrig = cSize; cSize = 0; - CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(CNBuffSize), + CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, compressedBufferSize, (const char*)CNBuffer + dictSize, CNBuffSize - dictSize), cSize += r); if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> same size */ @@ -483,7 +641,7 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "OK : %u \n", dictID); DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++); - cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize), + cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, dictBuffer, dictSize, 4); if (ZSTD_isError(cSize)) goto _output_error; @@ -521,7 +679,7 @@ static int basicUnitTests(U32 seed, double compressibility) 1 /* byReference */, ZSTD_dm_auto, cParams, ZSTD_defaultCMem); DISPLAYLEVEL(4, "(size : %u) : ", (U32)ZSTD_sizeof_CDict(cdict)); - cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize), + cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, cdict); ZSTD_freeCDict(cdict); if (ZSTD_isError(cSize)) goto _output_error; @@ -556,7 +714,7 @@ static int basicUnitTests(U32 seed, double compressibility) goto _output_error; } cSize = ZSTD_compress_usingCDict(cctx, - compressedBuffer, ZSTD_compressBound(CNBuffSize), + compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, cdict); if (ZSTD_isError(cSize)) { DISPLAY("ZSTD_compress_usingCDict failed "); @@ -570,7 +728,7 @@ static int basicUnitTests(U32 seed, double compressibility) { ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ }; ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1 /*byRef*/, ZSTD_dm_auto, cParams, ZSTD_defaultCMem); - cSize = ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize), + cSize = ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, cdict, fParams); ZSTD_freeCDict(cdict); if (ZSTD_isError(cSize)) goto _output_error; @@ -594,7 +752,7 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++); { ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize); p.fParams.noDictIDFlag = 1; - cSize = ZSTD_compress_advanced(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize), + cSize = ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, dictBuffer, dictSize, p); if (ZSTD_isError(cSize)) goto _output_error; @@ -1245,6 +1403,7 @@ int main(int argc, const char** argv) U32 mainPause = 0; U32 maxDuration = 0; int bigTests = 1; + U32 memTestsOnly = 0; const char* const programName = argv[0]; /* Check command line */ @@ -1255,6 +1414,7 @@ int main(int argc, const char** argv) /* Handle commands. Aggregated commands are allowed */ if (argument[0]=='-') { + if (!strcmp(argument, "--memtest")) { memTestsOnly=1; continue; } if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; } argument++; @@ -1326,6 +1486,11 @@ int main(int argc, const char** argv) DISPLAY("Seed = %u\n", seed); if (proba!=FUZ_compressibility_default) DISPLAY("Compressibility : %u%%\n", proba); + if (memTestsOnly) { + g_displayLevel=3; + return FUZ_mallocTests(seed, ((double)proba) / 100); + } + if (nbTests < testNb) nbTests = testNb; if (testNb==0)