From d7883a2c0e3b75e9aa5215bc7aa247a97c1cfb5e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 12 Aug 2016 16:48:02 +0200 Subject: [PATCH] added zstreamtest --- programs/.gitignore | 2 + programs/Makefile | 18 +- programs/zstreamtest.c | 658 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 676 insertions(+), 2 deletions(-) create mode 100644 programs/zstreamtest.c diff --git a/programs/.gitignore b/programs/.gitignore index adf78081..369a27cc 100644 --- a/programs/.gitignore +++ b/programs/.gitignore @@ -9,6 +9,8 @@ fuzzer fuzzer32 zbufftest zbufftest32 +zstreamtest +zstreamtest32 datagen paramgrill paramgrill32 diff --git a/programs/Makefile b/programs/Makefile index 9a9fcd34..285d7f9e 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -152,6 +152,13 @@ zbufftest32 : CFLAGS += -m32 zbufftest32 : EXT := 32$(EXT) zbufftest32 : zbufftest +zstreamtest : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zstreamtest.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + +zstreamtest32 : CFLAGS += -m32 +zstreamtest32 : EXT := 32$(EXT) +zstreamtest32 : zstreamtest + paramgrill : $(ZSTD_FILES) datagen.c paramgrill.c $(CC) $(FLAGS) $^ -lm -o $@$(EXT) @@ -229,9 +236,9 @@ ifneq (,$(filter $(HOST_OS),MSYS POSIX)) zstd-playTests: datagen ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST) -test: test-zstd test-fullbench test-fuzzer test-zbuff +test: test-zstd test-fullbench test-fuzzer test-zbuff test-zstream -test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 +test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-zstream32 test-all: test test32 valgrindTest @@ -263,4 +270,11 @@ test-zbuff: zbufftest test-zbuff32: zbufftest32 ./zbufftest32 $(ZBUFFTEST) + +test-zstream: zstreamtest + ./zstreamtest $(ZBUFFTEST) + +test-zstream32: zstreamtest32 + ./zstreamtest32 $(ZBUFFTEST) + endif diff --git a/programs/zstreamtest.c b/programs/zstreamtest.c new file mode 100644 index 00000000..7ef58fe7 --- /dev/null +++ b/programs/zstreamtest.c @@ -0,0 +1,658 @@ +/* + Fuzzer test tool for zstd streaming API + Copyright (C) Yann Collet 2016 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - ZSTD homepage : https://www.zstd.net/ +*/ + +/*-************************************ +* Compiler specific +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define _CRT_SECURE_NO_WARNINGS /* fgets */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ +#endif + + +/*-************************************ +* Includes +**************************************/ +#include /* free */ +#include /* fgets, sscanf */ +#include /* timeb */ +#include /* strcmp */ +#include "mem.h" +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */ +#include "zstd.h" /* ZSTD_compressBound */ +#include "datagen.h" /* RDG_genBuffer */ +#define XXH_STATIC_LINKING_ONLY +#include "xxhash.h" /* XXH64_* */ + + +/*-************************************ +* Constants +**************************************/ +#define KB *(1U<<10) +#define MB *(1U<<20) +#define GB *(1U<<30) + +static const U32 nbTestsDefault = 10000; +#define COMPRESSIBLE_NOISE_LENGTH (10 MB) +#define FUZ_COMPRESSIBILITY_DEFAULT 50 +static const U32 prime1 = 2654435761U; +static const U32 prime2 = 2246822519U; + + + +/*-************************************ +* Display Macros +**************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static U32 g_displayLevel = 2; + +#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ + if ((FUZ_GetMilliSpan(g_displayTime) > g_refreshRate) || (g_displayLevel>=4)) \ + { g_displayTime = FUZ_GetMilliStart(); DISPLAY(__VA_ARGS__); \ + if (g_displayLevel>=4) fflush(stdout); } } +static const U32 g_refreshRate = 150; +static U32 g_displayTime = 0; + +static U32 g_testTime = 0; + + +/*-******************************************************* +* Fuzzer functions +*********************************************************/ +#define MAX(a,b) ((a)>(b)?(a):(b)) + +static U32 FUZ_GetMilliStart(void) +{ + struct timeb tb; + U32 nCount; + ftime( &tb ); + nCount = (U32) (((tb.time & 0xFFFFF) * 1000) + tb.millitm); + return nCount; +} + + +static U32 FUZ_GetMilliSpan(U32 nTimeStart) +{ + U32 const nCurrent = FUZ_GetMilliStart(); + U32 nSpan = nCurrent - nTimeStart; + if (nTimeStart > nCurrent) + nSpan += 0x100000 * 1000; + return nSpan; +} + +/*! FUZ_rand() : + @return : a 27 bits random value, from a 32-bits `seed`. + `seed` is also modified */ +# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) +unsigned int FUZ_rand(unsigned int* seedPtr) +{ + U32 rand32 = *seedPtr; + rand32 *= prime1; + rand32 += prime2; + rand32 = FUZ_rotl32(rand32, 13); + *seedPtr = rand32; + return rand32 >> 5; +} + + +/* +static unsigned FUZ_highbit32(U32 v32) +{ + unsigned nbBits = 0; + if (v32==0) return 0; + for ( ; v32 ; v32>>=1) nbBits++; + return nbBits; +} +*/ + +static void* allocFunction(void* opaque, size_t size) +{ + void* address = malloc(size); + (void)opaque; + return address; +} + +static void freeFunction(void* opaque, void* address) +{ + (void)opaque; + free(address); +} + +static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem) +{ + int testResult = 0; + size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH; + void* CNBuffer = malloc(CNBufferSize); + size_t const skippableFrameSize = 11; + size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH); + void* compressedBuffer = malloc(compressedBufferSize); + size_t const decodedBufferSize = CNBufferSize; + void* decodedBuffer = malloc(decodedBufferSize); + size_t cSize; + U32 testNb=0; + ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem); + ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem); + ZSTD_rCursor rCursor; + ZSTD_wCursor wCursor; + + /* Create compressible test buffer */ + if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) { + DISPLAY("Not enough memory, aborting\n"); + goto _output_error; + } + RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed); + + /* generate skippable frame */ + MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START); + MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize); + cSize = skippableFrameSize + 8; + + /* Basic compression test */ + DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); + ZSTD_initCStream_usingDict(zc, CNBuffer, 128 KB, 1); + wCursor.ptr = (char*)(compressedBuffer)+cSize; + wCursor.size = compressedBufferSize; + wCursor.nbBytesWritten = 0; + rCursor.ptr = CNBuffer; + rCursor.size = CNBufferSize; + { size_t const r = ZSTD_compressStream(zc, &wCursor, &rCursor); + if (ZSTD_isError(r)) goto _output_error; } + if (rCursor.size != 0) goto _output_error; /* entire input should be consumed */ + { size_t const r = ZSTD_endStream(zc, &wCursor); + if (r != 0) goto _output_error; } /*< error, or some data not flushed */ + cSize += wCursor.nbBytesWritten; + DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); + + /* skippable frame test */ + DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++); + ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); + rCursor.ptr = compressedBuffer; + rCursor.size = cSize; + wCursor.ptr = decodedBuffer; + wCursor.size = CNBufferSize; + wCursor.nbBytesWritten = 0; + { size_t const r = ZSTD_decompressStream(zd, &wCursor, &rCursor); + if (r != 0) goto _output_error; } + if (wCursor.nbBytesWritten != 0) goto _output_error; /* skippable frame len is 0 */ + DISPLAYLEVEL(4, "OK \n"); + + /* Basic decompression test */ + DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); + ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); + { size_t const r = ZSTD_decompressStream(zd, &wCursor, &rCursor); + if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ + if (wCursor.nbBytesWritten != CNBufferSize) goto _output_error; /* should regenerate the same amount */ + if (rCursor.size != 0) goto _output_error; /* should have read the entire frame */ + DISPLAYLEVEL(4, "OK \n"); + + /* check regenerated data is byte exact */ + DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); + { size_t i; + for (i=0; i "); DISPLAY(__VA_ARGS__); \ + DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } + +static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility) +{ + static const U32 maxSrcLog = 24; + static const U32 maxSampleLog = 19; + BYTE* cNoiseBuffer[5]; + size_t srcBufferSize = (size_t)1<= testNb) DISPLAYUPDATE(2, "/%6u ", nbTests); + FUZ_rand(&coreSeed); + lseed = coreSeed ^ prime1; + + /* states full reset (unsynchronized) */ + /* some issues only happen when reusing states in a specific sequence of parameters */ + if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZSTD_freeCStream(zc); zc = ZSTD_createCStream(); } + if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZSTD_freeDStream(zd); zd = ZSTD_createDStream(); } + + /* srcBuffer selection [0-4] */ + { U32 buffNb = FUZ_rand(&lseed) & 0x7F; + if (buffNb & 7) buffNb=2; /* most common : compressible (P) */ + else { + buffNb >>= 3; + if (buffNb & 7) { + const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */ + buffNb = tnb[buffNb >> 3]; + } else { + const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */ + buffNb = tnb[buffNb >> 3]; + } } + srcBuffer = cNoiseBuffer[buffNb]; + } + + /* compression init */ + { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; + U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1; + maxTestSize = FUZ_rLogLength(&lseed, testLog); + dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0; + /* random dictionary selection */ + { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); + dict = srcBuffer + dictStart; + } + { ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictSize); + params.fParams.checksumFlag = FUZ_rand(&lseed) & 1; + params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; + { size_t const initError = ZSTD_initCStream_advanced(zc, dict, dictSize, params, 0); + CHECK (ZSTD_isError(initError),"init error : %s", ZSTD_getErrorName(initError)); + } } } + + /* multi-segments compression test */ + XXH64_reset(&xxhState, 0); + nbChunks = (FUZ_rand(&lseed) & 127) + 2; + { ZSTD_wCursor wCursor = { cBuffer, cBufferSize, 0 } ; + for (n=0, cSize=0, totalTestSize=0 ; (n= remainingToFlush); + wCursor.size = adjustedDstSize; + remainingToFlush = ZSTD_endStream(zc, &wCursor); + CHECK (ZSTD_isError(remainingToFlush), "flush error : %s", ZSTD_getErrorName(remainingToFlush)); + CHECK (enoughDstSize && remainingToFlush, "ZSTD_endStream() not fully flushed (%u remaining), but enough space available", (U32)remainingToFlush); + } } + crcOrig = XXH64_digest(&xxhState); + cSize = wCursor.nbBytesWritten; + } + + /* multi - fragments decompression test */ + ZSTD_initDStream_usingDict(zd, dict, dictSize); + { size_t decompressionResult = 1; + ZSTD_rCursor rCursor = { cBuffer, cSize }; + ZSTD_wCursor wCursor = { dstBuffer, dstBufferSize, 0 }; + for (totalCSize = 0, totalGenSize = 0 ; decompressionResult ; ) { + size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize); + rCursor.size = readCSrcSize; + wCursor.size = dstBuffSize; + decompressionResult = ZSTD_decompressStream(zd, &wCursor, &rCursor); + CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult)); + totalCSize += readCSrcSize - rCursor.size; + } + CHECK (decompressionResult != 0, "frame not fully decoded"); + CHECK (wCursor.nbBytesWritten != totalTestSize, "decompressed data : wrong size") + CHECK (totalCSize != cSize, "compressed data should be fully read") + { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); + if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); + CHECK (crcDest!=crcOrig, "decompressed data corrupted"); + } } + + /*===== noisy/erroneous src decompression test =====*/ + + /* add some noise */ + { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2; + U32 nn; for (nn=0; nn='0') && (*argument<='9')) { + nbTests *= 10; + nbTests += *argument - '0'; + argument++; + } + break; + + case 'T': + argument++; + nbTests=0; g_testTime=0; + while ((*argument>='0') && (*argument<='9')) { + g_testTime *= 10; + g_testTime += *argument - '0'; + argument++; + } + if (*argument=='m') g_testTime *=60, argument++; + if (*argument=='n') argument++; + g_testTime *= 1000; + break; + + case 's': + argument++; + seed=0; + seedset=1; + while ((*argument>='0') && (*argument<='9')) { + seed *= 10; + seed += *argument - '0'; + argument++; + } + break; + + case 't': + argument++; + testNb=0; + while ((*argument>='0') && (*argument<='9')) { + testNb *= 10; + testNb += *argument - '0'; + argument++; + } + break; + + case 'P': /* compressibility % */ + argument++; + proba=0; + while ((*argument>='0') && (*argument<='9')) { + proba *= 10; + proba += *argument - '0'; + argument++; + } + if (proba<0) proba=0; + if (proba>100) proba=100; + break; + + default: + return FUZ_usage(programName); + } + } } } /* for(argNb=1; argNb