From 18a68714d0edb736e13f088bdc9ff0760f7e8da4 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 24 Jan 2015 12:08:52 +0100 Subject: [PATCH 1/2] Fixed : issue within fullbench, reported by AJ Baxter --- programs/datagen.c | 34 +++++++++++++++++++++++----------- programs/fullbench.c | 8 ++++---- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/programs/datagen.c b/programs/datagen.c index 4956499d..0f011baa 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -70,6 +70,7 @@ #define CDG_SIZE_DEFAULT (64 KB) #define CDG_SEED_DEFAULT 0 #define CDG_COMPRESSIBILITY_DEFAULT 50 +#define CDG_LITDENSITY_DEFAULT 12 #define PRIME1 2654435761U #define PRIME2 2246822519U @@ -85,7 +86,6 @@ * Local Parameters **************************************/ static unsigned no_prompt = 0; -static char* programName; static unsigned displayLevel = 2; @@ -137,7 +137,7 @@ static char CDG_genChar(U32* seed, const void* ltctx) #define CDG_RAND15BITS ((CDG_rand(seed) >> 3) & 32767) #define CDG_RANDLENGTH ( ((CDG_rand(seed) >> 7) & 7) ? (CDG_rand(seed) & 15) : (CDG_rand(seed) & 511) + 15) #define CDG_DICTSIZE (32 KB) -static void CDG_generate(U64 size, U32* seed, double matchProba) +static void CDG_generate(U64 size, U32* seed, double matchProba, double litProba) { BYTE fullbuff[CDG_DICTSIZE + 128 KB + 1]; BYTE* buff = fullbuff + CDG_DICTSIZE; @@ -145,8 +145,7 @@ static void CDG_generate(U64 size, U32* seed, double matchProba) U32 P32 = (U32)(32768 * matchProba); U32 pos=1; U32 genBlockSize = 128 KB; - double literalDistrib = 0.13; - void* ldctx = CDG_createLiteralDistrib(literalDistrib); + void* ldctx = CDG_createLiteralDistrib(litProba); /* Build initial prefix */ fullbuff[0] = CDG_genChar(seed, ldctx); @@ -221,7 +220,7 @@ static void CDG_generate(U64 size, U32* seed, double matchProba) /********************************************************* * Command line *********************************************************/ -static int CDG_usage(void) +static int CDG_usage(char* programName) { DISPLAY( "Compressible data generator\n"); DISPLAY( "Usage :\n"); @@ -239,9 +238,11 @@ static int CDG_usage(void) int main(int argc, char** argv) { int argNb; - int proba = CDG_COMPRESSIBILITY_DEFAULT; + U32 proba = CDG_COMPRESSIBILITY_DEFAULT; + U32 litProba = CDG_LITDENSITY_DEFAULT; U64 size = CDG_SIZE_DEFAULT; U32 seed = CDG_SEED_DEFAULT; + char* programName; /* Check command line */ programName = argv[0]; @@ -262,7 +263,7 @@ int main(int argc, char** argv) switch(*argument) { case 'h': - return CDG_usage(); + return CDG_usage(programName); case 'g': argument++; size=0; @@ -287,7 +288,7 @@ int main(int argc, char** argv) argument++; } break; - case 'p': + case 'P': argument++; proba=0; while ((*argument>='0') && (*argument<='9')) @@ -296,14 +297,25 @@ int main(int argc, char** argv) proba += *argument - '0'; argument++; } - if (proba<0) proba=0; if (proba>100) proba=100; break; + case 'L': + argument++; + litProba=0; + while ((*argument>='0') && (*argument<='9')) + { + litProba *= 10; + litProba += *argument - '0'; + argument++; + } + if (litProba>100) litProba=100; + break; case 'v': displayLevel = 4; argument++; break; - default: ; + default: + return CDG_usage(programName); } } @@ -314,7 +326,7 @@ int main(int argc, char** argv) DISPLAYLEVEL(3, "Seed = %u \n", seed); if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", proba); - CDG_generate(size, &seed, ((double)proba) / 100); + CDG_generate(size, &seed, ((double)proba) / 100, ((double)litProba) / 100); return 0; } diff --git a/programs/fullbench.c b/programs/fullbench.c index b13ea60a..3a9c4993 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -383,7 +383,7 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb) } /* Allocation */ - dstBuffSize = srcSize + 512; + dstBuffSize = ZSTD_compressBound(srcSize); dstBuff = malloc(dstBuffSize); buff2 = malloc(dstBuffSize); if ((!dstBuff) || (!buff2)) @@ -545,7 +545,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb) DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); } - // Alloc + /* Alloc */ origBuff = (char*) malloc((size_t)benchedSize); if(!origBuff) { @@ -554,7 +554,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb) return 12; } - // Fill input buffer + /* Fill input buffer */ DISPLAY("Loading %s... \r", inFileName); readSize = fread(origBuff, 1, benchedSize, inFile); fclose(inFile); @@ -566,7 +566,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb) return 13; } - // bench + /* bench */ DISPLAY("\r%79s\r", ""); DISPLAY(" %s : \n", inFileName); if (benchNb) From 759433d6797922e576788eb35213ed18cda5af24 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 24 Jan 2015 13:31:55 +0100 Subject: [PATCH 2/2] Updated fullbench tests --- lib/zstd.c | 14 +++++----- programs/Makefile | 10 +++++-- programs/datagen.c | 65 ++++++++++++++++++++++++++++++-------------- programs/fullbench.c | 18 +++++++++++- 4 files changed, 77 insertions(+), 30 deletions(-) diff --git a/lib/zstd.c b/lib/zstd.c index 1fc1dcc3..dcc08bc0 100644 --- a/lib/zstd.c +++ b/lib/zstd.c @@ -1004,9 +1004,9 @@ size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, con //U32 limit = 4 * BLOCKSIZE; //const U32 updateRate = 2 * BLOCKSIZE; - // Init + /* Init */ if (ctx->base==NULL) ctx->base = src, ctx->current=0; - if (src != ctx->base + ctx->current) // not contiguous + if (src != ctx->base + ctx->current) /* not contiguous */ { ZSTD_resetCCtx(ctx); ctx->base = src; @@ -1035,12 +1035,12 @@ size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, con } */ - // compress + /* compress */ if (maxDstSize < ZSTD_blockHeaderSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; cSize = ZSTD_compressBlock(ctx, op+ZSTD_blockHeaderSize, maxDstSize-ZSTD_blockHeaderSize, ip, blockSize); if (cSize == 0) { - cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize); + cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize); /* block is not compressible */ if (ZSTD_isError(cSize)) return cSize; } else @@ -1084,7 +1084,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v BYTE* const ostart = dst; BYTE* op = ostart; - // Header + /* Header */ { size_t headerSize = ZSTD_compressBegin(ctx, dst, maxDstSize); if(ZSTD_isError(headerSize)) return headerSize; @@ -1092,7 +1092,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v maxDstSize -= headerSize; } - // Compression + /* Compression */ { size_t cSize = ZSTD_compressContinue(ctx, op, maxDstSize, src, srcSize); if (ZSTD_isError(cSize)) return cSize; @@ -1100,7 +1100,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v maxDstSize -= cSize; } - // Close frame + /* Close frame */ { size_t endSize = ZSTD_compressEnd(ctx, op, maxDstSize); if(ZSTD_isError(endSize)) return endSize; diff --git a/programs/Makefile b/programs/Makefile index 8ab27066..23a482de 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -127,11 +127,17 @@ test-zstd32: zstd32 datagen ./datagen -g256MB | ./zstd32 -v | ./zstd32 -d > $(VOID) ./datagen -g6GB | ./zstd32 -vq | ./zstd32 -d > $(VOID) -test-fullbench: fullbench +test-fullbench: fullbench datagen ./fullbench -i1 + ./datagen -P0 -g516K > tmp + ./fullbench -i1 tmp + @rm tmp -test-fullbench32: fullbench32 +test-fullbench32: fullbench32 datagen ./fullbench32 -i1 + ./datagen -P0 -g516K > tmp + ./fullbench32 -i1 tmp + @rm tmp test-fuzzer: fuzzer ./fuzzer diff --git a/programs/datagen.c b/programs/datagen.c index 0f011baa..09dfe71e 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -56,6 +56,24 @@ #endif +/************************************** +* OS-specific Includes +**************************************/ +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +# include /* _O_BINARY */ +# include /* _setmode, _isatty */ +# ifdef __MINGW32__ + int _fileno(FILE *stream); /* MINGW somehow forgets to include this windows declaration into */ +# endif +# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) +# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#else +# include /* isatty */ +# define SET_BINARY_MODE(file) +# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +#endif + + /************************************** * Constants **************************************/ @@ -70,7 +88,6 @@ #define CDG_SIZE_DEFAULT (64 KB) #define CDG_SEED_DEFAULT 0 #define CDG_COMPRESSIBILITY_DEFAULT 50 -#define CDG_LITDENSITY_DEFAULT 12 #define PRIME1 2654435761U #define PRIME2 2246822519U @@ -97,7 +114,7 @@ static unsigned int CDG_rand(U32* src) { U32 rand32 = *src; rand32 *= PRIME1; - rand32 += PRIME2; + rand32 ^= PRIME2; rand32 = CDG_rotl32(rand32, 13); *src = rand32; return rand32; @@ -106,14 +123,20 @@ static unsigned int CDG_rand(U32* src) #define LTSIZE 8192 #define LTMASK (LTSIZE-1) -static const char firstChar = '('; -static const char lastChar = '}'; static void* CDG_createLiteralDistrib(double ld) { - char* lt = malloc(LTSIZE); + BYTE* lt = malloc(LTSIZE); U32 i = 0; - char character = '0'; + BYTE character = '0'; + BYTE firstChar = '('; + BYTE lastChar = '}'; + if (ld==0.0) + { + character = 0; + firstChar = 0; + lastChar =255; + } while (i='0') && (*argument<='9')) { proba *= 10; proba += *argument - '0'; argument++; } - if (proba>100) proba=100; + if (proba>100.) proba=100.; + proba /= 100.; + litProba = proba / 4.; break; case 'L': argument++; - litProba=0; + litProba=0.; while ((*argument>='0') && (*argument<='9')) { litProba *= 10; litProba += *argument - '0'; argument++; } - if (litProba>100) litProba=100; + if (litProba>100.) litProba=100.; + litProba /= 100.; break; case 'v': displayLevel = 4; @@ -324,9 +349,9 @@ int main(int argc, char** argv) DISPLAYLEVEL(4, "Data Generator %s \n", ZSTD_VERSION); DISPLAYLEVEL(3, "Seed = %u \n", seed); - if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", proba); + if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", (U32)(proba*100)); - CDG_generate(size, &seed, ((double)proba) / 100, ((double)litProba) / 100); + CDG_generate(size, &seed, proba, litProba); return 0; } diff --git a/programs/fullbench.c b/programs/fullbench.c index 3a9c4993..1992df6e 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -403,6 +403,14 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb) { blockProperties_t bp; ZSTD_compress(dstBuff, dstBuffSize, src, srcSize); + ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp); // Get first block compressed size + if (bp.blockType != bt_compressed) + { + DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n"); + free(dstBuff); + free(buff2); + return 0; + } g_cSize = ZSTD_getcBlockSize(dstBuff+7, dstBuffSize, &bp) + 3; memcpy(buff2, dstBuff+7, g_cSize); //srcSize = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize); // real speed @@ -418,6 +426,13 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb) ZSTD_compress(dstBuff, dstBuffSize, src, srcSize); ip += 4; // Jump magic Number blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); // Get first block compressed size + if (bp.blockType != bt_compressed) + { + DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n"); + free(dstBuff); + free(buff2); + return 0; + } iend = ip + 3 + blockSize; // Get end of first block ip += 3; // jump first block header ip += ZSTD_getcBlockSize(ip, iend - ip, &bp) + 3; // jump literal sub block and its header @@ -450,6 +465,8 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb) default : ; } + { size_t i; for (i=0; i