diff --git a/lib/zstd.c b/lib/zstd.c index 1fc1dcc3..dcc08bc0 100644 --- a/lib/zstd.c +++ b/lib/zstd.c @@ -1004,9 +1004,9 @@ size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, con //U32 limit = 4 * BLOCKSIZE; //const U32 updateRate = 2 * BLOCKSIZE; - // Init + /* Init */ if (ctx->base==NULL) ctx->base = src, ctx->current=0; - if (src != ctx->base + ctx->current) // not contiguous + if (src != ctx->base + ctx->current) /* not contiguous */ { ZSTD_resetCCtx(ctx); ctx->base = src; @@ -1035,12 +1035,12 @@ size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, con } */ - // compress + /* compress */ if (maxDstSize < ZSTD_blockHeaderSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; cSize = ZSTD_compressBlock(ctx, op+ZSTD_blockHeaderSize, maxDstSize-ZSTD_blockHeaderSize, ip, blockSize); if (cSize == 0) { - cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize); + cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize); /* block is not compressible */ if (ZSTD_isError(cSize)) return cSize; } else @@ -1084,7 +1084,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v BYTE* const ostart = dst; BYTE* op = ostart; - // Header + /* Header */ { size_t headerSize = ZSTD_compressBegin(ctx, dst, maxDstSize); if(ZSTD_isError(headerSize)) return headerSize; @@ -1092,7 +1092,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v maxDstSize -= headerSize; } - // Compression + /* Compression */ { size_t cSize = ZSTD_compressContinue(ctx, op, maxDstSize, src, srcSize); if (ZSTD_isError(cSize)) return cSize; @@ -1100,7 +1100,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v maxDstSize -= cSize; } - // Close frame + /* Close frame */ { size_t endSize = ZSTD_compressEnd(ctx, op, maxDstSize); if(ZSTD_isError(endSize)) return endSize; diff --git a/programs/Makefile b/programs/Makefile index 8ab27066..23a482de 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -127,11 +127,17 @@ test-zstd32: zstd32 datagen ./datagen -g256MB | ./zstd32 -v | ./zstd32 -d > $(VOID) ./datagen -g6GB | ./zstd32 -vq | ./zstd32 -d > $(VOID) -test-fullbench: fullbench +test-fullbench: fullbench datagen ./fullbench -i1 + ./datagen -P0 -g516K > tmp + ./fullbench -i1 tmp + @rm tmp -test-fullbench32: fullbench32 +test-fullbench32: fullbench32 datagen ./fullbench32 -i1 + ./datagen -P0 -g516K > tmp + ./fullbench32 -i1 tmp + @rm tmp test-fuzzer: fuzzer ./fuzzer diff --git a/programs/datagen.c b/programs/datagen.c index 4956499d..09dfe71e 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -56,6 +56,24 @@ #endif +/************************************** +* OS-specific Includes +**************************************/ +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +# include /* _O_BINARY */ +# include /* _setmode, _isatty */ +# ifdef __MINGW32__ + int _fileno(FILE *stream); /* MINGW somehow forgets to include this windows declaration into */ +# endif +# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) +# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#else +# include /* isatty */ +# define SET_BINARY_MODE(file) +# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +#endif + + /************************************** * Constants **************************************/ @@ -85,7 +103,6 @@ * Local Parameters **************************************/ static unsigned no_prompt = 0; -static char* programName; static unsigned displayLevel = 2; @@ -97,7 +114,7 @@ static unsigned int CDG_rand(U32* src) { U32 rand32 = *src; rand32 *= PRIME1; - rand32 += PRIME2; + rand32 ^= PRIME2; rand32 = CDG_rotl32(rand32, 13); *src = rand32; return rand32; @@ -106,14 +123,20 @@ static unsigned int CDG_rand(U32* src) #define LTSIZE 8192 #define LTMASK (LTSIZE-1) -static const char firstChar = '('; -static const char lastChar = '}'; static void* CDG_createLiteralDistrib(double ld) { - char* lt = malloc(LTSIZE); + BYTE* lt = malloc(LTSIZE); U32 i = 0; - char character = '0'; + BYTE character = '0'; + BYTE firstChar = '('; + BYTE lastChar = '}'; + if (ld==0.0) + { + character = 0; + firstChar = 0; + lastChar =255; + } while (i> 3) & 32767) #define CDG_RANDLENGTH ( ((CDG_rand(seed) >> 7) & 7) ? (CDG_rand(seed) & 15) : (CDG_rand(seed) & 511) + 15) #define CDG_DICTSIZE (32 KB) -static void CDG_generate(U64 size, U32* seed, double matchProba) +static void CDG_generate(U64 size, U32* seed, double matchProba, double litProba) { BYTE fullbuff[CDG_DICTSIZE + 128 KB + 1]; BYTE* buff = fullbuff + CDG_DICTSIZE; @@ -145,10 +168,11 @@ static void CDG_generate(U64 size, U32* seed, double matchProba) U32 P32 = (U32)(32768 * matchProba); U32 pos=1; U32 genBlockSize = 128 KB; - double literalDistrib = 0.13; - void* ldctx = CDG_createLiteralDistrib(literalDistrib); + void* ldctx = CDG_createLiteralDistrib(litProba); + FILE* fout = stdout; - /* Build initial prefix */ + /* init */ + SET_BINARY_MODE(stdout); fullbuff[0] = CDG_genChar(seed, ldctx); while (pos<32 KB) { @@ -207,11 +231,8 @@ static void CDG_generate(U64 size, U32* seed, double matchProba) } } - /* output datagen */ - pos=0; - for (;pos+512<=genBlockSize;pos+=512) - printf("%512.512s", buff+pos); - for (;pos='0') && (*argument<='9')) { proba *= 10; proba += *argument - '0'; argument++; } - if (proba<0) proba=0; - if (proba>100) proba=100; + if (proba>100.) proba=100.; + proba /= 100.; + litProba = proba / 4.; + break; + case 'L': + argument++; + litProba=0.; + while ((*argument>='0') && (*argument<='9')) + { + litProba *= 10; + litProba += *argument - '0'; + argument++; + } + if (litProba>100.) litProba=100.; + litProba /= 100.; break; case 'v': displayLevel = 4; argument++; break; - default: ; + default: + return CDG_usage(programName); } } @@ -312,9 +349,9 @@ int main(int argc, char** argv) DISPLAYLEVEL(4, "Data Generator %s \n", ZSTD_VERSION); DISPLAYLEVEL(3, "Seed = %u \n", seed); - if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", proba); + if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", (U32)(proba*100)); - CDG_generate(size, &seed, ((double)proba) / 100); + CDG_generate(size, &seed, proba, litProba); return 0; } diff --git a/programs/fullbench.c b/programs/fullbench.c index b13ea60a..1992df6e 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -383,7 +383,7 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb) } /* Allocation */ - dstBuffSize = srcSize + 512; + dstBuffSize = ZSTD_compressBound(srcSize); dstBuff = malloc(dstBuffSize); buff2 = malloc(dstBuffSize); if ((!dstBuff) || (!buff2)) @@ -403,6 +403,14 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb) { blockProperties_t bp; ZSTD_compress(dstBuff, dstBuffSize, src, srcSize); + ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp); // Get first block compressed size + if (bp.blockType != bt_compressed) + { + DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n"); + free(dstBuff); + free(buff2); + return 0; + } g_cSize = ZSTD_getcBlockSize(dstBuff+7, dstBuffSize, &bp) + 3; memcpy(buff2, dstBuff+7, g_cSize); //srcSize = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize); // real speed @@ -418,6 +426,13 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb) ZSTD_compress(dstBuff, dstBuffSize, src, srcSize); ip += 4; // Jump magic Number blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); // Get first block compressed size + if (bp.blockType != bt_compressed) + { + DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n"); + free(dstBuff); + free(buff2); + return 0; + } iend = ip + 3 + blockSize; // Get end of first block ip += 3; // jump first block header ip += ZSTD_getcBlockSize(ip, iend - ip, &bp) + 3; // jump literal sub block and its header @@ -450,6 +465,8 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb) default : ; } + { size_t i; for (i=0; i>20)); } - // Alloc + /* Alloc */ origBuff = (char*) malloc((size_t)benchedSize); if(!origBuff) { @@ -554,7 +570,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb) return 12; } - // Fill input buffer + /* Fill input buffer */ DISPLAY("Loading %s... \r", inFileName); readSize = fread(origBuff, 1, benchedSize, inFile); fclose(inFile); @@ -566,7 +582,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb) return 13; } - // bench + /* bench */ DISPLAY("\r%79s\r", ""); DISPLAY(" %s : \n", inFileName); if (benchNb)