diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 2e62ec2c..43e6e663 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -97,6 +97,7 @@ struct ZSTD_CCtx_s U32 nextToUpdate; /* index from which to continue dictionary update */ U32 nextToUpdate3; /* index from which to continue dictionary update */ U32 hashLog3; /* dispatch table : larger == faster, more memory */ + U32 targetSrcSize; /* optimize compression for this source size */ U32 loadedDictEnd; U32 stage; ZSTD_parameters params; @@ -240,6 +241,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, zc->hbSize = 0; zc->stage = 0; zc->loadedDictEnd = 0; + zc->targetSrcSize = 0; return 0; } @@ -277,6 +279,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) dstCCtx->dictLimit = srcCCtx->dictLimit; dstCCtx->lowLimit = srcCCtx->lowLimit; dstCCtx->loadedDictEnd= srcCCtx->loadedDictEnd; + dstCCtx->targetSrcSize= srcCCtx->targetSrcSize; /* copy entropy tables */ dstCCtx->flagStaticTables = srcCCtx->flagStaticTables; @@ -2180,6 +2183,7 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, si } } +extern int g_additionalParam; /*! ZSTD_compressBegin_advanced() : * @return : 0, or an error code */ @@ -2187,10 +2191,10 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc, const void* dict, size_t dictSize, ZSTD_parameters params) { -// printf("windowLog=%d hashLog=%d\n", params.windowLog, params.hashLog); +// printf("windowLog=%d hashLog=%d targetSrcSize=%d\n", params.windowLog, params.hashLog, zc->targetSrcSize); ZSTD_validateParams(¶ms); - zc->hashLog3 = (params.searchLength==3) ? ZSTD_HASHLOG3 : 0; -// if (zc->hashLog3 > params.windowLog) zc->hashLog3 = params.windowLog; + U32 hashLog3 = (!zc->targetSrcSize || zc->targetSrcSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((zc->targetSrcSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN); + zc->hashLog3 = (params.searchLength==3) ? hashLog3 : 0; // printf("windowLog=%d hashLog=%d hashLog3=%d \n", params.windowLog, params.hashLog, zc->hashLog3); { size_t const errorCode = ZSTD_resetCCtx_advanced(zc, params); @@ -2220,6 +2224,15 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc, } +size_t ZSTD_compressBegin_targetSrcSize(ZSTD_CCtx* zc, const void* dict, size_t dictSize, size_t targetSrcSize, int compressionLevel) +{ + zc->targetSrcSize = dictSize ? dictSize : targetSrcSize; + ZSTD_parameters params = ZSTD_getParams(compressionLevel, zc->targetSrcSize); + params.srcSize = 0; + ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_targetSrcSize compressionLevel=%d\n", zc->base, compressionLevel); + return ZSTD_compressBegin_advanced(zc, dict, dictSize, params); +} + size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_parameters params = ZSTD_getParams(compressionLevel, dictSize); @@ -2309,12 +2322,14 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel); + ctx->targetSrcSize = srcSize; return ZSTD_compress_advanced(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, ZSTD_getParams(compressionLevel, srcSize)); } size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) { ZSTD_LOG_BLOCK("%p: ZSTD_compressCCtx srcSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, compressionLevel); + ctx->targetSrcSize = srcSize; return ZSTD_compress_advanced(ctx, dst, dstCapacity, src, srcSize, NULL, 0, ZSTD_getParams(compressionLevel, srcSize)); } diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index 3b68654e..1358eb8a 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -218,6 +218,6 @@ typedef struct { } seqStore_t; seqStore_t ZSTD_copySeqStore(const ZSTD_CCtx* ctx); - +size_t ZSTD_compressBegin_targetSrcSize(ZSTD_CCtx* zc, const void* dict, size_t dictSize, size_t targetSrcSize, int compressionLevel); #endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/lib/zstd_static.h b/lib/zstd_static.h index d41bc841..d88fbce8 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -63,7 +63,8 @@ extern "C" { #define ZSTD_CONTENTLOG_MIN 4 #define ZSTD_HASHLOG_MAX 28 #define ZSTD_HASHLOG_MIN 12 -#define ZSTD_HASHLOG3 17 +#define ZSTD_HASHLOG3_MAX 17 +#define ZSTD_HASHLOG3_MIN 15 #define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1) #define ZSTD_SEARCHLOG_MIN 1 #define ZSTD_SEARCHLENGTH_MAX 7 diff --git a/programs/bench.c b/programs/bench.c index d3e9c1f1..1468bcb2 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -72,8 +72,9 @@ #include "mem.h" #include "zstd_static.h" +#include "zstd_internal.h" /* ZSTD_compressBegin_targetSrcSize */ +#include "datagen.h" /* RDG_genBuffer */ #include "xxhash.h" -#include "datagen.h" /* RDG_genBuffer */ /* ************************************* @@ -139,9 +140,12 @@ static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result ***************************************/ static U32 g_nbIterations = NBLOOPS; static size_t g_blockSize = 0; +int g_additionalParam = 0; void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; } +void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; } + void BMK_SetNbIterations(unsigned nbLoops) { g_nbIterations = nbLoops; @@ -288,15 +292,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, for (nbLoops = 0 ; BMK_clockSpan(clockStart) < clockLoop ; nbLoops++) { U32 blockNb; -#if 0 - ZSTD_compressBegin_usingDict(refCtx, dictBuffer, dictBufferSize, cLevel); -#else - ZSTD_parameters params = ZSTD_getParams(cLevel, dictBufferSize ? dictBufferSize : blockSize); - // printf("cLevel=%d dictBufferSize=%d srcSize=%d params.srcSize=%d \n", cLevel, (int)dictBufferSize, (int)blockTable[0].srcSize, (int)params.srcSize); - params.srcSize = 0; - ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, params); -#endif - + ZSTD_compressBegin_targetSrcSize(refCtx, dictBuffer, dictBufferSize, blockSize, cLevel); for (blockNb=0; blockNb>10)); if (cLevelLast < cLevel) cLevelLast = cLevel; @@ -439,8 +435,8 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, fileSizes, nbFiles, dictBuffer, dictBufferSize, &result); if (g_displayLevel == 1) { - if (additionalParam) - DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s (p=%d)\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName, additionalParam); + if (g_additionalParam) + DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s (param=%d)\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName, g_additionalParam); else DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName); total.cSize += result.cSize; @@ -491,7 +487,7 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, } static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel, int cLevelLast, int additionalParam) + const char* dictFileName, int cLevel, int cLevelLast) { void* srcBuffer; size_t benchedSize; @@ -531,7 +527,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, else displayName = fileNamesTable[0]; BMK_benchCLevel(srcBuffer, benchedSize, - displayName, cLevel, cLevelLast, additionalParam, + displayName, cLevel, cLevelLast, fileSizes, nbFiles, dictBuffer, dictBufferSize); @@ -542,7 +538,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, } -static void BMK_syntheticTest(int cLevel, int cLevelLast, int additionalParam, double compressibility) +static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility) { char name[20] = {0}; size_t benchedSize = 10000000; @@ -556,7 +552,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, int additionalParam, d /* Bench */ snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); - BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, additionalParam, &benchedSize, 1, NULL, 0); + BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0); /* clean up */ free(srcBuffer); @@ -564,14 +560,14 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, int additionalParam, d int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel, int cLevelLast, int additionalParam) + const char* dictFileName, int cLevel, int cLevelLast) { double const compressibility = (double)g_compressibilityDefault / 100; if (nbFiles == 0) - BMK_syntheticTest(cLevel, cLevelLast, additionalParam, compressibility); + BMK_syntheticTest(cLevel, cLevelLast, compressibility); else - BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, additionalParam); + BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast); return 0; } diff --git a/programs/bench.h b/programs/bench.h index bc5ffa42..3a1ca3a2 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -27,10 +27,11 @@ /* Main function */ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel, int cLevelLast, int additionalParam); + const char* dictFileName, int cLevel, int cLevelLast); /* Set Parameters */ void BMK_SetNbIterations(unsigned nbLoops); void BMK_SetBlockSize(size_t blockSize); +void BMK_setAdditionalParam(int additionalParam); void BMK_setNotificationLevel(unsigned level); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index a14fe26d..0c060bdb 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -183,7 +183,6 @@ int main(int argCount, const char** argv) nextArgumentIsMaxDict=0; unsigned cLevel = 1; unsigned cLevelLast = 1; - int additionalParam = 0; const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ unsigned filenameIdx = 0; const char* programName = argv[0]; @@ -195,7 +194,7 @@ int main(int argCount, const char** argv) unsigned dictSelect = g_defaultSelectivityLevel; /* init */ - (void)additionalParam; (void)cLevelLast; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */ + (void)cLevelLast; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */ if (filenameTable==NULL) { DISPLAY("not enough memory\n"); exit(1); } displayOut = stderr; /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */ @@ -321,7 +320,6 @@ int main(int argCount, const char** argv) cLevelLast = 0; while ((*argument >= '0') && (*argument <= '9')) cLevelLast *= 10, cLevelLast += *argument++ - '0'; - continue; } break; #endif /* ZSTD_NOBENCH */ @@ -336,12 +334,14 @@ int main(int argCount, const char** argv) /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */ case 'p': argument++; if ((*argument>='0') && (*argument<='9')) { - additionalParam = 0; + int additionalParam = 0; while ((*argument >= '0') && (*argument <= '9')) additionalParam *= 10, additionalParam += *argument++ - '0'; - continue; + BMK_setAdditionalParam(additionalParam); + } else { + main_pause=1; } - main_pause=1; break; + break; /* unknown command */ default : CLEAN_RETURN(badusage(programName)); } @@ -383,7 +383,7 @@ int main(int argCount, const char** argv) if (bench) { #ifndef ZSTD_NOBENCH BMK_setNotificationLevel(displayLevel); - BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, additionalParam); + BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast); #endif goto _end; }