facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added ZSTD_compressBegin_targetSrcSize

Browse Source
This commit is contained in:
inikep 2016-03-23 20:30:26 +01:00
parent 7adceef974
commit eaba91a161
6 changed files with 47 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
lib/zstd_compress.c
View File

@ -97,6 +97,7 @@ struct ZSTD_CCtx_s
U32 nextToUpdate; /* index from which to continue dictionary update */ U32 nextToUpdate; /* index from which to continue dictionary update */
U32 nextToUpdate3; /* index from which to continue dictionary update */ U32 nextToUpdate3; /* index from which to continue dictionary update */
U32 hashLog3; /* dispatch table : larger == faster, more memory */ U32 hashLog3; /* dispatch table : larger == faster, more memory */
U32 targetSrcSize; /* optimize compression for this source size */
U32 loadedDictEnd; U32 loadedDictEnd;
U32 stage; U32 stage;
ZSTD_parameters params; ZSTD_parameters params;
@ -240,6 +241,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
zc->hbSize = 0; zc->hbSize = 0;
zc->stage = 0; zc->stage = 0;
zc->loadedDictEnd = 0; zc->loadedDictEnd = 0;
zc->targetSrcSize = 0;
return 0; return 0;
} }
@ -277,6 +279,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
dstCCtx->dictLimit = srcCCtx->dictLimit; dstCCtx->dictLimit = srcCCtx->dictLimit;
dstCCtx->lowLimit = srcCCtx->lowLimit; dstCCtx->lowLimit = srcCCtx->lowLimit;
dstCCtx->loadedDictEnd= srcCCtx->loadedDictEnd; dstCCtx->loadedDictEnd= srcCCtx->loadedDictEnd;
dstCCtx->targetSrcSize= srcCCtx->targetSrcSize;
/* copy entropy tables */ /* copy entropy tables */
dstCCtx->flagStaticTables = srcCCtx->flagStaticTables; dstCCtx->flagStaticTables = srcCCtx->flagStaticTables;
@ -2180,6 +2183,7 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, si
} }
} }
extern int g_additionalParam;
/*! ZSTD_compressBegin_advanced() : /*! ZSTD_compressBegin_advanced() :
* @return : 0, or an error code */ * @return : 0, or an error code */
@ -2187,10 +2191,10 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_parameters params) ZSTD_parameters params)
{ {
// printf("windowLog=%d hashLog=%d\n", params.windowLog, params.hashLog); // printf("windowLog=%d hashLog=%d targetSrcSize=%d\n", params.windowLog, params.hashLog, zc->targetSrcSize);
ZSTD_validateParams(&params); ZSTD_validateParams(&params);
zc->hashLog3 = (params.searchLength==3) ? ZSTD_HASHLOG3 : 0; U32 hashLog3 = (!zc->targetSrcSize || zc->targetSrcSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((zc->targetSrcSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN);
// if (zc->hashLog3 > params.windowLog) zc->hashLog3 = params.windowLog; zc->hashLog3 = (params.searchLength==3) ? hashLog3 : 0;
// printf("windowLog=%d hashLog=%d hashLog3=%d \n", params.windowLog, params.hashLog, zc->hashLog3); // printf("windowLog=%d hashLog=%d hashLog3=%d \n", params.windowLog, params.hashLog, zc->hashLog3);
{ size_t const errorCode = ZSTD_resetCCtx_advanced(zc, params); { size_t const errorCode = ZSTD_resetCCtx_advanced(zc, params);
@ -2220,6 +2224,15 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc,
} }
size_t ZSTD_compressBegin_targetSrcSize(ZSTD_CCtx* zc, const void* dict, size_t dictSize, size_t targetSrcSize, int compressionLevel)
{
zc->targetSrcSize = dictSize ? dictSize : targetSrcSize;
ZSTD_parameters params = ZSTD_getParams(compressionLevel, zc->targetSrcSize);
params.srcSize = 0;
ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_targetSrcSize compressionLevel=%d\n", zc->base, compressionLevel);
return ZSTD_compressBegin_advanced(zc, dict, dictSize, params);
}
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel) size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel)
{ {
ZSTD_parameters params = ZSTD_getParams(compressionLevel, dictSize); ZSTD_parameters params = ZSTD_getParams(compressionLevel, dictSize);
@ -2309,12 +2322,14 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel) size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel)
{ {
ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel); ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel);
ctx->targetSrcSize = srcSize;
return ZSTD_compress_advanced(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, ZSTD_getParams(compressionLevel, srcSize)); return ZSTD_compress_advanced(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, ZSTD_getParams(compressionLevel, srcSize));
} }
size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel)
{ {
ZSTD_LOG_BLOCK("%p: ZSTD_compressCCtx srcSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, compressionLevel); ZSTD_LOG_BLOCK("%p: ZSTD_compressCCtx srcSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, compressionLevel);
ctx->targetSrcSize = srcSize;
return ZSTD_compress_advanced(ctx, dst, dstCapacity, src, srcSize, NULL, 0, ZSTD_getParams(compressionLevel, srcSize)); return ZSTD_compress_advanced(ctx, dst, dstCapacity, src, srcSize, NULL, 0, ZSTD_getParams(compressionLevel, srcSize));
} }

2
lib/zstd_internal.h
View File

@ -218,6 +218,6 @@ typedef struct {
} seqStore_t; } seqStore_t;
seqStore_t ZSTD_copySeqStore(const ZSTD_CCtx* ctx); seqStore_t ZSTD_copySeqStore(const ZSTD_CCtx* ctx);
size_t ZSTD_compressBegin_targetSrcSize(ZSTD_CCtx* zc, const void* dict, size_t dictSize, size_t targetSrcSize, int compressionLevel);
#endif /* ZSTD_CCOMMON_H_MODULE */ #endif /* ZSTD_CCOMMON_H_MODULE */

3
lib/zstd_static.h
View File

@ -63,7 +63,8 @@ extern "C" {
#define ZSTD_CONTENTLOG_MIN 4 #define ZSTD_CONTENTLOG_MIN 4
#define ZSTD_HASHLOG_MAX 28 #define ZSTD_HASHLOG_MAX 28
#define ZSTD_HASHLOG_MIN 12 #define ZSTD_HASHLOG_MIN 12
#define ZSTD_HASHLOG3 17 #define ZSTD_HASHLOG3_MAX 17
#define ZSTD_HASHLOG3_MIN 15
#define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1) #define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1)
#define ZSTD_SEARCHLOG_MIN 1 #define ZSTD_SEARCHLOG_MIN 1
#define ZSTD_SEARCHLENGTH_MAX 7 #define ZSTD_SEARCHLENGTH_MAX 7

38
programs/bench.c
View File

@ -72,8 +72,9 @@
#include "mem.h" #include "mem.h"
#include "zstd_static.h" #include "zstd_static.h"
#include "zstd_internal.h" /* ZSTD_compressBegin_targetSrcSize */
#include "datagen.h" /* RDG_genBuffer */
#include "xxhash.h" #include "xxhash.h"
#include "datagen.h" /* RDG_genBuffer */
/* ************************************* /* *************************************
@ -139,9 +140,12 @@ static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result
***************************************/ ***************************************/
static U32 g_nbIterations = NBLOOPS; static U32 g_nbIterations = NBLOOPS;
static size_t g_blockSize = 0; static size_t g_blockSize = 0;
int g_additionalParam = 0;
void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; } void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
void BMK_SetNbIterations(unsigned nbLoops) void BMK_SetNbIterations(unsigned nbLoops)
{ {
g_nbIterations = nbLoops; g_nbIterations = nbLoops;
@ -288,15 +292,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
for (nbLoops = 0 ; BMK_clockSpan(clockStart) < clockLoop ; nbLoops++) { for (nbLoops = 0 ; BMK_clockSpan(clockStart) < clockLoop ; nbLoops++) {
U32 blockNb; U32 blockNb;
#if 0 ZSTD_compressBegin_targetSrcSize(refCtx, dictBuffer, dictBufferSize, blockSize, cLevel);
ZSTD_compressBegin_usingDict(refCtx, dictBuffer, dictBufferSize, cLevel);
#else
ZSTD_parameters params = ZSTD_getParams(cLevel, dictBufferSize ? dictBufferSize : blockSize);
// printf("cLevel=%d dictBufferSize=%d srcSize=%d params.srcSize=%d \n", cLevel, (int)dictBufferSize, (int)blockTable[0].srcSize, (int)params.srcSize);
params.srcSize = 0;
ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, params);
#endif
for (blockNb=0; blockNb<nbBlocks; blockNb++) { for (blockNb=0; blockNb<nbBlocks; blockNb++) {
size_t const rSize = ZSTD_compress_usingPreparedCCtx(ctx, refCtx, size_t const rSize = ZSTD_compress_usingPreparedCCtx(ctx, refCtx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom, blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
@ -412,7 +408,7 @@ static size_t BMK_findMaxMem(U64 requiredMem)
} }
static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
const char* displayName, int cLevel, int cLevelLast, int additionalParam, const char* displayName, int cLevel, int cLevelLast,
const size_t* fileSizes, unsigned nbFiles, const size_t* fileSizes, unsigned nbFiles,
const void* dictBuffer, size_t dictBufferSize) const void* dictBuffer, size_t dictBufferSize)
{ {
@ -428,7 +424,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
memset(&result, 0, sizeof(result)); memset(&result, 0, sizeof(result));
memset(&total, 0, sizeof(total)); memset(&total, 0, sizeof(total));
if (g_displayLevel == 1 && !additionalParam) if (g_displayLevel == 1 && !g_additionalParam)
DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10)); DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
if (cLevelLast < cLevel) cLevelLast = cLevel; if (cLevelLast < cLevel) cLevelLast = cLevel;
@ -439,8 +435,8 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
fileSizes, nbFiles, fileSizes, nbFiles,
dictBuffer, dictBufferSize, &result); dictBuffer, dictBufferSize, &result);
if (g_displayLevel == 1) { if (g_displayLevel == 1) {
if (additionalParam) if (g_additionalParam)
DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s (p=%d)\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName, additionalParam); DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s (param=%d)\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName, g_additionalParam);
else else
DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName); DISPLAY("%-3i%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", -l, (int)result.cSize, result.ratio, result.cSpeed, result.dSpeed, displayName);
total.cSize += result.cSize; total.cSize += result.cSize;
@ -491,7 +487,7 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
} }
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName, int cLevel, int cLevelLast, int additionalParam) const char* dictFileName, int cLevel, int cLevelLast)
{ {
void* srcBuffer; void* srcBuffer;
size_t benchedSize; size_t benchedSize;
@ -531,7 +527,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
else displayName = fileNamesTable[0]; else displayName = fileNamesTable[0];
BMK_benchCLevel(srcBuffer, benchedSize, BMK_benchCLevel(srcBuffer, benchedSize,
displayName, cLevel, cLevelLast, additionalParam, displayName, cLevel, cLevelLast,
fileSizes, nbFiles, fileSizes, nbFiles,
dictBuffer, dictBufferSize); dictBuffer, dictBufferSize);
@ -542,7 +538,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
} }
static void BMK_syntheticTest(int cLevel, int cLevelLast, int additionalParam, double compressibility) static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility)
{ {
char name[20] = {0}; char name[20] = {0};
size_t benchedSize = 10000000; size_t benchedSize = 10000000;
@ -556,7 +552,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, int additionalParam, d
/* Bench */ /* Bench */
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, additionalParam, &benchedSize, 1, NULL, 0); BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0);
/* clean up */ /* clean up */
free(srcBuffer); free(srcBuffer);
@ -564,14 +560,14 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, int additionalParam, d
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName, int cLevel, int cLevelLast, int additionalParam) const char* dictFileName, int cLevel, int cLevelLast)
{ {
double const compressibility = (double)g_compressibilityDefault / 100; double const compressibility = (double)g_compressibilityDefault / 100;
if (nbFiles == 0) if (nbFiles == 0)
BMK_syntheticTest(cLevel, cLevelLast, additionalParam, compressibility); BMK_syntheticTest(cLevel, cLevelLast, compressibility);
else else
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, additionalParam); BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast);
return 0; return 0;
} }

3
programs/bench.h
View File

@ -27,10 +27,11 @@
/* Main function */ /* Main function */
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName, int cLevel, int cLevelLast, int additionalParam); const char* dictFileName, int cLevel, int cLevelLast);
/* Set Parameters */ /* Set Parameters */
void BMK_SetNbIterations(unsigned nbLoops); void BMK_SetNbIterations(unsigned nbLoops);
void BMK_SetBlockSize(size_t blockSize); void BMK_SetBlockSize(size_t blockSize);
void BMK_setAdditionalParam(int additionalParam);
void BMK_setNotificationLevel(unsigned level); void BMK_setNotificationLevel(unsigned level);

14
programs/zstdcli.c
View File

@ -183,7 +183,6 @@ int main(int argCount, const char** argv)
nextArgumentIsMaxDict=0; nextArgumentIsMaxDict=0;
unsigned cLevel = 1; unsigned cLevel = 1;
unsigned cLevelLast = 1; unsigned cLevelLast = 1;
int additionalParam = 0;
const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */
unsigned filenameIdx = 0; unsigned filenameIdx = 0;
const char* programName = argv[0]; const char* programName = argv[0];
@ -195,7 +194,7 @@ int main(int argCount, const char** argv)
unsigned dictSelect = g_defaultSelectivityLevel; unsigned dictSelect = g_defaultSelectivityLevel;
/* init */ /* init */
(void)additionalParam; (void)cLevelLast; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */ (void)cLevelLast; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */
if (filenameTable==NULL) { DISPLAY("not enough memory\n"); exit(1); } if (filenameTable==NULL) { DISPLAY("not enough memory\n"); exit(1); }
displayOut = stderr; displayOut = stderr;
/* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */ /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */
@ -321,7 +320,6 @@ int main(int argCount, const char** argv)
cLevelLast = 0; cLevelLast = 0;
while ((*argument >= '0') && (*argument <= '9')) while ((*argument >= '0') && (*argument <= '9'))
cLevelLast *= 10, cLevelLast += *argument++ - '0'; cLevelLast *= 10, cLevelLast += *argument++ - '0';
continue;
} }
break; break;
#endif /* ZSTD_NOBENCH */ #endif /* ZSTD_NOBENCH */
@ -336,12 +334,14 @@ int main(int argCount, const char** argv)
/* Pause at the end (-p) or set an additional param (-p#) (hidden option) */ /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */
case 'p': argument++; case 'p': argument++;
if ((*argument>='0') && (*argument<='9')) { if ((*argument>='0') && (*argument<='9')) {
additionalParam = 0; int additionalParam = 0;
while ((*argument >= '0') && (*argument <= '9')) while ((*argument >= '0') && (*argument <= '9'))
additionalParam *= 10, additionalParam += *argument++ - '0'; additionalParam *= 10, additionalParam += *argument++ - '0';
continue; BMK_setAdditionalParam(additionalParam);
} else {
main_pause=1;
} }
main_pause=1; break; break;
/* unknown command */ /* unknown command */
default : CLEAN_RETURN(badusage(programName)); default : CLEAN_RETURN(badusage(programName));
} }
@ -383,7 +383,7 @@ int main(int argCount, const char** argv)
if (bench) { if (bench) {
#ifndef ZSTD_NOBENCH #ifndef ZSTD_NOBENCH
BMK_setNotificationLevel(displayLevel); BMK_setNotificationLevel(displayLevel);
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, additionalParam); BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast);
#endif #endif
goto _end; goto _end;
} }