facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

removed g_displayLevel

Browse Source
This commit is contained in:
Yann Collet 2016-09-02 17:28:41 -07:00
parent d69798f858
commit d56dbc02d3
1 changed files with 32 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
lib/dictBuilder/zdict.c
View File

@ -71,19 +71,18 @@ static const size_t g_min_fast_dictContent = 192;
* Console display * Console display
***************************************/ ***************************************/
#define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush( stderr ); } #define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush( stderr ); }
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } #define DISPLAYLEVEL(l, ...) if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
static unsigned g_displayLevel = 0; /* 0 : no display; 1: errors; 2: default; 4: full information */
static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; } static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; }
static void ZDICT_printHex(U32 dlevel, const void* ptr, size_t length) static void ZDICT_printHex(const void* ptr, size_t length)
{ {
const BYTE* const b = (const BYTE*)ptr; const BYTE* const b = (const BYTE*)ptr;
size_t u; size_t u;
for (u=0; u<length; u++) { for (u=0; u<length; u++) {
BYTE c = b[u]; BYTE c = b[u];
if (c<32 || c>126) c = '.'; /* non-printable char */ if (c<32 || c>126) c = '.'; /* non-printable char */
DISPLAYLEVEL(dlevel, "%c", c); DISPLAY("%c", c);
} }
} }
@ -204,7 +203,7 @@ static void ZDICT_initDictItem(dictItem* d)
static dictItem ZDICT_analyzePos( static dictItem ZDICT_analyzePos(
BYTE* doneMarks, BYTE* doneMarks,
const int* suffix, U32 start, const int* suffix, U32 start,
const void* buffer, U32 minRatio) const void* buffer, U32 minRatio, U32 notificationLevel)
{ {
U32 lengthList[LLIMIT] = {0}; U32 lengthList[LLIMIT] = {0};
U32 cumulLength[LLIMIT] = {0}; U32 cumulLength[LLIMIT] = {0};
@ -466,7 +465,7 @@ static U32 ZDICT_dictSize(const dictItem* dictList)
static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize, static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize,
const void* const buffer, size_t bufferSize, /* buffer must end with noisy guard band */ const void* const buffer, size_t bufferSize, /* buffer must end with noisy guard band */
const size_t* fileSizes, unsigned nbFiles, const size_t* fileSizes, unsigned nbFiles,
U32 minRatio) U32 minRatio, U32 notificationLevel)
{ {
int* const suffix0 = (int*)malloc((bufferSize+2)*sizeof(*suffix0)); int* const suffix0 = (int*)malloc((bufferSize+2)*sizeof(*suffix0));
int* const suffix = suffix0+1; int* const suffix = suffix0+1;
@ -477,10 +476,10 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize,
clock_t displayClock = 0; clock_t displayClock = 0;
clock_t const refreshRate = CLOCKS_PER_SEC * 3 / 10; clock_t const refreshRate = CLOCKS_PER_SEC * 3 / 10;
# define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ # define DISPLAYUPDATE(l, ...) if (notificationLevel>=l) { \
if (ZDICT_clockSpan(displayClock) > refreshRate) \ if (ZDICT_clockSpan(displayClock) > refreshRate) \
{ displayClock = clock(); DISPLAY(__VA_ARGS__); \ { displayClock = clock(); DISPLAY(__VA_ARGS__); \
if (g_displayLevel>=4) fflush(stdout); } } if (notificationLevel>=4) fflush(stdout); } }
/* init */ /* init */
DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */ DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */
@ -518,7 +517,7 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize,
{ U32 cursor; for (cursor=0; cursor < bufferSize; ) { { U32 cursor; for (cursor=0; cursor < bufferSize; ) {
dictItem solution; dictItem solution;
if (doneMarks[cursor]) { cursor++; continue; } if (doneMarks[cursor]) { cursor++; continue; }
solution = ZDICT_analyzePos(doneMarks, suffix, reverseSuffix[cursor], buffer, minRatio); solution = ZDICT_analyzePos(doneMarks, suffix, reverseSuffix[cursor], buffer, minRatio, notificationLevel);
if (solution.length==0) { cursor++; continue; } if (solution.length==0) { cursor++; continue; }
ZDICT_insertDictItem(dictList, dictListSize, solution); ZDICT_insertDictItem(dictList, dictListSize, solution);
cursor += solution.length; cursor += solution.length;
@ -558,15 +557,15 @@ typedef struct
static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params, static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets, U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets,
const void* src, size_t srcSize) const void* src, size_t srcSize, U32 notificationLevel)
{ {
size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << params.cParams.windowLog); size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << params.cParams.windowLog);
size_t cSize; size_t cSize;
if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */ if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */
{ size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref); { size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref);
if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; } if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; }
} }
cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_ABSOLUTEMAX, src, srcSize); cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_ABSOLUTEMAX, src, srcSize);
if (ZSTD_isError(cSize)) { DISPLAYLEVEL(1, "warning : could not compress sample size %u \n", (U32)srcSize); return; } if (ZSTD_isError(cSize)) { DISPLAYLEVEL(1, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
@ -647,9 +646,10 @@ static void ZDICT_insertSortCount(offsetCount_t table[ZSTD_REP_NUM+1], U32 val,
#define OFFCODE_MAX 30 /* only applicable to first block */ #define OFFCODE_MAX 30 /* only applicable to first block */
static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
unsigned compressionLevel, unsigned compressionLevel,
const void* srcBuffer, const size_t* fileSizes, unsigned nbFiles, const void* srcBuffer, const size_t* fileSizes, unsigned nbFiles,
const void* dictBuffer, size_t dictBufferSize) const void* dictBuffer, size_t dictBufferSize,
unsigned notificationLevel)
{ {
U32 countLit[256]; U32 countLit[256];
HUF_CREATE_STATIC_CTABLE(hufTable, 255); HUF_CREATE_STATIC_CTABLE(hufTable, 255);
@ -690,18 +690,19 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
memset(bestRepOffset, 0, sizeof(bestRepOffset)); memset(bestRepOffset, 0, sizeof(bestRepOffset));
if (compressionLevel==0) compressionLevel=g_compressionLevel_default; if (compressionLevel==0) compressionLevel=g_compressionLevel_default;
params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize); params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize);
{ size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0); { size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0);
if (ZSTD_isError(beginResult)) { if (ZSTD_isError(beginResult)) {
eSize = ERROR(GENERIC); eSize = ERROR(GENERIC);
DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced failed \n"); DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced failed \n");
goto _cleanup; goto _cleanup;
} } } }
/* collect stats on all files */ /* collect stats on all files */
for (u=0; u<nbFiles; u++) { for (u=0; u<nbFiles; u++) {
ZDICT_countEStats(esr, params, ZDICT_countEStats(esr, params,
countLit, offcodeCount, matchLengthCount, litLengthCount, repOffset, countLit, offcodeCount, matchLengthCount, litLengthCount, repOffset,
(const char*)srcBuffer + pos, fileSizes[u]); (const char*)srcBuffer + pos, fileSizes[u],
notificationLevel);
pos += fileSizes[u]; pos += fileSizes[u];
} }
@ -748,7 +749,6 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
} }
llLog = (U32)errorCode; llLog = (U32)errorCode;
/* write result to buffer */ /* write result to buffer */
{ size_t const hhSize = HUF_writeCTable(dstPtr, maxDstSize, hufTable, 255, huffLog); { size_t const hhSize = HUF_writeCTable(dstPtr, maxDstSize, hufTable, 255, huffLog);
if (HUF_isError(hhSize)) { if (HUF_isError(hhSize)) {
@ -828,6 +828,7 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo
{ {
size_t hSize; size_t hSize;
int const compressionLevel = (params.compressionLevel <= 0) ? g_compressionLevel_default : params.compressionLevel; int const compressionLevel = (params.compressionLevel <= 0) ? g_compressionLevel_default : params.compressionLevel;
U32 const notificationLevel = params.notificationLevel;
/* dictionary header */ /* dictionary header */
MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC); MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC);
@ -844,7 +845,8 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo
{ size_t const eSize = ZDICT_analyzeEntropy((char*)dictBuffer+hSize, dictBufferCapacity-hSize, { size_t const eSize = ZDICT_analyzeEntropy((char*)dictBuffer+hSize, dictBufferCapacity-hSize,
compressionLevel, compressionLevel,
samplesBuffer, samplesSizes, nbSamples, samplesBuffer, samplesSizes, nbSamples,
(char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize); (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize,
notificationLevel);
if (ZDICT_isError(eSize)) return eSize; if (ZDICT_isError(eSize)) return eSize;
hSize += eSize; hSize += eSize;
} }
@ -872,6 +874,7 @@ size_t ZDICT_trainFromBuffer_unsafe(
size_t const targetDictSize = maxDictSize; size_t const targetDictSize = maxDictSize;
size_t const samplesBuffSize = ZDICT_totalSampleSize(samplesSizes, nbSamples); size_t const samplesBuffSize = ZDICT_totalSampleSize(samplesSizes, nbSamples);
size_t dictSize = 0; size_t dictSize = 0;
U32 const notificationLevel = params.notificationLevel;
/* checks */ /* checks */
if (!dictList) return ERROR(memory_allocation); if (!dictList) return ERROR(memory_allocation);
@ -880,16 +883,15 @@ size_t ZDICT_trainFromBuffer_unsafe(
/* init */ /* init */
ZDICT_initDictItem(dictList); ZDICT_initDictItem(dictList);
g_displayLevel = params.notificationLevel;
/* build dictionary */ /* build dictionary */
ZDICT_trainBuffer(dictList, dictListSize, ZDICT_trainBuffer(dictList, dictListSize,
samplesBuffer, samplesBuffSize, samplesBuffer, samplesBuffSize,
samplesSizes, nbSamples, samplesSizes, nbSamples,
minRep); minRep, notificationLevel);
/* display best matches */ /* display best matches */
if (g_displayLevel>= 3) { if (params.notificationLevel>= 3) {
U32 const nb = MIN(25, dictList[0].pos); U32 const nb = MIN(25, dictList[0].pos);
U32 const dictContentSize = ZDICT_dictSize(dictList); U32 const dictContentSize = ZDICT_dictSize(dictList);
U32 u; U32 u;
@ -901,7 +903,7 @@ size_t ZDICT_trainFromBuffer_unsafe(
U32 printedLength = MIN(40, length); U32 printedLength = MIN(40, length);
DISPLAYLEVEL(3, "%3u:%3u bytes at pos %8u, savings %7u bytes |", DISPLAYLEVEL(3, "%3u:%3u bytes at pos %8u, savings %7u bytes |",
u, length, pos, dictList[u].savings); u, length, pos, dictList[u].savings);
ZDICT_printHex(3, (const char*)samplesBuffer+pos, printedLength); ZDICT_printHex((const char*)samplesBuffer+pos, printedLength);
DISPLAYLEVEL(3, "| \n"); DISPLAYLEVEL(3, "| \n");
} } } }