facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[dictBuilder] Fix displayLevel for corpus warning 

Pass the displaylevel into the corpus warning, because it is used in
fast cover and cover, so it needs to respect the local level.
dev
Nick Terrell 2019-04-08 20:00:18 -07:00
parent bfcd5b81d7
commit e649fad7aa
3 changed files with 14 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
lib/dictBuilder/cover.c
View File

@ -627,19 +627,20 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
return 1; return 1;
} }
void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers) void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers, int displayLevel)
{ {
const double ratio = (double)nbDmers / maxDictSize; const double ratio = (double)nbDmers / maxDictSize;
if (ratio >= 10) { if (ratio >= 10) {
return; return;
} }
DISPLAYLEVEL(1, "WARNING: The maximum dictionary size %u is too large " LOCALDISPLAYLEVEL(displayLevel, 1,
"compared to the source size %u! " "WARNING: The maximum dictionary size %u is too large "
"size(source)/size(dictionary) = %f, but it should be >= " "compared to the source size %u! "
"10! This may lead to a subpar dictionary! We recommend " "size(source)/size(dictionary) = %f, but it should be >= "
"training on sources at least 10x, and up to 100x the " "10! This may lead to a subpar dictionary! We recommend "
"size of the dictionary!\n", (U32)maxDictSize, "training on sources at least 10x, and up to 100x the "
(U32)nbDmers, ratio); "size of the dictionary!\n", (U32)maxDictSize,
(U32)nbDmers, ratio);
} }
COVER_epoch_info_t COVER_computeEpochs(U32 maxDictSize, COVER_epoch_info_t COVER_computeEpochs(U32 maxDictSize,
@ -744,7 +745,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
parameters.d, parameters.splitPoint)) { parameters.d, parameters.splitPoint)) {
return ERROR(GENERIC); return ERROR(GENERIC);
} }
COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.suffixSize); COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.suffixSize, g_displayLevel);
if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) { if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n"); DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
COVER_ctx_destroy(&ctx); COVER_ctx_destroy(&ctx);
@ -1060,7 +1061,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
return ERROR(GENERIC); return ERROR(GENERIC);
} }
if (!warned) { if (!warned) {
COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.suffixSize); COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.suffixSize, displayLevel);
warned = 1; warned = 1;
} }
/* Loop through k reusing the same context */ /* Loop through k reusing the same context */

2
lib/dictBuilder/cover.h
View File

@ -65,7 +65,7 @@ COVER_epoch_info_t COVER_computeEpochs(U32 maxDictSize, U32 nbDmers,
/** /**
* Warns the user when their corpus is too small. * Warns the user when their corpus is too small.
*/ */
void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers); void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers, int displayLevel);
/** /**
* Checks total compressed size of a dictionary * Checks total compressed size of a dictionary

4
lib/dictBuilder/fastcover.c
View File

@ -570,7 +570,7 @@ ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity,
DISPLAYLEVEL(1, "Failed to initialize context\n"); DISPLAYLEVEL(1, "Failed to initialize context\n");
return ERROR(GENERIC); return ERROR(GENERIC);
} }
COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers); COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers, g_displayLevel);
/* Build the dictionary */ /* Build the dictionary */
DISPLAYLEVEL(2, "Building dictionary\n"); DISPLAYLEVEL(2, "Building dictionary\n");
{ {
@ -673,7 +673,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
return ERROR(GENERIC); return ERROR(GENERIC);
} }
if (!warned) { if (!warned) {
COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers); COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers, displayLevel);
warned = 1; warned = 1;
} }
/* Loop through k reusing the same context */ /* Loop through k reusing the same context */