facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2397 from dscheg/fixtrainfrombuffer

Browse Source 
Pass dictBufferCapacity to COVER_selectDict()
This commit is contained in:
Nick Terrell 2020-11-30 10:38:45 -08:00 committed by GitHub
commit bcd17db2fb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
lib/dictBuilder/cover.c
View File

@ -955,7 +955,7 @@ void COVER_dictSelectionFree(COVER_dictSelection_t selection){
free(selection.dictContent); free(selection.dictContent);
} }
COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity,
size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples, size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples,
size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize) { size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize) {
@ -963,8 +963,8 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
size_t largestCompressed = 0; size_t largestCompressed = 0;
BYTE* customDictContentEnd = customDictContent + dictContentSize; BYTE* customDictContentEnd = customDictContent + dictContentSize;
BYTE * largestDictbuffer = (BYTE *)malloc(dictContentSize); BYTE * largestDictbuffer = (BYTE *)malloc(dictBufferCapacity);
BYTE * candidateDictBuffer = (BYTE *)malloc(dictContentSize); BYTE * candidateDictBuffer = (BYTE *)malloc(dictBufferCapacity);
double regressionTolerance = ((double)params.shrinkDictMaxRegression / 100.0) + 1.00; double regressionTolerance = ((double)params.shrinkDictMaxRegression / 100.0) + 1.00;
if (!largestDictbuffer || !candidateDictBuffer) { if (!largestDictbuffer || !candidateDictBuffer) {
@ -976,7 +976,7 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
/* Initial dictionary size and compressed size */ /* Initial dictionary size and compressed size */
memcpy(largestDictbuffer, customDictContent, dictContentSize); memcpy(largestDictbuffer, customDictContent, dictContentSize);
dictContentSize = ZDICT_finalizeDictionary( dictContentSize = ZDICT_finalizeDictionary(
largestDictbuffer, dictContentSize, customDictContent, dictContentSize, largestDictbuffer, dictBufferCapacity, customDictContent, dictContentSize,
samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams); samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams);
if (ZDICT_isError(dictContentSize)) { if (ZDICT_isError(dictContentSize)) {
@ -1010,7 +1010,7 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
while (dictContentSize < largestDict) { while (dictContentSize < largestDict) {
memcpy(candidateDictBuffer, largestDictbuffer, largestDict); memcpy(candidateDictBuffer, largestDictbuffer, largestDict);
dictContentSize = ZDICT_finalizeDictionary( dictContentSize = ZDICT_finalizeDictionary(
candidateDictBuffer, dictContentSize, customDictContentEnd - dictContentSize, dictContentSize, candidateDictBuffer, dictBufferCapacity, customDictContentEnd - dictContentSize, dictContentSize,
samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams); samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams);
if (ZDICT_isError(dictContentSize)) { if (ZDICT_isError(dictContentSize)) {
@ -1088,7 +1088,7 @@ static void COVER_tryParameters(void *opaque) {
{ {
const size_t tail = COVER_buildDictionary(ctx, freqs, &activeDmers, dict, const size_t tail = COVER_buildDictionary(ctx, freqs, &activeDmers, dict,
dictBufferCapacity, parameters); dictBufferCapacity, parameters);
selection = COVER_selectDict(dict + tail, dictBufferCapacity - tail, selection = COVER_selectDict(dict + tail, dictBufferCapacity, dictBufferCapacity - tail,
ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbTrainSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets, ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbTrainSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets,
totalCompressedSize); totalCompressedSize);

2
lib/dictBuilder/cover.h
View File

@ -152,6 +152,6 @@ void COVER_dictSelectionFree(COVER_dictSelection_t selection);
* smallest dictionary within a specified regression of the compressed size * smallest dictionary within a specified regression of the compressed size
* from the largest dictionary. * from the largest dictionary.
*/ */
COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity,
size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples, size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples,
size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize); size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize);

2
lib/dictBuilder/fastcover.c
View File

@ -487,7 +487,7 @@ static void FASTCOVER_tryParameters(void *opaque)
parameters, segmentFreqs); parameters, segmentFreqs);
const unsigned nbFinalizeSamples = (unsigned)(ctx->nbTrainSamples * ctx->accelParams.finalize / 100); const unsigned nbFinalizeSamples = (unsigned)(ctx->nbTrainSamples * ctx->accelParams.finalize / 100);
selection = COVER_selectDict(dict + tail, dictBufferCapacity - tail, selection = COVER_selectDict(dict + tail, dictBufferCapacity, dictBufferCapacity - tail,
ctx->samples, ctx->samplesSizes, nbFinalizeSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets, ctx->samples, ctx->samplesSizes, nbFinalizeSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets,
totalCompressedSize); totalCompressedSize);