facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #769 from terrelln/real-block-split

Browse Source 
[libzstd] Fix bug in Huffman encoding
This commit is contained in:
Yann Collet 2017-07-18 14:58:26 -07:00 committed by GitHub
commit 2aba13b770
2 changed files with 37 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lib/compress/huf_compress.c
View File

@ -436,7 +436,7 @@ static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt*
size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) #define HUF_FLUSHBITS(s) BIT_flushBits(s)
#define HUF_FLUSHBITS_1(stream) \ #define HUF_FLUSHBITS_1(stream) \
if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*2+7) HUF_FLUSHBITS(stream) if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*2+7) HUF_FLUSHBITS(stream)
@ -451,7 +451,6 @@ size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, si
BYTE* const oend = ostart + dstSize; BYTE* const oend = ostart + dstSize;
BYTE* op = ostart; BYTE* op = ostart;
size_t n; size_t n;
const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize));
BIT_CStream_t bitC; BIT_CStream_t bitC;
/* init */ /* init */

36
tests/fuzzer.c
View File

@ -1003,6 +1003,42 @@ static int basicUnitTests(U32 seed, double compressibility)
if (r != _3BYTESTESTLENGTH) goto _output_error; } if (r != _3BYTESTESTLENGTH) goto _output_error; }
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : incompressible data and ill suited dictionary : ", testNb++);
RDG_genBuffer(CNBuffer, CNBuffSize, 0.0, 0.1, seed);
{ /* Train a dictionary on low characters */
size_t dictSize = 16 KB;
void* const dictBuffer = malloc(dictSize);
size_t const totalSampleSize = 1 MB;
size_t const sampleUnitSize = 8 KB;
U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
if (!dictBuffer || !samplesSizes) goto _output_error;
{ U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize, CNBuffer, samplesSizes, nbSamples);
if (ZDICT_isError(dictSize)) goto _output_error;
/* Reverse the characters to make the dictionary ill suited */
{ U32 u;
for (u = 0; u < CNBuffSize; ++u) {
((BYTE*)CNBuffer)[u] = 255 - ((BYTE*)CNBuffer)[u];
}
}
{ /* Compress the data */
size_t const inputSize = 500;
size_t const outputSize = ZSTD_compressBound(inputSize);
void* const outputBuffer = malloc(outputSize);
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
if (!outputBuffer || !cctx) goto _output_error;
CHECK(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1));
free(outputBuffer);
ZSTD_freeCCtx(cctx);
}
free(dictBuffer);
free(samplesSizes);
}
DISPLAYLEVEL(4, "OK \n");
/* findFrameCompressedSize on skippable frames */ /* findFrameCompressedSize on skippable frames */
DISPLAYLEVEL(4, "test%3i : frame compressed size of skippable frame : ", testNb++); DISPLAYLEVEL(4, "test%3i : frame compressed size of skippable frame : ", testNb++);
{ const char* frame = "\x50\x2a\x4d\x18\x05\x0\x0\0abcde"; { const char* frame = "\x50\x2a\x4d\x18\x05\x0\x0\0abcde";