From 7e11bd012b4e25e1b55b43e97bbef16d2c886e14 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Mon, 21 Dec 2020 11:13:22 -0500 Subject: [PATCH 1/5] Implement skippable frame function --- lib/compress/zstd_compress.c | 20 ++++++++++++++++++++ lib/zstd.h | 16 ++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 386b051d..3e6bf746 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2954,6 +2954,26 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, return pos; } +/* ZSTD_generateSkippableFrame_advanced() : + * Writes out a skippable frame with the specified magic number variant (16 are supported), + * from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15, and the desired source data. + * + * Returns the total number of bytes written, or a ZSTD error code. + */ +size_t ZSTD_generateSkippableFrame_advanced(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, U32 magicVariant) { + BYTE* op = dst; + RETURN_ERROR_IF(dstCapacity < srcSize + 8 /* Skippable frame overhead */, + dstSize_tooSmall, "Not enough room for skippable frame"); + RETURN_ERROR_IF(srcSize > (unsigned)0xFFFFFFFF, srcSize_wrong, "Src size too large for skippable frame"); + RETURN_ERROR_IF(magicVariant > 15, parameter_outOfBound, "Skippable frame magic number variant not supported"); + + MEM_writeLE32(op, (U32)(ZSTD_MAGIC_SKIPPABLE_START + magicVariant)); + MEM_writeLE32(op+4, (U32)srcSize); + ZSTD_memcpy(op+8, src, srcSize); + return srcSize + 8; +} + /* ZSTD_writeLastEmptyBlock() : * output an empty Block with end-of-frame mark to complete a frame * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h)) diff --git a/lib/zstd.h b/lib/zstd.h index b0ecdf55..c34b4397 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1372,6 +1372,22 @@ ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size const void* src, size_t srcSize); +/*! ZSTD_generateSkippableFrame() : + * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer. + * + * Skippable frames have a 4-byte magic number, which can range from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15. + * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so + * the final magic number will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. + * + * Returns an error if destination buffer is not large enough, if the source size is not representable + * with a 4-byte unsigned int, or if the magicVariant is greater than 15. + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_API size_t ZSTD_generateSkippableFrame(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, unsigned magicVariant); + + /*************************************** * Memory management ***************************************/ From 339d8ba103b3dda92801bd8a8ea511f14a087716 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Mon, 21 Dec 2020 11:33:27 -0500 Subject: [PATCH 2/5] Add unit test --- tests/fuzzer.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 2e5d70ef..eb84dbfa 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1570,6 +1570,11 @@ static int basicUnitTests(U32 const seed, double compressibility) int const segs = 4; /* only use the first half so we don't push against size limit of compressedBuffer */ size_t const segSize = (CNBuffSize / 2) / segs; + + U32 skippableSize; + const U32 skipLen = 129 KB; + char* skipBuff = (char*)malloc(skipLen); + memset(skipBuff, 0, skipLen); for (i = 0; i < segs; i++) { CHECK_NEWV(r, ZSTD_compress( (BYTE*)compressedBuffer + off, CNBuffSize - off, @@ -1578,13 +1583,13 @@ static int basicUnitTests(U32 const seed, double compressibility) off += r; if (i == segs/2) { /* insert skippable frame */ - const U32 skipLen = 129 KB; - MEM_writeLE32((BYTE*)compressedBuffer + off, ZSTD_MAGIC_SKIPPABLE_START); - MEM_writeLE32((BYTE*)compressedBuffer + off + 4, skipLen); - off += skipLen + ZSTD_SKIPPABLEHEADERSIZE; + skippableSize = ZSTD_generateSkippableFrame(compressedBuffer + off, compressedBufferSize, skipBuff, skipLen, seed % 15); + CHECK_Z(skippableSize); + off += skippableSize; } } cSize = off; + free(skipBuff); } DISPLAYLEVEL(3, "OK \n"); From 5c41490bfe8149933213a1245acff586fb65e587 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Mon, 21 Dec 2020 11:33:41 -0500 Subject: [PATCH 3/5] Use pre-defined constants --- lib/compress/zstd_compress.c | 12 ++++++------ lib/zstd.h | 13 +++++++------ tests/fuzzer.c | 2 +- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 3e6bf746..2e0edaf9 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2954,16 +2954,16 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, return pos; } -/* ZSTD_generateSkippableFrame_advanced() : +/* ZSTD_writeSkippableFrame_advanced() : * Writes out a skippable frame with the specified magic number variant (16 are supported), * from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15, and the desired source data. * * Returns the total number of bytes written, or a ZSTD error code. */ -size_t ZSTD_generateSkippableFrame_advanced(void* dst, size_t dstCapacity, - const void* src, size_t srcSize, U32 magicVariant) { - BYTE* op = dst; - RETURN_ERROR_IF(dstCapacity < srcSize + 8 /* Skippable frame overhead */, +size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, unsigned magicVariant) { + BYTE* op = (BYTE*)dst; + RETURN_ERROR_IF(dstCapacity < srcSize + ZSTD_SKIPPABLEHEADERSIZE /* Skippable frame overhead */, dstSize_tooSmall, "Not enough room for skippable frame"); RETURN_ERROR_IF(srcSize > (unsigned)0xFFFFFFFF, srcSize_wrong, "Src size too large for skippable frame"); RETURN_ERROR_IF(magicVariant > 15, parameter_outOfBound, "Skippable frame magic number variant not supported"); @@ -2971,7 +2971,7 @@ size_t ZSTD_generateSkippableFrame_advanced(void* dst, size_t dstCapacity, MEM_writeLE32(op, (U32)(ZSTD_MAGIC_SKIPPABLE_START + magicVariant)); MEM_writeLE32(op+4, (U32)srcSize); ZSTD_memcpy(op+8, src, srcSize); - return srcSize + 8; + return srcSize + ZSTD_SKIPPABLEHEADERSIZE; } /* ZSTD_writeLastEmptyBlock() : diff --git a/lib/zstd.h b/lib/zstd.h index c34b4397..61e11864 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1372,20 +1372,21 @@ ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size const void* src, size_t srcSize); -/*! ZSTD_generateSkippableFrame() : +/*! ZSTD_writeSkippableFrame() : * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer. * - * Skippable frames have a 4-byte magic number, which can range from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15. + * Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number, + * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15. * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so - * the final magic number will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. + * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. * * Returns an error if destination buffer is not large enough, if the source size is not representable - * with a 4-byte unsigned int, or if the magicVariant is greater than 15. + * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid). * * @return : number of bytes written or a ZSTD error. */ -ZSTDLIB_API size_t ZSTD_generateSkippableFrame(void* dst, size_t dstCapacity, - const void* src, size_t srcSize, unsigned magicVariant); +ZSTDLIB_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, unsigned magicVariant); /*************************************** diff --git a/tests/fuzzer.c b/tests/fuzzer.c index eb84dbfa..71ed3e0b 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1583,7 +1583,7 @@ static int basicUnitTests(U32 const seed, double compressibility) off += r; if (i == segs/2) { /* insert skippable frame */ - skippableSize = ZSTD_generateSkippableFrame(compressedBuffer + off, compressedBufferSize, skipBuff, skipLen, seed % 15); + skippableSize = ZSTD_writeSkippableFrame(compressedBuffer + off, compressedBufferSize, skipBuff, skipLen, seed % 15); CHECK_Z(skippableSize); off += skippableSize; } From dd1a7e41ee6619a8d01373259979008db4f13a06 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Sat, 26 Dec 2020 10:00:53 -0700 Subject: [PATCH 4/5] Add ifndef guards for _LARGEFILE_SOURCE and _LARGEFILE64_SOURCE This ensures the symbols aren't redefined, which would result in a compiler error. I was getting redefined symbols for _LARGEFILE64_SOURCE when building for 32-bit x86 Linux on an older CentOS release in a CI environment. With this change, I'm able to compile the single file library in this environment. Closes #2443. --- lib/dictBuilder/zdict.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 79c522ef..baaa5715 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -23,9 +23,13 @@ /* Unix Large Files support (>4GB) */ #define _FILE_OFFSET_BITS 64 #if (defined(__sun__) && (!defined(__LP64__))) /* Sun Solaris 32-bits requires specific definitions */ +# ifndef _LARGEFILE_SOURCE # define _LARGEFILE_SOURCE +# endif #elif ! defined(__LP64__) /* No point defining Large file for 64 bit */ +# ifndef _LARGEFILE64_SOURCE # define _LARGEFILE64_SOURCE +# endif #endif From 7f8be046b95a9aef96db5a6e85696144d94caeee Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 28 Dec 2020 14:07:31 -0800 Subject: [PATCH 5/5] fixed minor warnings introduced in #2439 --- tests/fuzzer.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 71ed3e0b..d8199300 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1571,9 +1571,9 @@ static int basicUnitTests(U32 const seed, double compressibility) /* only use the first half so we don't push against size limit of compressedBuffer */ size_t const segSize = (CNBuffSize / 2) / segs; - U32 skippableSize; const U32 skipLen = 129 KB; - char* skipBuff = (char*)malloc(skipLen); + char* const skipBuff = (char*)malloc(skipLen); + assert(skipBuff != NULL); memset(skipBuff, 0, skipLen); for (i = 0; i < segs; i++) { CHECK_NEWV(r, ZSTD_compress( @@ -1583,7 +1583,9 @@ static int basicUnitTests(U32 const seed, double compressibility) off += r; if (i == segs/2) { /* insert skippable frame */ - skippableSize = ZSTD_writeSkippableFrame(compressedBuffer + off, compressedBufferSize, skipBuff, skipLen, seed % 15); + size_t const skippableSize = + ZSTD_writeSkippableFrame(compressedBuffer + off, compressedBufferSize, + skipBuff, skipLen, seed % 15); CHECK_Z(skippableSize); off += skippableSize; } @@ -2744,7 +2746,7 @@ static int basicUnitTests(U32 const seed, double compressibility) free(seqs); } DISPLAYLEVEL(3, "OK \n"); - + DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences followed by ZSTD_compressSequences : ", testNb++); { size_t srcSize = 500 KB;