diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 386b051d..2e0edaf9 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2954,6 +2954,26 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, return pos; } +/* ZSTD_writeSkippableFrame_advanced() : + * Writes out a skippable frame with the specified magic number variant (16 are supported), + * from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15, and the desired source data. + * + * Returns the total number of bytes written, or a ZSTD error code. + */ +size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, unsigned magicVariant) { + BYTE* op = (BYTE*)dst; + RETURN_ERROR_IF(dstCapacity < srcSize + ZSTD_SKIPPABLEHEADERSIZE /* Skippable frame overhead */, + dstSize_tooSmall, "Not enough room for skippable frame"); + RETURN_ERROR_IF(srcSize > (unsigned)0xFFFFFFFF, srcSize_wrong, "Src size too large for skippable frame"); + RETURN_ERROR_IF(magicVariant > 15, parameter_outOfBound, "Skippable frame magic number variant not supported"); + + MEM_writeLE32(op, (U32)(ZSTD_MAGIC_SKIPPABLE_START + magicVariant)); + MEM_writeLE32(op+4, (U32)srcSize); + ZSTD_memcpy(op+8, src, srcSize); + return srcSize + ZSTD_SKIPPABLEHEADERSIZE; +} + /* ZSTD_writeLastEmptyBlock() : * output an empty Block with end-of-frame mark to complete a frame * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h)) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 79c522ef..baaa5715 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -23,9 +23,13 @@ /* Unix Large Files support (>4GB) */ #define _FILE_OFFSET_BITS 64 #if (defined(__sun__) && (!defined(__LP64__))) /* Sun Solaris 32-bits requires specific definitions */ +# ifndef _LARGEFILE_SOURCE # define _LARGEFILE_SOURCE +# endif #elif ! defined(__LP64__) /* No point defining Large file for 64 bit */ +# ifndef _LARGEFILE64_SOURCE # define _LARGEFILE64_SOURCE +# endif #endif diff --git a/lib/zstd.h b/lib/zstd.h index b0ecdf55..61e11864 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1372,6 +1372,23 @@ ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size const void* src, size_t srcSize); +/*! ZSTD_writeSkippableFrame() : + * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer. + * + * Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number, + * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15. + * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so + * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. + * + * Returns an error if destination buffer is not large enough, if the source size is not representable + * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid). + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, unsigned magicVariant); + + /*************************************** * Memory management ***************************************/ diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 2e5d70ef..d8199300 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1570,6 +1570,11 @@ static int basicUnitTests(U32 const seed, double compressibility) int const segs = 4; /* only use the first half so we don't push against size limit of compressedBuffer */ size_t const segSize = (CNBuffSize / 2) / segs; + + const U32 skipLen = 129 KB; + char* const skipBuff = (char*)malloc(skipLen); + assert(skipBuff != NULL); + memset(skipBuff, 0, skipLen); for (i = 0; i < segs; i++) { CHECK_NEWV(r, ZSTD_compress( (BYTE*)compressedBuffer + off, CNBuffSize - off, @@ -1578,13 +1583,15 @@ static int basicUnitTests(U32 const seed, double compressibility) off += r; if (i == segs/2) { /* insert skippable frame */ - const U32 skipLen = 129 KB; - MEM_writeLE32((BYTE*)compressedBuffer + off, ZSTD_MAGIC_SKIPPABLE_START); - MEM_writeLE32((BYTE*)compressedBuffer + off + 4, skipLen); - off += skipLen + ZSTD_SKIPPABLEHEADERSIZE; + size_t const skippableSize = + ZSTD_writeSkippableFrame(compressedBuffer + off, compressedBufferSize, + skipBuff, skipLen, seed % 15); + CHECK_Z(skippableSize); + off += skippableSize; } } cSize = off; + free(skipBuff); } DISPLAYLEVEL(3, "OK \n"); @@ -2739,7 +2746,7 @@ static int basicUnitTests(U32 const seed, double compressibility) free(seqs); } DISPLAYLEVEL(3, "OK \n"); - + DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences followed by ZSTD_compressSequences : ", testNb++); { size_t srcSize = 500 KB;