From 41e90653fe35242291f248a66f1da70e62965d1d Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Tue, 30 Jul 2019 17:17:07 -0700 Subject: [PATCH 001/144] zstd: Don't use utime on Linux utime is deprecated by POSIX 2008 and optionally not available with uClibc-ng. Got rid of a few useless headers in timefn.h. Signed-off-by: Rosen Penev --- programs/platform.h | 2 +- programs/timefn.h | 6 ------ programs/util.c | 10 ++++++++++ programs/util.h | 7 ++++++- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/programs/platform.h b/programs/platform.h index 38ded872..5934e59c 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -92,7 +92,7 @@ extern "C" { # if defined(__linux__) || defined(__linux) # ifndef _POSIX_C_SOURCE -# define _POSIX_C_SOURCE 200112L /* feature test macro : https://www.gnu.org/software/libc/manual/html_node/Feature-Test-Macros.html */ +# define _POSIX_C_SOURCE 200809L /* feature test macro : https://www.gnu.org/software/libc/manual/html_node/Feature-Test-Macros.html */ # endif # endif # include /* declares _POSIX_VERSION */ diff --git a/programs/timefn.h b/programs/timefn.h index d1ddd31b..2db3765b 100644 --- a/programs/timefn.h +++ b/programs/timefn.h @@ -19,12 +19,6 @@ extern "C" { /*-**************************************** * Dependencies ******************************************/ -#include /* utime */ -#if defined(_MSC_VER) -# include /* utime */ -#else -# include /* utime */ -#endif #include /* clock_t, clock, CLOCKS_PER_SEC */ diff --git a/programs/util.c b/programs/util.c index fb77d178..3a2c0a9d 100644 --- a/programs/util.c +++ b/programs/util.c @@ -54,14 +54,24 @@ int UTIL_getFileStat(const char* infilename, stat_t *statbuf) int UTIL_setFileStat(const char *filename, stat_t *statbuf) { int res = 0; +#if defined(_WIN32) || (PLATFORM_POSIX_VERSION < 200809L) struct utimbuf timebuf; +#else + struct timespec timebuf[2] = {}; +#endif if (!UTIL_isRegularFile(filename)) return -1; +#if defined(_WIN32) || (PLATFORM_POSIX_VERSION < 200809L) timebuf.actime = time(NULL); timebuf.modtime = statbuf->st_mtime; res += utime(filename, &timebuf); /* set access and modification times */ +#else + timebuf[0].tv_nsec = UTIME_NOW; + timebuf[1].tv_sec = statbuf->st_mtime; + res += utimensat(AT_FDCWD, filename, timebuf, 0); /* set access and modification times */ +#endif #if !defined(_WIN32) res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */ diff --git a/programs/util.h b/programs/util.h index d6e5bb55..0080b63c 100644 --- a/programs/util.h +++ b/programs/util.h @@ -25,12 +25,17 @@ extern "C" { #include /* fprintf */ #include /* stat, utime */ #include /* stat, chmod */ -#if defined(_MSC_VER) +#if defined(_WIN32) # include /* utime */ # include /* _chmod */ #else # include /* chown, stat */ +#if PLATFORM_POSIX_VERSION < 200809L # include /* utime */ +#else +# include /* AT_FDCWD */ +# include /* utimensat */ +#endif #endif #include /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */ #include "mem.h" /* U32, U64 */ From b83059958246dfcb5b91af9c187fad8c706869a0 Mon Sep 17 00:00:00 2001 From: mgrice Date: Tue, 27 Aug 2019 14:49:23 -0700 Subject: [PATCH 002/144] Improvements in zstd decode performance Summary: The idea behind wildcopy is that it can be cheaper to copy more bytes (say 8) than it is to copy less (say, 3). This change takes that further by exploiting some properties: 1. it's almost always OK to copy 16 bytes instead of 8, which means fewer copy instructions, and fewer branches 2. A 16 byte chunk size means that ~90% of wildcopy invocations will have a trip count of 1, so branch prediction will be improved. Speedup on Xeon E5-2680v4 is in the range of 3-5%. Measured wildcopy length distributions on silesia.tar: level <=8 <=16 <=24 >24 1 78.05% 11.49% 3.52% 6.94% 3 82.14% 8.99% 2.44% 6.43% 6 85.81% 6.51% 2.92% 4.76% 8 83.02% 7.31% 3.64% 6.03% 10 84.13% 6.67% 3.29% 5.91% 15 77.58% 7.55% 5.21% 9.66% 16 80.07% 7.20% 3.98% 8.75% Test Plan: benchmark silesia, make check --- lib/common/zstd_internal.h | 48 ++++++++++++++++++-------- lib/compress/zstd_compress_internal.h | 2 +- lib/decompress/zstd_decompress_block.c | 12 +++---- 3 files changed, 40 insertions(+), 22 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 585fd6b1..74769d1d 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -191,9 +191,11 @@ static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG; /*-******************************************* * Shared functions to include for inlining *********************************************/ +FORCE_INLINE_ATTR static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; } +FORCE_INLINE_ATTR static void ZSTD_copy16(void* dst, const void* src) { memcpy(dst, src, 16); } #define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; } @@ -209,7 +211,7 @@ typedef enum { /*! ZSTD_wildcopy() : * custom version of memcpy(), can overwrite up to WILDCOPY_OVERLENGTH bytes (if length==0) */ MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE -void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e ovtype) +void ZSTD_wildcopy(void* dst, const void* src, BYTE* oend_g, ptrdiff_t length, ZSTD_overlap_e ovtype) { ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; const BYTE* ip = (const BYTE*)src; @@ -217,25 +219,33 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e BYTE* const oend = op + length; assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8)); + if (length < VECLEN || (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN)) { do COPY8(op, ip) while (op < oend); } else { - if ((length & 8) == 0) - COPY8(op, ip); - do { - COPY16(op, ip); + if (oend < oend_g-16) { + /* common case */ + do { + COPY16(op, ip); + } + while (op < oend); + } + else { + do { + COPY8(op, ip); + } + while (op < oend); } - while (op < oend); } } /*! ZSTD_wildcopy_16min() : - * same semantics as ZSTD_wilcopy() except guaranteed to be able to copy 16 bytes at the start */ + * same semantics as ZSTD_wildcopy() except guaranteed to be able to copy 16 bytes at the start */ MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE -void ZSTD_wildcopy_16min(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e ovtype) +void ZSTD_wildcopy_16min(void* dst, const void* src, BYTE* oend_g, ptrdiff_t length, ZSTD_overlap_e ovtype) { ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; const BYTE* ip = (const BYTE*)src; @@ -246,17 +256,25 @@ void ZSTD_wildcopy_16min(void* dst, const void* src, ptrdiff_t length, ZSTD_over assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8)); if (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN) { - do - COPY8(op, ip) + do { + COPY8(op, ip); + } while (op < oend); } else { - if ((length & 8) == 0) - COPY8(op, ip); - do { - COPY16(op, ip); + if (oend < oend_g-16) { + /* common case */ + do { + COPY16(op, ip); + } + while (op < oend); + } + else { + do { + COPY8(op, ip); + } + while (op < oend); } - while (op < oend); } } diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 3e590ec3..5fbc68e9 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -359,7 +359,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v /* copy Literals */ assert(seqStorePtr->maxNbLit <= 128 KB); assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit); - ZSTD_wildcopy(seqStorePtr->lit, literals, (ptrdiff_t)litLength, ZSTD_no_overlap); + ZSTD_wildcopy(seqStorePtr->lit, literals, seqStorePtr->lit + litLength + 8, (ptrdiff_t)litLength, ZSTD_no_overlap); seqStorePtr->lit += litLength; /* literal Length */ diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c index cbcfc084..bab96fb7 100644 --- a/lib/decompress/zstd_decompress_block.c +++ b/lib/decompress/zstd_decompress_block.c @@ -641,7 +641,7 @@ size_t ZSTD_execSequence(BYTE* op, /* copy Literals */ if (sequence.litLength > 8) - ZSTD_wildcopy_16min(op, (*litPtr), sequence.litLength, ZSTD_no_overlap); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + ZSTD_wildcopy_16min(op, (*litPtr), oend, sequence.litLength, ZSTD_no_overlap); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ else ZSTD_copy8(op, *litPtr); op = oLitEnd; @@ -690,13 +690,13 @@ size_t ZSTD_execSequence(BYTE* op, if (oMatchEnd > oend-(16-MINMATCH)) { if (op < oend_w) { - ZSTD_wildcopy(op, match, oend_w - op, ZSTD_overlap_src_before_dst); + ZSTD_wildcopy(op, match, oend, oend_w - op, ZSTD_overlap_src_before_dst); match += oend_w - op; op = oend_w; } while (op < oMatchEnd) *op++ = *match++; } else { - ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); /* works even if matchLength < 8 */ + ZSTD_wildcopy(op, match, oend, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); /* works even if matchLength < 8 */ } return sequenceLength; } @@ -722,7 +722,7 @@ size_t ZSTD_execSequenceLong(BYTE* op, /* copy Literals */ if (sequence.litLength > 8) - ZSTD_wildcopy_16min(op, *litPtr, sequence.litLength, ZSTD_no_overlap); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + ZSTD_wildcopy_16min(op, *litPtr, oend, sequence.litLength, ZSTD_no_overlap); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ else ZSTD_copy8(op, *litPtr); /* note : op <= oLitEnd <= oend_w == oend - 8 */ @@ -772,13 +772,13 @@ size_t ZSTD_execSequenceLong(BYTE* op, if (oMatchEnd > oend-(16-MINMATCH)) { if (op < oend_w) { - ZSTD_wildcopy(op, match, oend_w - op, ZSTD_overlap_src_before_dst); + ZSTD_wildcopy(op, match, oend, oend_w - op, ZSTD_overlap_src_before_dst); match += oend_w - op; op = oend_w; } while (op < oMatchEnd) *op++ = *match++; } else { - ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); /* works even if matchLength < 8 */ + ZSTD_wildcopy(op, match, oend, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); /* works even if matchLength < 8 */ } return sequenceLength; } From ece465644be7c45449bc4c06170afafa53d37119 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 29 Aug 2019 12:29:39 -0700 Subject: [PATCH 003/144] Adding api for extracting sequences from seqstore --- lib/compress/zstd_compress.c | 71 ++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index cd73db13..73f5cbff 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2190,6 +2190,77 @@ void ZSTD_resetSeqStore(seqStore_t* ssPtr) ssPtr->longLengthID = 0; } +typedef struct { + U32 matchPos; + U32 offset; + U32 litLength; + U32 matchLength; + int rep; +} Sequence; + +static size_t ZSTD_getSequencesForOneBlock(ZSTD_CCtx* zc, ZSTD_CDict* cdict, + void* dst, size_t dstSize, + const void* src, size_t srcSize, + Sequence* outSeqs, size_t outSeqsSize) +{ + const seqStore_t* seqStore; + const seqDef* seqs; + size_t seqsSize; + + int i; int repIdx; U32 position; + + size_t blockSize = ZSTD_getBlockSize(zc); + size_t maxOutput = ZSTD_compressBound(blockSize); + + ASSERT(!ZSTD_isError(ZSTD_compressBegin_usingCDict(zc, cdict))); + ASSERT(dstSize >= maxOutput); dstSize = maxOutput; + ASSERT(srcSize >= blockSize); srcSize = blockSize; + ASSERT(!ZSTD_isError(ZSTD_compressBlock(zc, dst, dstSize, src, srcSize))); + + seqStore = ZSTD_getSeqStore(zc); + seqs = seqStore->sequencesStart; + seqsSize = seqStore->sequences - seqStore->sequencesStart; + + ASSERT(outSeqsSize >= seqsSize); outSeqsSize = seqsSize; + + for (i = 0, position = 0; i < seqsSize; ++i) { + outSeqs[i].offset = seqs[i].offset; + outSeqs[i].litLength = seqs[i].litLength; + outSeqs[i].matchLength = seqs[i].matchLength + 3 /* min match */; + + if (i == seqStore->longLengthPos) { + if (seqStore->longLengthID == 1) { + outSeqs[i].litLength += 0x10000; + } else if (seqStore->longLengthID == 2) { + outSeqs[i].matchLength += 0x10000; + } + } + + if (outSeqs[i].offset <= 3 /* num reps */) { + outSeqs[i].rep = 1; + repIdx = i - outSeqs[i].offset; + + if (repIdx >= 0) { + outSeqs[i].offset = outSeqs[repIdx].offset; + } + + if (repIdx == -1) { + outSeqs[i].offset = 1; + } else if (repIdx == -2) { + outSeqs[i].offset = 4; + } else if (repIdx == -3) { + outSeqs[i].offset = 8; + } + } else { + outSeqs[i].offset -= 3 /* num reps */; + } + + position += outSeqs[i].litLength; + outSeqs[i].matchPos = position; + position += outSeqs[i].matchLength; + } +} + typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e; static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) From 5d897715290883c46aeb1370ba6b54bfd556fa90 Mon Sep 17 00:00:00 2001 From: mgrice Date: Thu, 29 Aug 2019 12:32:15 -0700 Subject: [PATCH 004/144] fix warning: always_inline function might not be inlinable --- lib/common/zstd_internal.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 74769d1d..fb6246a1 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -191,11 +191,9 @@ static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG; /*-******************************************* * Shared functions to include for inlining *********************************************/ -FORCE_INLINE_ATTR static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; } -FORCE_INLINE_ATTR static void ZSTD_copy16(void* dst, const void* src) { memcpy(dst, src, 16); } #define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; } From 623b90f85d94f57ebfa0c7ae61e25dd489f27ecb Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 29 Aug 2019 13:09:42 -0700 Subject: [PATCH 005/144] Fixing ci-circle test complaints --- lib/compress/zstd_compress.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 73f5cbff..f0b6136b 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2207,21 +2207,21 @@ static size_t ZSTD_getSequencesForOneBlock(ZSTD_CCtx* zc, ZSTD_CDict* cdict, const seqDef* seqs; size_t seqsSize; - int i; int repIdx; U32 position; + size_t i; int repIdx; size_t position; size_t blockSize = ZSTD_getBlockSize(zc); size_t maxOutput = ZSTD_compressBound(blockSize); - ASSERT(!ZSTD_isError(ZSTD_compressBegin_usingCDict(zc, cdict))); - ASSERT(dstSize >= maxOutput); dstSize = maxOutput; - ASSERT(srcSize >= blockSize); srcSize = blockSize; - ASSERT(!ZSTD_isError(ZSTD_compressBlock(zc, dst, dstSize, src, srcSize))); + assert(!ZSTD_isError(ZSTD_compressBegin_usingCDict(zc, cdict))); + assert(dstSize >= maxOutput); dstSize = maxOutput; + assert(srcSize >= blockSize); srcSize = blockSize; + assert(!ZSTD_isError(ZSTD_compressBlock(zc, dst, dstSize, src, srcSize))); seqStore = ZSTD_getSeqStore(zc); seqs = seqStore->sequencesStart; seqsSize = seqStore->sequences - seqStore->sequencesStart; - ASSERT(outSeqsSize >= seqsSize); outSeqsSize = seqsSize; + assert(outSeqsSize >= seqsSize); outSeqsSize = seqsSize; for (i = 0, position = 0; i < seqsSize; ++i) { outSeqs[i].offset = seqs[i].offset; From 5f8b0f6890e03050a3840ed996f95a5e8263c1ea Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Fri, 30 Aug 2019 09:18:44 -0700 Subject: [PATCH 006/144] Changing api to get sequences across all blocks --- lib/compress/zstd_compress.c | 150 ++++++++++++++------------ lib/compress/zstd_compress_internal.h | 8 ++ lib/zstd.h | 11 ++ tests/fuzzer.c | 5 + 4 files changed, 103 insertions(+), 71 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index f0b6136b..f8588b34 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -13,6 +13,7 @@ ***************************************/ #include /* INT_MAX */ #include /* memset */ +#include #include "cpu.h" #include "mem.h" #include "hist.h" /* HIST_countFast_wksp */ @@ -2190,77 +2191,6 @@ void ZSTD_resetSeqStore(seqStore_t* ssPtr) ssPtr->longLengthID = 0; } -typedef struct { - U32 matchPos; - U32 offset; - U32 litLength; - U32 matchLength; - int rep; -} Sequence; - -static size_t ZSTD_getSequencesForOneBlock(ZSTD_CCtx* zc, ZSTD_CDict* cdict, - void* dst, size_t dstSize, - const void* src, size_t srcSize, - Sequence* outSeqs, size_t outSeqsSize) -{ - const seqStore_t* seqStore; - const seqDef* seqs; - size_t seqsSize; - - size_t i; int repIdx; size_t position; - - size_t blockSize = ZSTD_getBlockSize(zc); - size_t maxOutput = ZSTD_compressBound(blockSize); - - assert(!ZSTD_isError(ZSTD_compressBegin_usingCDict(zc, cdict))); - assert(dstSize >= maxOutput); dstSize = maxOutput; - assert(srcSize >= blockSize); srcSize = blockSize; - assert(!ZSTD_isError(ZSTD_compressBlock(zc, dst, dstSize, src, srcSize))); - - seqStore = ZSTD_getSeqStore(zc); - seqs = seqStore->sequencesStart; - seqsSize = seqStore->sequences - seqStore->sequencesStart; - - assert(outSeqsSize >= seqsSize); outSeqsSize = seqsSize; - - for (i = 0, position = 0; i < seqsSize; ++i) { - outSeqs[i].offset = seqs[i].offset; - outSeqs[i].litLength = seqs[i].litLength; - outSeqs[i].matchLength = seqs[i].matchLength + 3 /* min match */; - - if (i == seqStore->longLengthPos) { - if (seqStore->longLengthID == 1) { - outSeqs[i].litLength += 0x10000; - } else if (seqStore->longLengthID == 2) { - outSeqs[i].matchLength += 0x10000; - } - } - - if (outSeqs[i].offset <= 3 /* num reps */) { - outSeqs[i].rep = 1; - repIdx = i - outSeqs[i].offset; - - if (repIdx >= 0) { - outSeqs[i].offset = outSeqs[repIdx].offset; - } - - if (repIdx == -1) { - outSeqs[i].offset = 1; - } else if (repIdx == -2) { - outSeqs[i].offset = 4; - } else if (repIdx == -3) { - outSeqs[i].offset = 8; - } - } else { - outSeqs[i].offset -= 3 /* num reps */; - } - - position += outSeqs[i].litLength; - outSeqs[i].matchPos = position; - position += outSeqs[i].matchLength; - } -} - typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e; static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) @@ -2394,6 +2324,81 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, ZSTD_CCtx_params } } +static void ZSTD_copyBlockSequences(const seqStore_t* seqStore, seqDef* seqs, + ZSTD_Sequence* outSeqs, size_t seqsSize) +{ + size_t i; size_t position; int repIdx; + for (i = 0, position = 0; i < seqsSize; ++i) { + outSeqs[i].offset = seqs[i].offset; + outSeqs[i].litLength = seqs[i].litLength; + outSeqs[i].matchLength = seqs[i].matchLength + 3 /* min match */; + + if (i == seqStore->longLengthPos) { + if (seqStore->longLengthID == 1) { + outSeqs[i].litLength += 0x10000; + } else if (seqStore->longLengthID == 2) { + outSeqs[i].matchLength += 0x10000; + } + } + + if (outSeqs[i].offset <= 3 /* num reps */) { + outSeqs[i].rep = 1; + repIdx = i - outSeqs[i].offset; + + if (repIdx >= 0) { + outSeqs[i].offset = outSeqs[repIdx].offset; + } + + if (repIdx == -1) { + outSeqs[i].offset = 1; + } else if (repIdx == -2) { + outSeqs[i].offset = 4; + } else if (repIdx == -3) { + outSeqs[i].offset = 8; + } + } else { + outSeqs[i].offset -= 3 /* num reps */; + } + + position += outSeqs[i].litLength; + outSeqs[i].matchPos = position; + position += outSeqs[i].matchLength; + } +} + +static void ZSTD_getBlockSequences(ZSTD_CCtx* cctx, const seqStore_t* seqStore) +{ + size_t seqsSize = seqStore->sequences - seqStore->sequencesStart; + + assert(cctx->seqCollector.maxSequences > + (cctx->seqCollector.seqCurrent - cctx->seqCollector.seqStart) + seqsSize); + + ZSTD_copyBlockSequences(seqStore, seqStore->sequencesStart, + cctx->seqCollector.seqCurrent, seqsSize); + cctx->seqCollector.seqCurrent += seqsSize; +} + +size_t ZSTD_getSequences(ZSTD_CCtx* zc, const void* src, + size_t srcSize, ZSTD_Sequence* outSeqs, size_t outSeqsSize, + int level) +{ + size_t dstCapacity = ZSTD_compressBound(srcSize * sizeof(void*)); + void* dst = malloc(dstCapacity); + size_t seqsSize; + + SeqCollector seqCollector; + seqCollector.collectSequences = 1; + seqCollector.seqStart = outSeqs; + seqCollector.seqCurrent = outSeqs; + seqCollector.maxSequences = outSeqsSize; + zc->seqCollector = seqCollector; + + ZSTD_compressCCtx(zc, dst, dstCapacity, src, srcSize, level); + seqsSize = zc->seqCollector.seqCurrent - zc->seqCollector.seqStart; + + free(dst); + return seqsSize; +} /*! ZSTD_compress_frameChunk() : * Compress a chunk of data into one or multiple blocks. @@ -2438,6 +2443,9 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); FORWARD_IF_ERROR(cSize); + if (cctx->seqCollector.collectSequences) { + ZSTD_getBlockSequences(cctx, ZSTD_getSeqStore(cctx)); + } if (cSize == 0) { /* block is not compressible */ cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 6d623cc6..d40d5340 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -192,6 +192,13 @@ typedef struct { size_t capacity; /* The capacity starting from `seq` pointer */ } rawSeqStore_t; +typedef struct { + int collectSequences; + ZSTD_Sequence* seqStart; + ZSTD_Sequence* seqCurrent; + size_t maxSequences; +} SeqCollector; + struct ZSTD_CCtx_params_s { ZSTD_format_e format; ZSTD_compressionParameters cParams; @@ -238,6 +245,7 @@ struct ZSTD_CCtx_s { XXH64_state_t xxhState; ZSTD_customMem customMem; size_t staticSize; + SeqCollector seqCollector; seqStore_t seqStore; /* sequences storage ptrs */ ldmState_t ldmState; /* long distance matching state */ diff --git a/lib/zstd.h b/lib/zstd.h index f8e95f22..782940ef 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1072,6 +1072,14 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params; +typedef struct { + unsigned int matchPos; + unsigned int offset; + unsigned int litLength; + unsigned int matchLength; + int rep; +} ZSTD_Sequence; + typedef struct { unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ @@ -1210,6 +1218,9 @@ ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcS * or an error code (if srcSize is too small) */ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_getSequences(ZSTD_CCtx* zc, const void* src, + size_t srcSize, ZSTD_Sequence* outSeqs, size_t outSeqsSize, int level); + /*************************************** * Memory management diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 2de7c009..09fe4695 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1960,6 +1960,11 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "OK \n"); } + DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences zeros : ", testNb++); + memset(CNBuffer, 0, 1000000); + assert(ZSTD_getSequences(ZSTD_createCCtx(), CNBuffer, 1000000, + compressedBuffer, 1000000, 3) == 1000000 / 131071 + 1); + /* All zeroes test (test bug #137) */ #define ZEROESLENGTH 100 DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH); From 25878b2de955c5dbe65dddc28d5a5ac3856ef1f6 Mon Sep 17 00:00:00 2001 From: Mike Swanson Date: Mon, 19 Aug 2019 11:11:04 -0700 Subject: [PATCH 007/144] =?UTF-8?q?[programs]=20don=E2=80=99t=20do=20chmod?= =?UTF-8?q?=20when=20coming=20from=20stdin=20or=20multiple=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #1729 --- programs/fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index 20e2ee2a..185b4811 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -585,7 +585,7 @@ static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName, { FILE* const f = fopen( dstFileName, "wb" ); if (f == NULL) { DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno)); - } else { + } else if(srcFileName != NULL && strcmp (srcFileName, stdinmark)) { chmod(dstFileName, 00600); } return f; From 771645471f6e7fccc59d9a7e4c8c9c5ef1c6d1f8 Mon Sep 17 00:00:00 2001 From: Varun S Nair <> Date: Thu, 5 Sep 2019 15:28:30 +0530 Subject: [PATCH 008/144] Passing ZSTD_CCtx_params by const pointer --- lib/compress/zstd_compress.c | 108 +++++++++++++------------- lib/compress/zstd_compress_internal.h | 6 +- lib/compress/zstdmt_compress.c | 46 +++++------ 3 files changed, 80 insertions(+), 80 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index df4e828c..ec16860d 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -239,9 +239,9 @@ size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_paramete /* ZSTD_assignParamsToCCtxParams() : * params is presumed valid at this stage */ static ZSTD_CCtx_params ZSTD_assignParamsToCCtxParams( - ZSTD_CCtx_params cctxParams, ZSTD_parameters params) + const ZSTD_CCtx_params* cctxParams, ZSTD_parameters params) { - ZSTD_CCtx_params ret = cctxParams; + ZSTD_CCtx_params ret = *cctxParams; ret.cParams = params.cParams; ret.fParams = params.fParams; ret.compressionLevel = ZSTD_CLEVEL_DEFAULT; /* should not matter, as all cParams are presumed properly defined */ @@ -1272,24 +1272,24 @@ static U32 ZSTD_sufficientBuff(size_t bufferSize1, size_t maxNbSeq1, } /** Equivalence for resetCCtx purposes */ -static U32 ZSTD_equivalentParams(ZSTD_CCtx_params params1, - ZSTD_CCtx_params params2, +static U32 ZSTD_equivalentParams(const ZSTD_CCtx_params* params1, + const ZSTD_CCtx_params* params2, size_t buffSize1, size_t maxNbSeq1, size_t maxNbLit1, ZSTD_buffered_policy_e buffPol2, U64 pledgedSrcSize) { DEBUGLOG(4, "ZSTD_equivalentParams: pledgedSrcSize=%u", (U32)pledgedSrcSize); - if (!ZSTD_equivalentCParams(params1.cParams, params2.cParams)) { + if (!ZSTD_equivalentCParams(params1->cParams, params2->cParams)) { DEBUGLOG(4, "ZSTD_equivalentCParams() == 0"); return 0; } - if (!ZSTD_equivalentLdmParams(params1.ldmParams, params2.ldmParams)) { + if (!ZSTD_equivalentLdmParams(params1->ldmParams, params2->ldmParams)) { DEBUGLOG(4, "ZSTD_equivalentLdmParams() == 0"); return 0; } if (!ZSTD_sufficientBuff(buffSize1, maxNbSeq1, maxNbLit1, buffPol2, - params2.cParams, pledgedSrcSize)) { + params2->cParams, pledgedSrcSize)) { DEBUGLOG(4, "ZSTD_sufficientBuff() == 0"); return 0; } @@ -1323,15 +1323,15 @@ static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms) /*! ZSTD_continueCCtx() : * reuse CCtx without reset (note : requires no dictionary) */ -static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pledgedSrcSize) +static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params, U64 pledgedSrcSize) { - size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params.cParams.windowLog), pledgedSrcSize)); + size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize)); size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); DEBUGLOG(4, "ZSTD_continueCCtx: re-use context in place"); cctx->blockSize = blockSize; /* previous block size could be different even for same windowLog, due to pledgedSrcSize */ - cctx->appliedParams = params; - cctx->blockState.matchState.cParams = params.cParams; + cctx->appliedParams = *params; + cctx->blockState.matchState.cParams = params->cParams; cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; cctx->consumedSrcSize = 0; cctx->isFirstBlock = 1; @@ -1342,7 +1342,7 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pl (U32)pledgedSrcSize, cctx->appliedParams.fParams.contentSizeFlag); cctx->stage = ZSTDcs_init; cctx->dictID = 0; - if (params.ldmParams.enableLdm) + if (params->ldmParams.enableLdm) ZSTD_window_clear(&cctx->ldmState.window); ZSTD_referenceExternalSequences(cctx, NULL, 0); ZSTD_invalidateMatchState(&cctx->blockState.matchState); @@ -1439,7 +1439,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->isFirstBlock = 1; if (crp == ZSTDcrp_continue) { - if (ZSTD_equivalentParams(zc->appliedParams, params, + if (ZSTD_equivalentParams(&zc->appliedParams, ¶ms, zc->inBuffSize, zc->seqStore.maxNbSeq, zc->seqStore.maxNbLit, zbuff, pledgedSrcSize) ) { @@ -1455,7 +1455,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ¶ms.cParams, crp, ZSTD_resetTarget_CCtx); } - return ZSTD_continueCCtx(zc, params, pledgedSrcSize); + return ZSTD_continueCCtx(zc, ¶ms, pledgedSrcSize); } } } DEBUGLOG(4, "ZSTD_equivalentParams()==0 -> reset CCtx"); @@ -1626,15 +1626,15 @@ static const size_t attachDictSizeCutoffs[ZSTD_STRATEGY_MAX+1] = { }; static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, + const ZSTD_CCtx_params* params, U64 pledgedSrcSize) { size_t cutoff = attachDictSizeCutoffs[cdict->matchState.cParams.strategy]; return ( pledgedSrcSize <= cutoff || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN - || params.attachDictPref == ZSTD_dictForceAttach ) - && params.attachDictPref != ZSTD_dictForceCopy - && !params.forceWindow; /* dictMatchState isn't correctly + || params->attachDictPref == ZSTD_dictForceAttach ) + && params->attachDictPref != ZSTD_dictForceCopy + && !params->forceWindow; /* dictMatchState isn't correctly * handled in _enforceMaxDist */ } @@ -1746,7 +1746,7 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, * in-place. We decide here which strategy to use. */ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, + const ZSTD_CCtx_params* params, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff) { @@ -1756,10 +1756,10 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, if (ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) { return ZSTD_resetCCtx_byAttachingCDict( - cctx, cdict, params, pledgedSrcSize, zbuff); + cctx, cdict, *params, pledgedSrcSize, zbuff); } else { return ZSTD_resetCCtx_byCopyingCDict( - cctx, cdict, params, pledgedSrcSize, zbuff); + cctx, cdict, *params, pledgedSrcSize, zbuff); } } @@ -2452,25 +2452,25 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, - ZSTD_CCtx_params params, U64 pledgedSrcSize, U32 dictID) + const ZSTD_CCtx_params* params, U64 pledgedSrcSize, U32 dictID) { BYTE* const op = (BYTE*)dst; U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ - U32 const dictIDSizeCode = params.fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */ - U32 const checksumFlag = params.fParams.checksumFlag>0; - U32 const windowSize = (U32)1 << params.cParams.windowLog; - U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); - BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); - U32 const fcsCode = params.fParams.contentSizeFlag ? + U32 const dictIDSizeCode = params->fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */ + U32 const checksumFlag = params->fParams.checksumFlag>0; + U32 const windowSize = (U32)1 << params->cParams.windowLog; + U32 const singleSegment = params->fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); + BYTE const windowLogByte = (BYTE)((params->cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); + U32 const fcsCode = params->fParams.contentSizeFlag ? (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : 0; /* 0-3 */ BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) ); size_t pos=0; - assert(!(params.fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)); + assert(!(params->fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)); RETURN_ERROR_IF(dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX, dstSize_tooSmall); DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u", !params.fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode); - if (params.format == ZSTD_f_zstd1) { + if (params->format == ZSTD_f_zstd1) { MEM_writeLE32(dst, ZSTD_MAGICNUMBER); pos = 4; } @@ -2536,7 +2536,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, "missing init (ZSTD_compressBegin)"); if (frame && (cctx->stage==ZSTDcs_init)) { - fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams, + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, cctx->pledgedSrcSizePlusOne-1, cctx->dictID); FORWARD_IF_ERROR(fhSize); assert(fhSize <= dstCapacity); @@ -2832,10 +2832,10 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, ZSTD_dictContentType_e dictContentType, ZSTD_dictTableLoadMethod_e dtlm, const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, U64 pledgedSrcSize, + const ZSTD_CCtx_params* params, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff) { - DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params.cParams.windowLog); + DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params->cParams.windowLog); /* params are supposed to be fully validated at this point */ assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); assert(!((dict) && (cdict))); /* either dict or cdict, not both */ @@ -2844,11 +2844,11 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); } - FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, *params, pledgedSrcSize, ZSTDcrp_continue, zbuff) ); { size_t const dictID = ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, - ¶ms, dict, dictSize, dictContentType, dtlm, cctx->entropyWorkspace); + params, dict, dictSize, dictContentType, dtlm, cctx->entropyWorkspace); FORWARD_IF_ERROR(dictID); assert(dictID <= UINT_MAX); cctx->dictID = (U32)dictID; @@ -2861,12 +2861,12 @@ size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, ZSTD_dictContentType_e dictContentType, ZSTD_dictTableLoadMethod_e dtlm, const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, + const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize) { - DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params.cParams.windowLog); + DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params->cParams.windowLog); /* compression parameters verification and optimization */ - FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) ); + FORWARD_IF_ERROR( ZSTD_checkCParams(params->cParams) ); return ZSTD_compressBegin_internal(cctx, dict, dictSize, dictContentType, dtlm, cdict, @@ -2881,21 +2881,21 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, ZSTD_parameters params, unsigned long long pledgedSrcSize) { ZSTD_CCtx_params const cctxParams = - ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); + ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, params); return ZSTD_compressBegin_advanced_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL /*cdict*/, - cctxParams, pledgedSrcSize); + &cctxParams, pledgedSrcSize); } size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_parameters const params = ZSTD_getParams(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); ZSTD_CCtx_params const cctxParams = - ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); + ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, params); DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize); return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, - cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered); + &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered); } size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel) @@ -2918,7 +2918,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) /* special case : empty frame */ if (cctx->stage == ZSTDcs_init) { - fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams, 0, 0); + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, 0, 0); FORWARD_IF_ERROR(fhSize); dstCapacity -= fhSize; op += fhSize; @@ -2979,13 +2979,13 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx, ZSTD_parameters params) { ZSTD_CCtx_params const cctxParams = - ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); + ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, params); DEBUGLOG(4, "ZSTD_compress_internal"); return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, - cctxParams); + &cctxParams); } size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx, @@ -3009,7 +3009,7 @@ size_t ZSTD_compress_advanced_internal( void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict,size_t dictSize, - ZSTD_CCtx_params params) + const ZSTD_CCtx_params* params) { DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", (unsigned)srcSize); FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, @@ -3025,9 +3025,9 @@ size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx, int compressionLevel) { ZSTD_parameters const params = ZSTD_getParams(compressionLevel, srcSize + (!srcSize), dict ? dictSize : 0); - ZSTD_CCtx_params cctxParams = ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); + ZSTD_CCtx_params cctxParams = ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, params); assert(params.fParams.contentSizeFlag == 1); - return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, cctxParams); + return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctxParams); } size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, @@ -3268,7 +3268,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced( return ZSTD_compressBegin_internal(cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, cdict, - params, pledgedSrcSize, + ¶ms, pledgedSrcSize, ZSTDb_not_buffered); } } @@ -3359,7 +3359,7 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* cctx, FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, dict, dictSize, dictContentType, ZSTD_dtlm_fast, cdict, - params, pledgedSrcSize, + ¶ms, pledgedSrcSize, ZSTDb_buffered) ); cctx->inToCompress = 0; @@ -3393,13 +3393,13 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss) * Assumption 2 : either dict, or cdict, is defined, not both */ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, const void* dict, size_t dictSize, const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, unsigned long long pledgedSrcSize) + const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize) { DEBUGLOG(4, "ZSTD_initCStream_internal"); FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); - assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); - zcs->requestedParams = params; + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); + zcs->requestedParams = *params; assert(!((dict) && (cdict))); /* either dict or cdict, not both */ if (dict) { FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) ); @@ -3452,7 +3452,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) ); - zcs->requestedParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); + zcs->requestedParams = ZSTD_assignParamsToCCtxParams(&zcs->requestedParams, params); FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) ); return 0; } diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 8bffa6a1..262a6309 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -914,7 +914,7 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, const void* dict, size_t dictSize, const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); + const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize); void ZSTD_resetSeqStore(seqStore_t* ssPtr); @@ -929,7 +929,7 @@ size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, ZSTD_dictContentType_e dictContentType, ZSTD_dictTableLoadMethod_e dtlm, const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, + const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize); /* ZSTD_compress_advanced_internal() : @@ -938,7 +938,7 @@ size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict,size_t dictSize, - ZSTD_CCtx_params params); + const ZSTD_CCtx_params* params); /* ZSTD_writeLastEmptyBlock() : diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 9e537b88..44cbd94b 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -668,7 +668,7 @@ static void ZSTDMT_compressionJob(void* jobDescription) /* init */ if (job->cdict) { - size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict, jobParams, job->fullFrameSize); + size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict, &jobParams, job->fullFrameSize); assert(job->firstJob); /* only allowed for first job */ if (ZSTD_isError(initError)) JOB_ERROR(initError); } else { /* srcStart points at reloaded section */ @@ -680,7 +680,7 @@ static void ZSTDMT_compressionJob(void* jobDescription) job->prefix.start, job->prefix.size, ZSTD_dct_rawContent, /* load dictionary in "content-only" mode (no header analysis) */ ZSTD_dtlm_fast, NULL, /*cdict*/ - jobParams, pledgedSrcSize); + &jobParams, pledgedSrcSize); if (ZSTD_isError(initError)) JOB_ERROR(initError); } } @@ -1028,9 +1028,9 @@ size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, /* Sets parameters relevant to the compression job, * initializing others to default values. */ -static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(ZSTD_CCtx_params const params) +static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(const ZSTD_CCtx_params* params) { - ZSTD_CCtx_params jobParams = params; + ZSTD_CCtx_params jobParams = *params; /* Clear parameters related to multithreading */ jobParams.forceWindow = 0; jobParams.nbWorkers = 0; @@ -1151,16 +1151,16 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx) /* ===== Multi-threaded compression ===== */ /* ------------------------------------------ */ -static unsigned ZSTDMT_computeTargetJobLog(ZSTD_CCtx_params const params) +static unsigned ZSTDMT_computeTargetJobLog(const ZSTD_CCtx_params* params) { unsigned jobLog; - if (params.ldmParams.enableLdm) { + if (params->ldmParams.enableLdm) { /* In Long Range Mode, the windowLog is typically oversized. * In which case, it's preferable to determine the jobSize * based on chainLog instead. */ - jobLog = MAX(21, params.cParams.chainLog + 4); + jobLog = MAX(21, params->cParams.chainLog + 4); } else { - jobLog = MAX(20, params.cParams.windowLog + 2); + jobLog = MAX(20, params->cParams.windowLog + 2); } return MIN(jobLog, (unsigned)ZSTDMT_JOBLOG_MAX); } @@ -1193,27 +1193,27 @@ static int ZSTDMT_overlapLog(int ovlog, ZSTD_strategy strat) return ovlog; } -static size_t ZSTDMT_computeOverlapSize(ZSTD_CCtx_params const params) +static size_t ZSTDMT_computeOverlapSize(const ZSTD_CCtx_params* params) { - int const overlapRLog = 9 - ZSTDMT_overlapLog(params.overlapLog, params.cParams.strategy); - int ovLog = (overlapRLog >= 8) ? 0 : (params.cParams.windowLog - overlapRLog); + int const overlapRLog = 9 - ZSTDMT_overlapLog(params->overlapLog, params->cParams.strategy); + int ovLog = (overlapRLog >= 8) ? 0 : (params->cParams.windowLog - overlapRLog); assert(0 <= overlapRLog && overlapRLog <= 8); - if (params.ldmParams.enableLdm) { + if (params->ldmParams.enableLdm) { /* In Long Range Mode, the windowLog is typically oversized. * In which case, it's preferable to determine the jobSize * based on chainLog instead. * Then, ovLog becomes a fraction of the jobSize, rather than windowSize */ - ovLog = MIN(params.cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) + ovLog = MIN(params->cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) - overlapRLog; } assert(0 <= ovLog && ovLog <= ZSTD_WINDOWLOG_MAX); - DEBUGLOG(4, "overlapLog : %i", params.overlapLog); + DEBUGLOG(4, "overlapLog : %i", params->overlapLog); DEBUGLOG(4, "overlap size : %i", 1 << ovLog); return (ovLog==0) ? 0 : (size_t)1 << ovLog; } static unsigned -ZSTDMT_computeNbJobs(ZSTD_CCtx_params params, size_t srcSize, unsigned nbWorkers) +ZSTDMT_computeNbJobs(const ZSTD_CCtx_params* params, size_t srcSize, unsigned nbWorkers) { assert(nbWorkers>0); { size_t const jobSizeTarget = (size_t)1 << ZSTDMT_computeTargetJobLog(params); @@ -1236,9 +1236,9 @@ static size_t ZSTDMT_compress_advanced_internal( const ZSTD_CDict* cdict, ZSTD_CCtx_params params) { - ZSTD_CCtx_params const jobParams = ZSTDMT_initJobCCtxParams(params); - size_t const overlapSize = ZSTDMT_computeOverlapSize(params); - unsigned const nbJobs = ZSTDMT_computeNbJobs(params, srcSize, params.nbWorkers); + ZSTD_CCtx_params const jobParams = ZSTDMT_initJobCCtxParams(¶ms); + size_t const overlapSize = ZSTDMT_computeOverlapSize(¶ms); + unsigned const nbJobs = ZSTDMT_computeNbJobs(¶ms, srcSize, params.nbWorkers); size_t const proposedJobSize = (srcSize + (nbJobs-1)) / nbJobs; size_t const avgJobSize = (((proposedJobSize-1) & 0x1FFFF) < 0x7FFF) ? proposedJobSize + 0xFFFF : proposedJobSize; /* avoid too small last block */ const char* const srcStart = (const char*)src; @@ -1256,7 +1256,7 @@ static size_t ZSTDMT_compress_advanced_internal( ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0]; DEBUGLOG(4, "ZSTDMT_compress_advanced_internal: fallback to single-thread mode"); if (cdict) return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, jobParams.fParams); - return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, NULL, 0, jobParams); + return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, NULL, 0, &jobParams); } assert(avgJobSize >= 256 KB); /* condition for ZSTD_compressBound(A) + ZSTD_compressBound(B) <= ZSTD_compressBound(A+B), required to compress directly into Dst (no additional buffer) */ @@ -1404,12 +1404,12 @@ size_t ZSTDMT_initCStream_internal( mtctx->singleBlockingThread = (pledgedSrcSize <= ZSTDMT_JOBSIZE_MIN); /* do not trigger multi-threading when srcSize is too small */ if (mtctx->singleBlockingThread) { - ZSTD_CCtx_params const singleThreadParams = ZSTDMT_initJobCCtxParams(params); + ZSTD_CCtx_params const singleThreadParams = ZSTDMT_initJobCCtxParams(¶ms); DEBUGLOG(5, "ZSTDMT_initCStream_internal: switch to single blocking thread mode"); assert(singleThreadParams.nbWorkers == 0); return ZSTD_initCStream_internal(mtctx->cctxPool->cctx[0], dict, dictSize, cdict, - singleThreadParams, pledgedSrcSize); + &singleThreadParams, pledgedSrcSize); } DEBUGLOG(4, "ZSTDMT_initCStream_internal: %u workers", params.nbWorkers); @@ -1435,11 +1435,11 @@ size_t ZSTDMT_initCStream_internal( mtctx->cdict = cdict; } - mtctx->targetPrefixSize = ZSTDMT_computeOverlapSize(params); + mtctx->targetPrefixSize = ZSTDMT_computeOverlapSize(¶ms); DEBUGLOG(4, "overlapLog=%i => %u KB", params.overlapLog, (U32)(mtctx->targetPrefixSize>>10)); mtctx->targetSectionSize = params.jobSize; if (mtctx->targetSectionSize == 0) { - mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(params); + mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(¶ms); } assert(mtctx->targetSectionSize <= (size_t)ZSTDMT_JOBSIZE_MAX); From 98165606498d08345edc55a690783e1bd796041b Mon Sep 17 00:00:00 2001 From: Varun S Nair <> Date: Thu, 5 Sep 2019 15:47:17 +0530 Subject: [PATCH 009/144] Fixing assert and DEBUGLOG due to ZSTD_CCtx_params parameter change to const pointer --- lib/compress/zstd_compress.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index ec16860d..43cfedc9 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2468,7 +2468,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, assert(!(params->fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)); RETURN_ERROR_IF(dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX, dstSize_tooSmall); DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u", - !params.fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode); + !params->fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode); if (params->format == ZSTD_f_zstd1) { MEM_writeLE32(dst, ZSTD_MAGICNUMBER); @@ -2837,7 +2837,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, { DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params->cParams.windowLog); /* params are supposed to be fully validated at this point */ - assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); assert(!((dict) && (cdict))); /* either dict or cdict, not both */ if (cdict && cdict->dictContentSize>0) { From 7f98b46876c475d8a0ef4f895b83730d55a60f9b Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Thu, 5 Sep 2019 16:03:35 -0700 Subject: [PATCH 010/144] adding support for -O flag: multiple files into one directory for compressions (decompression to come) --- programs/fileio.c | 37 +++++++++++++++++++++---------------- programs/fileio.h | 15 ++++++++------- programs/util.c | 43 +++++++++++++++++++++++++++++++++++++++++++ programs/util.h | 4 ++++ programs/zstdcli.c | 26 +++++++++++++++++++++++++- 5 files changed, 101 insertions(+), 24 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 20e2ee2a..f2577186 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1276,9 +1276,7 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs, int result; stat_t statbuf; int transfer_permissions = 0; - assert(ress.srcFile != NULL); - if (ress.dstFile == NULL) { closeDstFile = 1; DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s", dstFileName); @@ -1369,11 +1367,9 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs, return result; } - -int FIO_compressFilename(FIO_prefs_t* const prefs, - const char* dstFileName, const char* srcFileName, - const char* dictFileName, int compressionLevel, - ZSTD_compressionParameters comprParams) +int FIO_compressFilename(FIO_prefs_t* const prefs, const char* dstFileName, + const char* srcFileName, const char* dictFileName, + int compressionLevel, ZSTD_compressionParameters comprParams) { cRess_t const ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams); int const result = FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); @@ -1416,22 +1412,30 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix) /* FIO_compressMultipleFilenames() : * compress nbFiles files - * into one destination (outFileName) - * or into one file each (outFileName == NULL, but suffix != NULL). + * into either one destination (outFileName), + * or into one file each (outFileName == NULL, but suffix != NULL), + * or into a destination folder (specified with -O) */ -int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, - const char** inFileNamesTable, unsigned nbFiles, - const char* outFileName, const char* suffix, - const char* dictFileName, int compressionLevel, - ZSTD_compressionParameters comprParams) +int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable, + const char* outDirName, char** dstFileNamesTable, + unsigned nbFiles, const char* outFileName, + const char* suffix, const char* dictFileName, + int compressionLevel, ZSTD_compressionParameters comprParams) { + printf("compressing multiple...\n"); int error = 0; cRess_t ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams); /* init */ assert(outFileName != NULL || suffix != NULL); - - if (outFileName != NULL) { /* output into a single destination (stdout typically) */ + if (outDirName != NULL) { /* output into a particular folder */ + unsigned u; + for (u = 0; u < nbFiles; ++u) { + const char* const srcFileName = inFileNamesTable[u]; + const char* const dstFileName = FIO_determineCompressedName(dstFileNamesTable[u], suffix); + error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); + } + } else if (outFileName != NULL) { /* output into a single destination (stdout typically) */ ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName); if (ress.dstFile == NULL) { /* could not open outFileName */ error = 1; @@ -1453,6 +1457,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, } } FIO_freeCResources(ress); + UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); return error; } diff --git a/programs/fileio.h b/programs/fileio.h index 096d90b5..1ed6d153 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -87,8 +87,9 @@ void FIO_setNotificationLevel(int level); /** FIO_compressFilename() : @return : 0 == ok; 1 == pb with src file. */ int FIO_compressFilename (FIO_prefs_t* const prefs, - const char* outfilename, const char* infilename, const char* dictFileName, - int compressionLevel, ZSTD_compressionParameters comprParams); + const char* outfilename, const char* infilename, + const char* dictFileName, int compressionLevel, + ZSTD_compressionParameters comprParams); /** FIO_decompressFilename() : @return : 0 == ok; 1 == pb with src file. */ @@ -103,11 +104,11 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis ***************************************/ /** FIO_compressMultipleFilenames() : @return : nb of missing files */ -int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, - const char** srcNamesTable, unsigned nbFiles, - const char* outFileName, const char* suffix, - const char* dictFileName, int compressionLevel, - ZSTD_compressionParameters comprParams); +int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable, + const char* outDirName, char** dstFileNamesTable, + unsigned nbFiles, const char* outFileName, + const char* suffix, const char* dictFileName, + int compressionLevel, ZSTD_compressionParameters comprParams); /** FIO_decompressMultipleFilenames() : @return : nb of missing or skipped files */ diff --git a/programs/util.c b/programs/util.c index 347e7698..e8b599aa 100644 --- a/programs/util.c +++ b/programs/util.c @@ -87,6 +87,49 @@ U32 UTIL_isDirectory(const char* infilename) return 0; } +int UTIL_createDir(const char* outDirName) { + if (UTIL_isDirectory(outDirName)) { + return 0; /* no need to create if directory already exists */ + } + int r; +#if defined(_MSC_VER) + r = _mkdir(outDirName); + if (r || !UTIL_isDirectory(outDirName)) return 1; +#else + r = mkdir(outDirName, S_IRWXU | S_IRWXG | S_IRWXO); /* dir has all permissions */ + if (r || !UTIL_isDirectory(outDirName)) return 1; +#endif + return 0; +} + +void UTIL_createDestinationDirTable(const char** filenameTable, unsigned nbFiles, + const char* outDirName, char** dstFilenameTable) +{ + unsigned u; + char c; + c = '/'; + + /* duplicate source file table */ + for (u = 0; u < nbFiles; ++u) { + char* filename; + char* finalPath; + size_t finalPathLen; + finalPathLen = strlen(outDirName); + filename = strrchr(filenameTable[u], c); /* filename is the last bit of string after '/' */ + finalPathLen += strlen(filename); + dstFilenameTable[u] = (char*) malloc(finalPathLen * sizeof(char) + 1); + strcpy(dstFilenameTable[u], outDirName); + strcat(dstFilenameTable[u], filename); + } +} + +void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles) { + unsigned u; + for (u = 0; u < nbFiles; ++u) + free(dstDirTable[u]); + free((void*)dstDirTable); +} + int UTIL_isSameFile(const char* file1, const char* file2) { #if defined(_MSC_VER) diff --git a/programs/util.h b/programs/util.h index d6e5bb55..1c127326 100644 --- a/programs/util.h +++ b/programs/util.h @@ -127,8 +127,12 @@ int UTIL_fileExist(const char* filename); int UTIL_isRegularFile(const char* infilename); int UTIL_setFileStat(const char* filename, stat_t* statbuf); U32 UTIL_isDirectory(const char* infilename); +int UTIL_createDir(const char* outDirName); int UTIL_getFileStat(const char* infilename, stat_t* statbuf); int UTIL_isSameFile(const char* file1, const char* file2); +void UTIL_createDestinationDirTable(const char** filenameTable, unsigned filenameIdx, + const char* outDirName, char** dstFilenameTable); +void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles); U32 UTIL_isLink(const char* infilename); #define UTIL_FILESIZE_UNKNOWN ((U64)(-1)) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 98df728a..2d3254ae 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -118,6 +118,7 @@ static int usage(const char* programName) #endif DISPLAY( " -D file: use `file` as Dictionary \n"); DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n"); + DISPLAY( " -O directory: result(s) stored into `directory`, creates one if non-existent \n"); DISPLAY( " -f : overwrite output without prompting and (de)compress links \n"); DISPLAY( "--rm : remove source file(s) after successful de/compression \n"); DISPLAY( " -k : preserve source file(s) (default) \n"); @@ -562,6 +563,7 @@ int main(int argCount, const char* argv[]) adaptMax = MAXCLEVEL, rsyncable = 0, nextArgumentIsOutFileName = 0, + nextArgumentIsOutDirName = 0, nextArgumentIsMaxDict = 0, nextArgumentIsDictID = 0, nextArgumentsAreFiles = 0, @@ -583,9 +585,11 @@ int main(int argCount, const char* argv[]) unsigned recursive = 0; unsigned memLimit = 0; const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ + char** dstFilenameTable = (char**)malloc(argCount * sizeof(char*)); unsigned filenameIdx = 0; const char* programName = argv[0]; const char* outFileName = NULL; + const char* outDirName = NULL; const char* dictFileName = NULL; const char* suffix = ZSTD_EXTENSION; unsigned maxDictSize = g_defaultMaxDictSize; @@ -853,6 +857,9 @@ int main(int argCount, const char* argv[]) /* destination file name */ case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break; + /* destination directory name */ + case 'O': nextArgumentIsOutDirName=1; lastCommand=1; argument++; break; + /* limit decompression memory */ case 'M': argument++; @@ -965,6 +972,13 @@ int main(int argCount, const char* argv[]) continue; } + if (nextArgumentIsOutDirName) { + nextArgumentIsOutDirName = 0; + lastCommand = 0; + outDirName = argument; + continue; + } + /* add filename to list */ filenameTable[filenameIdx++] = argument; } @@ -1163,10 +1177,20 @@ int main(int argCount, const char* argv[]) if (adaptMin > cLevel) cLevel = adaptMin; if (adaptMax < cLevel) cLevel = adaptMax; + if (outDirName) { + int dirResult; + dirResult = UTIL_createDir(outDirName); + if (dirResult) DISPLAY("Directory creation unsuccessful \n"); + + UTIL_createDestinationDirTable(filenameTable, filenameIdx, outDirName, dstFilenameTable); + if (outFileName) { + outFileName = dstFilenameTable[0]; /* in case -O is called with single file */ + } + } if ((filenameIdx==1) && outFileName) operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams); else - operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); + operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, dstFilenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); #else (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */ DISPLAY("Compression not supported \n"); From 6beb3c0159a60a6c49334f24e32ae92df5dbba4d Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Thu, 5 Sep 2019 17:56:24 -0700 Subject: [PATCH 011/144] added decompression support --- programs/fileio.c | 30 ++++++++++++++++++++++++++---- programs/fileio.h | 5 +++++ programs/util.c | 10 +++++++--- programs/zstdcli.c | 17 +++++++---------- 4 files changed, 45 insertions(+), 17 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index f2577186..c8a971c2 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1422,7 +1422,6 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN const char* suffix, const char* dictFileName, int compressionLevel, ZSTD_compressionParameters comprParams) { - printf("compressing multiple...\n"); int error = 0; cRess_t ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams); @@ -1457,7 +1456,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN } } FIO_freeCResources(ress); - UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); + /*UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles);*/ return error; } @@ -2240,14 +2239,24 @@ FIO_determineDstName(const char* srcFileName) int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, - const char* srcNamesTable[], unsigned nbFiles, + const char** srcNamesTable, unsigned nbFiles, + const char* outDirName, char** dstFileNamesTable, const char* outFileName, const char* dictFileName) { int error = 0; dRess_t ress = FIO_createDResources(prefs, dictFileName); - if (outFileName) { + if (outDirName != NULL) { /* output into a particular folder */ + unsigned u; + for (u = 0; u < nbFiles; ++u) { + const char* const srcFileName = srcNamesTable[u]; + const char* const dstFileName = FIO_determineDstName(dstFileNamesTable[u]); + if (dstFileName == NULL) { error=1; continue; } + + error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName); + } + } else if (outFileName) { unsigned u; ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName); if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName); @@ -2268,10 +2277,23 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, } FIO_freeDResources(ress); + /* UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); */ return error; } +void FIO_processMultipleFilenameDestinationDir(char** dstFilenameTable, + const char** filenameTable, unsigned filenameIdx, + const char* outFileName, const char* outDirName) { + int dirResult; + dirResult = UTIL_createDir(outDirName); + if (dirResult) + DISPLAY("Directory creation unsuccessful \n"); + UTIL_createDestinationDirTable(filenameTable, filenameIdx, outDirName, dstFilenameTable); + if (outFileName) { + outFileName = dstFilenameTable[0]; /* in case -O is called with single file */ + } +} /* ************************************************************************** * .zst file info (--list command) diff --git a/programs/fileio.h b/programs/fileio.h index 1ed6d153..1b435c5f 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -114,9 +114,14 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN @return : nb of missing or skipped files */ int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, const char** srcNamesTable, unsigned nbFiles, + const char* outDirName, char** dstFilenameTable, const char* outFileName, const char* dictFileName); +void FIO_processMultipleFilenameDestinationDir(char** dstFilenameTable, + const char** filenameTable, unsigned filenameIdx, + const char* outFileName, const char* outDirName); + /*-************************************* * Advanced stuff (should actually be hosted elsewhere) diff --git a/programs/util.c b/programs/util.c index e8b599aa..448a7889 100644 --- a/programs/util.c +++ b/programs/util.c @@ -108,6 +108,7 @@ void UTIL_createDestinationDirTable(const char** filenameTable, unsigned nbFiles unsigned u; char c; c = '/'; + printf("NBFILE: %u\n", nbFiles); /* duplicate source file table */ for (u = 0; u < nbFiles; ++u) { @@ -120,14 +121,17 @@ void UTIL_createDestinationDirTable(const char** filenameTable, unsigned nbFiles dstFilenameTable[u] = (char*) malloc(finalPathLen * sizeof(char) + 1); strcpy(dstFilenameTable[u], outDirName); strcat(dstFilenameTable[u], filename); + printf("%s %s\n", filenameTable[u], dstFilenameTable[u]); } } void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles) { unsigned u; - for (u = 0; u < nbFiles; ++u) - free(dstDirTable[u]); - free((void*)dstDirTable); + for (u = 0; u < nbFiles; ++u) { + if (dstDirTable[u] != NULL) + free(dstDirTable[u]); + } + if (dstDirTable != NULL) free((void*)dstDirTable); } int UTIL_isSameFile(const char* file1, const char* file2) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 2d3254ae..7c00b3d7 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1177,16 +1177,9 @@ int main(int argCount, const char* argv[]) if (adaptMin > cLevel) cLevel = adaptMin; if (adaptMax < cLevel) cLevel = adaptMax; - if (outDirName) { - int dirResult; - dirResult = UTIL_createDir(outDirName); - if (dirResult) DISPLAY("Directory creation unsuccessful \n"); + if (outDirName) + FIO_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); - UTIL_createDestinationDirTable(filenameTable, filenameIdx, outDirName, dstFilenameTable); - if (outFileName) { - outFileName = dstFilenameTable[0]; /* in case -O is called with single file */ - } - } if ((filenameIdx==1) && outFileName) operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams); else @@ -1205,10 +1198,14 @@ int main(int argCount, const char* argv[]) } } FIO_setMemLimit(prefs, memLimit); + + if (outDirName) + FIO_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); + if (filenameIdx==1 && outFileName) operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName); else - operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, dictFileName); + operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, dstFilenameTable, outFileName, dictFileName); #else DISPLAY("Decompression not supported \n"); #endif From a9c807a9489cc1c313ab027c8206fb95eb1adc0e Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Fri, 6 Sep 2019 10:17:04 -0700 Subject: [PATCH 012/144] kill memory leaks, cleanup, fix some dumb bugs --- programs/fileio.c | 7 ++----- programs/util.c | 14 ++++++-------- programs/zstdcli.c | 11 +++++++---- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index c8a971c2..194ea85c 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1390,10 +1390,8 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix) { static size_t dfnbCapacity = 0; static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ - size_t const sfnSize = strlen(srcFileName); size_t const suffixSize = strlen(suffix); - if (dfnbCapacity <= sfnSize+suffixSize+1) { /* resize buffer for dstName */ free(dstFileNameBuffer); @@ -1405,7 +1403,6 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix) assert(dstFileNameBuffer != NULL); memcpy(dstFileNameBuffer, srcFileName, sfnSize); memcpy(dstFileNameBuffer+sfnSize, suffix, suffixSize+1 /* Include terminating null */); - return dstFileNameBuffer; } @@ -1456,7 +1453,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN } } FIO_freeCResources(ress); - /*UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles);*/ + UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); return error; } @@ -2277,7 +2274,7 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, } FIO_freeDResources(ress); - /* UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); */ + UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); return error; } diff --git a/programs/util.c b/programs/util.c index 448a7889..3b134f9c 100644 --- a/programs/util.c +++ b/programs/util.c @@ -87,11 +87,12 @@ U32 UTIL_isDirectory(const char* infilename) return 0; } -int UTIL_createDir(const char* outDirName) { - if (UTIL_isDirectory(outDirName)) { - return 0; /* no need to create if directory already exists */ - } +int UTIL_createDir(const char* outDirName) +{ int r; + if (UTIL_isDirectory(outDirName)) + return 0; /* no need to create if directory already exists */ + #if defined(_MSC_VER) r = _mkdir(outDirName); if (r || !UTIL_isDirectory(outDirName)) return 1; @@ -108,20 +109,17 @@ void UTIL_createDestinationDirTable(const char** filenameTable, unsigned nbFiles unsigned u; char c; c = '/'; - printf("NBFILE: %u\n", nbFiles); /* duplicate source file table */ for (u = 0; u < nbFiles; ++u) { char* filename; - char* finalPath; size_t finalPathLen; finalPathLen = strlen(outDirName); filename = strrchr(filenameTable[u], c); /* filename is the last bit of string after '/' */ finalPathLen += strlen(filename); - dstFilenameTable[u] = (char*) malloc(finalPathLen * sizeof(char) + 1); + dstFilenameTable[u] = (char*) malloc((finalPathLen+5) * sizeof(char)); /* extra 1 bit for \0, extra 4 for .zst if compressing*/ strcpy(dstFilenameTable[u], outDirName); strcat(dstFilenameTable[u], filename); - printf("%s %s\n", filenameTable[u], dstFilenameTable[u]); } } diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 7c00b3d7..9029f634 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -585,7 +585,7 @@ int main(int argCount, const char* argv[]) unsigned recursive = 0; unsigned memLimit = 0; const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ - char** dstFilenameTable = (char**)malloc(argCount * sizeof(char*)); + char** dstFilenameTable; unsigned filenameIdx = 0; const char* programName = argv[0]; const char* outFileName = NULL; @@ -1177,8 +1177,10 @@ int main(int argCount, const char* argv[]) if (adaptMin > cLevel) cLevel = adaptMin; if (adaptMax < cLevel) cLevel = adaptMax; - if (outDirName) + if (outDirName) { + dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); FIO_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); + } if ((filenameIdx==1) && outFileName) operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams); @@ -1199,9 +1201,10 @@ int main(int argCount, const char* argv[]) } FIO_setMemLimit(prefs, memLimit); - if (outDirName) + if (outDirName) { + dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); FIO_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); - + } if (filenameIdx==1 && outFileName) operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName); else From d687d603e4a451de9620f5ffd548d29dd17000c4 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Fri, 6 Sep 2019 10:46:19 -0700 Subject: [PATCH 013/144] Removing redundant condition in decompression, making first block rles valid to deocmpress --- lib/compress/zstd_compress.c | 8 -------- lib/compress/zstd_compress_internal.h | 1 - programs/fileio.c | 5 ----- 3 files changed, 14 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 62cab4ed..4ba77980 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1314,7 +1314,6 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pl cctx->blockState.matchState.cParams = params.cParams; cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; cctx->consumedSrcSize = 0; - cctx->isFirstBlock = 1; cctx->producedCSize = 0; if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN) cctx->appliedParams.fParams.contentSizeFlag = 0; @@ -1417,7 +1416,6 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, (U32)pledgedSrcSize, params.cParams.windowLog); assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); - zc->isFirstBlock = 1; if (crp == ZSTDcrp_continue) { if (ZSTD_equivalentParams(zc->appliedParams, params, zc->inBuffSize, @@ -2307,11 +2305,6 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, zc->bmi2); if (frame && - /* We don't want to emit our first block as a RLE even if it qualifies because - * doing so will cause the decoder to throw a "should consume all input error." - * https://github.com/facebook/zstd/blob/dev/programs/fileio.c#L1723 - */ - !zc->isFirstBlock && cSize < rleMaxLength && ZSTD_isRLE(ip, srcSize)) { @@ -2416,7 +2409,6 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, op += cSize; assert(dstCapacity >= cSize); dstCapacity -= cSize; - cctx->isFirstBlock = 0; DEBUGLOG(5, "ZSTD_compress_frameChunk: adding a block of size %u", (unsigned)cSize); } } diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 3a5c7f2d..6d623cc6 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -238,7 +238,6 @@ struct ZSTD_CCtx_s { XXH64_state_t xxhState; ZSTD_customMem customMem; size_t staticSize; - int isFirstBlock; seqStore_t seqStore; /* sequences storage ptrs */ ldmState_t ldmState; /* long distance matching state */ diff --git a/programs/fileio.c b/programs/fileio.c index 569a410c..d63742d4 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1711,11 +1711,6 @@ static unsigned long long FIO_decompressZstdFrame( } if (readSizeHint == 0) break; /* end of frame */ - if (inBuff.size != inBuff.pos) { - DISPLAYLEVEL(1, "%s : Decoding error (37) : should consume entire input \n", - srcFileName); - return FIO_ERROR_FRAME_DECODING; - } /* Fill input buffer */ { size_t const toDecode = MIN(readSizeHint, ress->srcBufferSize); /* support large skippable frames */ From 30bff50e06d84d1d426fee62acd34223abc69f1b Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Fri, 6 Sep 2019 11:10:53 -0700 Subject: [PATCH 014/144] fixes for tests and segfault --- programs/fileio.c | 22 ++++++---------------- programs/fileio.h | 4 ---- programs/util.c | 14 ++++++++++++++ programs/util.h | 3 +++ programs/zstdcli.c | 5 +++-- 5 files changed, 26 insertions(+), 22 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 194ea85c..cc8809f9 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1453,7 +1453,9 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN } } FIO_freeCResources(ress); - UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); + if (outDirName) + UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); + return error; } @@ -2274,24 +2276,12 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, } FIO_freeDResources(ress); - UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); + if (outDirName) + UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); + return error; } -void FIO_processMultipleFilenameDestinationDir(char** dstFilenameTable, - const char** filenameTable, unsigned filenameIdx, - const char* outFileName, const char* outDirName) { - int dirResult; - dirResult = UTIL_createDir(outDirName); - if (dirResult) - DISPLAY("Directory creation unsuccessful \n"); - - UTIL_createDestinationDirTable(filenameTable, filenameIdx, outDirName, dstFilenameTable); - if (outFileName) { - outFileName = dstFilenameTable[0]; /* in case -O is called with single file */ - } -} - /* ************************************************************************** * .zst file info (--list command) ***************************************************************************/ diff --git a/programs/fileio.h b/programs/fileio.h index 1b435c5f..40cd8985 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -118,10 +118,6 @@ int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, const char* outFileName, const char* dictFileName); -void FIO_processMultipleFilenameDestinationDir(char** dstFilenameTable, - const char** filenameTable, unsigned filenameIdx, - const char* outFileName, const char* outDirName); - /*-************************************* * Advanced stuff (should actually be hosted elsewhere) diff --git a/programs/util.c b/programs/util.c index 3b134f9c..f00bfd60 100644 --- a/programs/util.c +++ b/programs/util.c @@ -123,6 +123,20 @@ void UTIL_createDestinationDirTable(const char** filenameTable, unsigned nbFiles } } +void UTIL_processMultipleFilenameDestinationDir(char** dstFilenameTable, + const char** filenameTable, unsigned filenameIdx, + const char* outFileName, const char* outDirName) { + int dirResult; + dirResult = UTIL_createDir(outDirName); + if (dirResult) + UTIL_DISPLAYLEVEL(1, "Directory creation unsuccessful\n"); + + UTIL_createDestinationDirTable(filenameTable, filenameIdx, outDirName, dstFilenameTable); + if (outFileName) { + outFileName = dstFilenameTable[0]; /* in case -O is called with single file */ + } +} + void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles) { unsigned u; for (u = 0; u < nbFiles; ++u) { diff --git a/programs/util.h b/programs/util.h index 1c127326..9615504c 100644 --- a/programs/util.h +++ b/programs/util.h @@ -133,6 +133,9 @@ int UTIL_isSameFile(const char* file1, const char* file2); void UTIL_createDestinationDirTable(const char** filenameTable, unsigned filenameIdx, const char* outDirName, char** dstFilenameTable); void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles); +void UTIL_processMultipleFilenameDestinationDir(char** dstFilenameTable, + const char** filenameTable, unsigned filenameIdx, + const char* outFileName, const char* outDirName); U32 UTIL_isLink(const char* infilename); #define UTIL_FILESIZE_UNKNOWN ((U64)(-1)) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 9029f634..606c162f 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1178,8 +1178,9 @@ int main(int argCount, const char* argv[]) if (adaptMax < cLevel) cLevel = adaptMax; if (outDirName) { + printf("ok\n"); dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); - FIO_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); + UTIL_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); } if ((filenameIdx==1) && outFileName) @@ -1203,7 +1204,7 @@ int main(int argCount, const char* argv[]) if (outDirName) { dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); - FIO_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); + UTIL_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); } if (filenameIdx==1 && outFileName) operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName); From 62616c4d90557ab070ddd7677840e882cf0e34b9 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Fri, 6 Sep 2019 13:20:50 -0700 Subject: [PATCH 015/144] fixes for windows compilation --- programs/util.c | 7 +++++-- programs/zstdcli.c | 5 +++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/programs/util.c b/programs/util.c index f00bfd60..920601fb 100644 --- a/programs/util.c +++ b/programs/util.c @@ -20,6 +20,9 @@ extern "C" { #include #include +#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) +#include /* needed for _mkdir in windows */ +#endif int UTIL_fileExist(const char* filename) { @@ -93,7 +96,7 @@ int UTIL_createDir(const char* outDirName) if (UTIL_isDirectory(outDirName)) return 0; /* no need to create if directory already exists */ -#if defined(_MSC_VER) +#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) r = _mkdir(outDirName); if (r || !UTIL_isDirectory(outDirName)) return 1; #else @@ -112,7 +115,7 @@ void UTIL_createDestinationDirTable(const char** filenameTable, unsigned nbFiles /* duplicate source file table */ for (u = 0; u < nbFiles; ++u) { - char* filename; + const char* filename; size_t finalPathLen; finalPathLen = strlen(outDirName); filename = strrchr(filenameTable[u], c); /* filename is the last bit of string after '/' */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 606c162f..5fc6b8aa 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1181,6 +1181,8 @@ int main(int argCount, const char* argv[]) printf("ok\n"); dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); UTIL_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); + } else { + dstFilenameTable = NULL; } if ((filenameIdx==1) && outFileName) @@ -1205,7 +1207,10 @@ int main(int argCount, const char* argv[]) if (outDirName) { dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); UTIL_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); + } else { + dstFilenameTable = NULL; } + if (filenameIdx==1 && outFileName) operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName); else From a917cd597dc0a1a37e2300bc2d131c833999621c Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Fri, 6 Sep 2019 13:44:25 -0700 Subject: [PATCH 016/144] Put back omission for first rle block and updated comment as suggested --- lib/compress/zstd_compress.c | 8 ++++++++ lib/compress/zstd_compress_internal.h | 1 + 2 files changed, 9 insertions(+) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 4ba77980..f88cc82f 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1314,6 +1314,7 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pl cctx->blockState.matchState.cParams = params.cParams; cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; cctx->consumedSrcSize = 0; + cctx->isFirstBlock = 1; cctx->producedCSize = 0; if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN) cctx->appliedParams.fParams.contentSizeFlag = 0; @@ -1416,6 +1417,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, (U32)pledgedSrcSize, params.cParams.windowLog); assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + zc->isFirstBlock = 1; if (crp == ZSTDcrp_continue) { if (ZSTD_equivalentParams(zc->appliedParams, params, zc->inBuffSize, @@ -2305,6 +2307,11 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, zc->bmi2); if (frame && + /* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + !zc->isFirstBlock && cSize < rleMaxLength && ZSTD_isRLE(ip, srcSize)) { @@ -2409,6 +2416,7 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, op += cSize; assert(dstCapacity >= cSize); dstCapacity -= cSize; + cctx->isFirstBlock = 0; DEBUGLOG(5, "ZSTD_compress_frameChunk: adding a block of size %u", (unsigned)cSize); } } diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 6d623cc6..3a5c7f2d 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -238,6 +238,7 @@ struct ZSTD_CCtx_s { XXH64_state_t xxhState; ZSTD_customMem customMem; size_t staticSize; + int isFirstBlock; seqStore_t seqStore; /* sequences storage ptrs */ ldmState_t ldmState; /* long distance matching state */ From 2b0a271ed216169d96dafed32713b9a6cb5f0697 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 6 Sep 2019 14:30:13 -0700 Subject: [PATCH 017/144] fix eductional decoder fix #1774 also : - fix minor compilation warnings - make sure the `test` is run during CI tests --- Makefile | 1 + doc/educational_decoder/Makefile | 5 +++- doc/educational_decoder/zstd_decompress.c | 28 +++++++++++------------ doc/educational_decoder/zstd_decompress.h | 4 +++- lib/dictBuilder/cover.c | 4 ++-- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index acf50cfa..da3112e3 100644 --- a/Makefile +++ b/Makefile @@ -69,6 +69,7 @@ test: MOREFLAGS += -g -DDEBUGLEVEL=$(DEBUGLEVEL) -Werror test: MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants $(MAKE) -C $(TESTDIR) $@ + $(MAKE) -C doc/educational_decoder test ## shortest: same as `make check` .PHONY: shortest diff --git a/doc/educational_decoder/Makefile b/doc/educational_decoder/Makefile index c1d2c4cc..ed3f3fef 100644 --- a/doc/educational_decoder/Makefile +++ b/doc/educational_decoder/Makefile @@ -26,7 +26,10 @@ test: harness @./harness tmp.zst tmp @diff -s tmp README.md @$(RM) -f tmp* - @zstd --train harness.c zstd_decompress.c zstd_decompress.h README.md + # present files for training multiple times, to reach minimum threshold + @zstd --train harness.c zstd_decompress.c zstd_decompress.h README.md \ + harness.c zstd_decompress.c zstd_decompress.h README.md \ + harness.c zstd_decompress.c zstd_decompress.h README.md @zstd -D dictionary README.md -o tmp.zst @./harness tmp.zst tmp dictionary @diff -s tmp README.md diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c index 8e231bbb..26143cd1 100644 --- a/doc/educational_decoder/zstd_decompress.c +++ b/doc/educational_decoder/zstd_decompress.c @@ -395,7 +395,7 @@ size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len, /* this decoder assumes decompression of a single frame */ decode_frame(&out, &in, parsed_dict); - return out.ptr - (u8 *)dst; + return (size_t)(out.ptr - (u8 *)dst); } /******* FRAME DECODING ******************************************************/ @@ -416,7 +416,7 @@ static void decompress_data(frame_context_t *const ctx, ostream_t *const out, static void decode_frame(ostream_t *const out, istream_t *const in, const dictionary_t *const dict) { - const u32 magic_number = IO_read_bits(in, 32); + const u32 magic_number = (u32)IO_read_bits(in, 32); // Zstandard frame // // "Magic_Number @@ -497,7 +497,7 @@ static void parse_frame_header(frame_header_t *const header, // 3 Reserved_bit // 2 Content_Checksum_flag // 1-0 Dictionary_ID_flag" - const u8 descriptor = IO_read_bits(in, 8); + const u8 descriptor = (u8)IO_read_bits(in, 8); // decode frame header descriptor into flags const u8 frame_content_size_flag = descriptor >> 6; @@ -521,7 +521,7 @@ static void parse_frame_header(frame_header_t *const header, // // Bit numbers 7-3 2-0 // Field name Exponent Mantissa" - u8 window_descriptor = IO_read_bits(in, 8); + u8 window_descriptor = (u8)IO_read_bits(in, 8); u8 exponent = window_descriptor >> 3; u8 mantissa = window_descriptor & 7; @@ -541,7 +541,7 @@ static void parse_frame_header(frame_header_t *const header, const int bytes_array[] = {0, 1, 2, 4}; const int bytes = bytes_array[dictionary_id_flag]; - header->dictionary_id = IO_read_bits(in, bytes * 8); + header->dictionary_id = (u32)IO_read_bits(in, bytes * 8); } else { header->dictionary_id = 0; } @@ -633,8 +633,8 @@ static void decompress_data(frame_context_t *const ctx, ostream_t *const out, // // The next 2 bits represent the Block_Type, while the remaining 21 bits // represent the Block_Size. Format is little-endian." - last_block = IO_read_bits(in, 1); - const int block_type = IO_read_bits(in, 2); + last_block = (int)IO_read_bits(in, 1); + const int block_type = (int)IO_read_bits(in, 2); const size_t block_len = IO_read_bits(in, 21); switch (block_type) { @@ -748,8 +748,8 @@ static size_t decode_literals(frame_context_t *const ctx, istream_t *const in, // types" // // size_format takes between 1 and 2 bits - int block_type = IO_read_bits(in, 2); - int size_format = IO_read_bits(in, 2); + int block_type = (int)IO_read_bits(in, 2); + int size_format = (int)IO_read_bits(in, 2); if (block_type <= 1) { // Raw or RLE literals block @@ -1005,7 +1005,7 @@ static const i16 SEQ_MATCH_LENGTH_DEFAULT_DIST[53] = { static const u32 SEQ_LITERAL_LENGTH_BASELINES[36] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 22, 24, 28, 32, 40, - 48, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65538}; + 48, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536}; static const u8 SEQ_LITERAL_LENGTH_EXTRA_BITS[36] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; @@ -1021,7 +1021,7 @@ static const u8 SEQ_MATCH_LENGTH_EXTRA_BITS[53] = { 2, 2, 3, 3, 4, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; /// Offset decoding is simpler so we just need a maximum code value -static const u8 SEQ_MAX_CODES[3] = {35, -1, 52}; +static const u8 SEQ_MAX_CODES[3] = {35, (u8)-1, 52}; static void decompress_sequences(frame_context_t *const ctx, istream_t *const in, @@ -1132,7 +1132,7 @@ static void decompress_sequences(frame_context_t *const ctx, istream_t *in, // a single 1-bit and then fills the byte with 0-7 0 bits of padding." const int padding = 8 - highest_set_bit(src[len - 1]); // The offset starts at the end because FSE streams are read backwards - i64 bit_offset = len * 8 - padding; + i64 bit_offset = (i64)(len * 8 - (size_t)padding); // "The bitstream starts with initial state values, each using the required // number of bits in their respective accuracy, decoded previously from @@ -1409,7 +1409,7 @@ size_t ZSTD_get_decompressed_size(const void *src, const size_t src_len) { // get decompressed size from ZSTD frame header { - const u32 magic_number = IO_read_bits(&in, 32); + const u32 magic_number = (u32)IO_read_bits(&in, 32); if (magic_number == 0xFD2FB528U) { // ZSTD frame @@ -1418,7 +1418,7 @@ size_t ZSTD_get_decompressed_size(const void *src, const size_t src_len) { if (header.frame_content_size == 0 && !header.single_segment_flag) { // Content size not provided, we can't tell - return -1; + return (size_t)-1; } return header.frame_content_size; diff --git a/doc/educational_decoder/zstd_decompress.h b/doc/educational_decoder/zstd_decompress.h index a01fde33..74b18533 100644 --- a/doc/educational_decoder/zstd_decompress.h +++ b/doc/educational_decoder/zstd_decompress.h @@ -7,6 +7,8 @@ * in the COPYING file in the root directory of this source tree). */ +#include /* size_t */ + /******* EXPOSED TYPES ********************************************************/ /* * Contains the parsed contents of a dictionary @@ -39,7 +41,7 @@ size_t ZSTD_get_decompressed_size(const void *const src, const size_t src_len); * Return a valid dictionary_t pointer for use with dictionary initialization * or decompression */ -dictionary_t* create_dictionary(); +dictionary_t* create_dictionary(void); /* * Parse a provided dictionary blob for use in decompression diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index 4721205d..2e129dd9 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -638,8 +638,8 @@ void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers, int displayLeve "compared to the source size %u! " "size(source)/size(dictionary) = %f, but it should be >= " "10! This may lead to a subpar dictionary! We recommend " - "training on sources at least 10x, and up to 100x the " - "size of the dictionary!\n", (U32)maxDictSize, + "training on sources at least 10x, and preferably 100x " + "the size of the dictionary! \n", (U32)maxDictSize, (U32)nbDmers, ratio); } From 44e122053bba31d39cf0e280eddc48b8f7edfeb7 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Fri, 6 Sep 2019 14:48:41 -0700 Subject: [PATCH 018/144] Mentioning cli only in the comment as suggested --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index f88cc82f..f7f571e8 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2308,7 +2308,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, if (frame && /* We don't want to emit our first block as a RLE even if it qualifies because - * doing so will cause the decoder to throw a "should consume all input error." + * doing so will cause the decoder (cli only) to throw a "should consume all input error." * This is only an issue for zstd <= v1.4.3 */ !zc->isFirstBlock && From a3815d233c0ad35d6da7e03e5d71ac71b303977f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 6 Sep 2019 16:51:16 -0700 Subject: [PATCH 019/144] fix minor compilation condition harness.c is not designed to pass -Wdeclaration-after-statement --- doc/educational_decoder/Makefile | 6 +++--- doc/educational_decoder/harness.c | 2 +- doc/educational_decoder/zstd_decompress.c | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/educational_decoder/Makefile b/doc/educational_decoder/Makefile index ed3f3fef..fc080375 100644 --- a/doc/educational_decoder/Makefile +++ b/doc/educational_decoder/Makefile @@ -6,10 +6,10 @@ CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) CFLAGS ?= -O3 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ - -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ - -Wstrict-prototypes -Wundef \ + -Wstrict-aliasing=1 -Wswitch-enum \ + -Wredundant-decls -Wstrict-prototypes -Wundef \ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ - -Wredundant-decls + -std=c99 CFLAGS += $(DEBUGFLAGS) CFLAGS += $(MOREFLAGS) FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MULTITHREAD_LDFLAGS) diff --git a/doc/educational_decoder/harness.c b/doc/educational_decoder/harness.c index 47882b16..36f3967a 100644 --- a/doc/educational_decoder/harness.c +++ b/doc/educational_decoder/harness.c @@ -33,7 +33,7 @@ size_t read_file(const char *path, u8 **ptr) { } fseek(f, 0L, SEEK_END); - size_t size = ftell(f); + size_t size = (size_t)ftell(f); rewind(f); *ptr = malloc(size); diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c index 26143cd1..f3e1b848 100644 --- a/doc/educational_decoder/zstd_decompress.c +++ b/doc/educational_decoder/zstd_decompress.c @@ -833,6 +833,7 @@ static size_t decode_literals_compressed(frame_context_t *const ctx, // bits (0-1023)." num_streams = 1; // Fall through as it has the same size format + /* fallthrough */ case 1: // "4 streams. Both Compressed_Size and Regenerated_Size use 10 bits // (0-1023)." From a71bbba7bed5cb7f0e4bf6c793a314f29edf6ebd Mon Sep 17 00:00:00 2001 From: Dario Pavlovic Date: Mon, 9 Sep 2019 08:43:22 -0700 Subject: [PATCH 020/144] [Fuzz] Improve data generation #1723 --- tests/fuzz/Makefile | 4 +-- tests/fuzz/README.md | 2 +- tests/fuzz/fuzz_data_producer.c | 57 +++++++++++++++++++++++++++++++++ tests/fuzz/fuzz_data_producer.h | 43 +++++++++++++++++++++++++ tests/fuzz/simple_decompress.c | 7 ++-- 5 files changed, 108 insertions(+), 5 deletions(-) create mode 100644 tests/fuzz/fuzz_data_producer.c create mode 100644 tests/fuzz/fuzz_data_producer.h diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile index 08dedd66..83837e62 100644 --- a/tests/fuzz/Makefile +++ b/tests/fuzz/Makefile @@ -40,8 +40,8 @@ FUZZ_LDFLAGS := -pthread $(LDFLAGS) FUZZ_ARFLAGS := $(ARFLAGS) FUZZ_TARGET_FLAGS = $(FUZZ_CPPFLAGS) $(FUZZ_CXXFLAGS) $(FUZZ_LDFLAGS) -FUZZ_HEADERS := fuzz_helpers.h fuzz.h zstd_helpers.h -FUZZ_SRC := $(PRGDIR)/util.c zstd_helpers.c +FUZZ_HEADERS := fuzz_helpers.h fuzz.h zstd_helpers.h fuzz_data_producer.h +FUZZ_SRC := $(PRGDIR)/util.c zstd_helpers.c fuzz_data_producer.c ZSTDCOMMON_SRC := $(ZSTDDIR)/common/*.c ZSTDCOMP_SRC := $(ZSTDDIR)/compress/*.c diff --git a/tests/fuzz/README.md b/tests/fuzz/README.md index 856a57f8..71afa406 100644 --- a/tests/fuzz/README.md +++ b/tests/fuzz/README.md @@ -90,7 +90,7 @@ CC=afl-clang CXX=afl-clang++ ./fuzz.py build all --enable-asan --enable-ubsan ## Regression Testing -The regression rest supports the `all` target to run all the fuzzers in one +The regression test supports the `all` target to run all the fuzzers in one command. ``` diff --git a/tests/fuzz/fuzz_data_producer.c b/tests/fuzz/fuzz_data_producer.c new file mode 100644 index 00000000..a083f636 --- /dev/null +++ b/tests/fuzz/fuzz_data_producer.c @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + */ + +#include "fuzz_data_producer.h" + +struct FUZZ_dataProducer_s{ + const uint8_t *data; + size_t size; +}; + +FUZZ_dataProducer_t *FUZZ_dataProducer_create(const uint8_t *data, size_t size) { + FUZZ_dataProducer_t *producer = malloc(sizeof(FUZZ_dataProducer_t)); + + FUZZ_ASSERT(producer != NULL); + + producer->data = data; + producer->size = size; + return producer; +} + +void FUZZ_dataProducer_free(FUZZ_dataProducer_t *producer) { free(producer); } + +uint32_t FUZZ_dataProducer_uint32Range(FUZZ_dataProducer_t *producer, uint32_t min, + uint32_t max) { + FUZZ_ASSERT(min <= max); + + uint32_t range = max - min; + uint32_t rolling = range; + uint32_t result = 0; + + while (rolling > 0 && producer->size > 0) { + uint8_t next = *(producer->data + producer->size - 1); + producer->size -= 1; + result = (result << 8) | next; + rolling >>= 8; + } + + if (range == 0xffffffff) { + return result; + } + + return min + result % (range + 1); +} + +uint32_t FUZZ_dataProducer_uint32(FUZZ_dataProducer_t *producer) { + return FUZZ_dataProducer_uint32Range(producer, 0, 0xffffffff); +} + +size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer){ + return producer->size; +} diff --git a/tests/fuzz/fuzz_data_producer.h b/tests/fuzz/fuzz_data_producer.h new file mode 100644 index 00000000..4a12e130 --- /dev/null +++ b/tests/fuzz/fuzz_data_producer.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + */ + +/** + * Helper APIs for generating random data from input data stream. + */ + +#ifndef FUZZ_DATA_PRODUCER_H +#define FUZZ_DATA_PRODUCER_H + +#include +#include +#include +#include + +#include "fuzz_helpers.h" + +/* Struct used for maintaining the state of the data */ +typedef struct FUZZ_dataProducer_s FUZZ_dataProducer_t; + +/* Returns a data producer state struct. Use for producer initialization. */ +FUZZ_dataProducer_t *FUZZ_dataProducer_create(const uint8_t *data, size_t size); + +/* Frees the data producer */ +void FUZZ_dataProducer_free(FUZZ_dataProducer_t *producer); + +/* Returns value between [min, max] */ +uint32_t FUZZ_dataProducer_uint32Range(FUZZ_dataProducer_t *producer, uint32_t min, + uint32_t max); + +/* Returns a uint32 value */ +uint32_t FUZZ_dataProducer_uint32(FUZZ_dataProducer_t *producer); + +/* Returns the size of the remaining bytes of data in the producer */ +size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer); + +#endif // FUZZ_DATA_PRODUCER_H diff --git a/tests/fuzz/simple_decompress.c b/tests/fuzz/simple_decompress.c index af3f302b..56ebb93e 100644 --- a/tests/fuzz/simple_decompress.c +++ b/tests/fuzz/simple_decompress.c @@ -17,13 +17,14 @@ #include #include "fuzz_helpers.h" #include "zstd.h" +#include "fuzz_data_producer.h" static ZSTD_DCtx *dctx = NULL; int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - uint32_t seed = FUZZ_seed(&src, &size); + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); int i; if (!dctx) { dctx = ZSTD_createDCtx(); @@ -31,13 +32,15 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) } /* Run it 10 times over 10 output sizes. Reuse the context. */ for (i = 0; i < 10; ++i) { - size_t const bufSize = FUZZ_rand32(&seed, 0, 2 * size); + size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 2 * size); void* rBuf = malloc(bufSize); FUZZ_ASSERT(rBuf); ZSTD_decompressDCtx(dctx, rBuf, bufSize, src, size); free(rBuf); } + FUZZ_dataProducer_free(producer); + #ifndef STATEFUL_FUZZING ZSTD_freeDCtx(dctx); dctx = NULL; #endif From ccaac852e83b311881076d24048cada5944bdb22 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 15 Jul 2019 15:10:33 -0400 Subject: [PATCH 021/144] Normalize Case 'workSpace' -> 'workspace' --- lib/compress/zstd_compress.c | 56 +++++++++++++-------------- lib/compress/zstd_compress_internal.h | 6 +-- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 43cfedc9..61dbebac 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -89,13 +89,13 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize) if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ memset(workspace, 0, workspaceSize); /* may be a bit generous, could memset be smaller ? */ cctx->staticSize = workspaceSize; - cctx->workSpace = (void*)(cctx+1); - cctx->workSpaceSize = workspaceSize - sizeof(ZSTD_CCtx); + cctx->workspace = (void*)(cctx+1); + cctx->workspaceSize = workspaceSize - sizeof(ZSTD_CCtx); /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ - if (cctx->workSpaceSize < HUF_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t)) return NULL; - assert(((size_t)cctx->workSpace & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ - cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)cctx->workSpace; + if (cctx->workspaceSize < HUF_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t)) return NULL; + assert(((size_t)cctx->workspace & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ + cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)cctx->workspace; cctx->blockState.nextCBlock = cctx->blockState.prevCBlock + 1; { void* const ptr = cctx->blockState.nextCBlock + 1; @@ -128,7 +128,7 @@ static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) { assert(cctx != NULL); assert(cctx->staticSize == 0); - ZSTD_free(cctx->workSpace, cctx->customMem); cctx->workSpace = NULL; + ZSTD_free(cctx->workspace, cctx->customMem); cctx->workspace = NULL; ZSTD_clearAllDicts(cctx); #ifdef ZSTD_MULTITHREAD ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; @@ -160,7 +160,7 @@ static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) { if (cctx==NULL) return 0; /* support sizeof on NULL */ - return sizeof(*cctx) + cctx->workSpaceSize + return sizeof(*cctx) + cctx->workspaceSize + ZSTD_sizeof_localDict(cctx->localDict) + ZSTD_sizeof_mtctx(cctx); } @@ -1101,7 +1101,7 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) matchStateSize + ldmSpace + ldmSeqSpace; DEBUGLOG(5, "sizeof(ZSTD_CCtx) : %u", (U32)sizeof(ZSTD_CCtx)); - DEBUGLOG(5, "estimate workSpace : %u", (U32)neededSpace); + DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace); return sizeof(ZSTD_CCtx) + neededSpace; } } @@ -1444,8 +1444,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->seqStore.maxNbSeq, zc->seqStore.maxNbLit, zbuff, pledgedSrcSize) ) { DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> consider continue mode"); - zc->workSpaceOversizedDuration += (zc->workSpaceOversizedDuration > 0); /* if it was too large, it still is */ - if (zc->workSpaceOversizedDuration <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) { + zc->workspaceOversizedDuration += (zc->workspaceOversizedDuration > 0); /* if it was too large, it still is */ + if (zc->workspaceOversizedDuration <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) { DEBUGLOG(4, "continue mode confirmed (wLog1=%u, blockSize1=%zu)", zc->appliedParams.cParams.windowLog, zc->blockSize); if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) { @@ -1476,9 +1476,9 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const buffInSize = (zbuff==ZSTDb_buffered) ? windowSize + blockSize : 0; size_t const matchStateSize = ZSTD_sizeof_matchState(¶ms.cParams, /* forCCtx */ 1); size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize); - void* ptr; /* used to partition workSpace */ + void* ptr; /* used to partition workspace */ - /* Check if workSpace is large enough, alloc a new one if needed */ + /* Check if workspace is large enough, alloc a new one if needed */ { size_t const entropySpace = HUF_WORKSPACE_SIZE; size_t const blockStateSpace = 2 * sizeof(ZSTD_compressedBlockState_t); size_t const bufferSpace = buffInSize + buffOutSize; @@ -1489,35 +1489,35 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ldmSeqSpace + matchStateSize + tokenSpace + bufferSpace; - int const workSpaceTooSmall = zc->workSpaceSize < neededSpace; - int const workSpaceTooLarge = zc->workSpaceSize > ZSTD_WORKSPACETOOLARGE_FACTOR * neededSpace; - int const workSpaceWasteful = workSpaceTooLarge && (zc->workSpaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION); - zc->workSpaceOversizedDuration = workSpaceTooLarge ? zc->workSpaceOversizedDuration+1 : 0; + int const workspaceTooSmall = zc->workspaceSize < neededSpace; + int const workspaceTooLarge = zc->workspaceSize > ZSTD_WORKSPACETOOLARGE_FACTOR * neededSpace; + int const workspaceWasteful = workspaceTooLarge && (zc->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION); + zc->workspaceOversizedDuration = workspaceTooLarge ? zc->workspaceOversizedDuration+1 : 0; DEBUGLOG(4, "Need %zuKB workspace, including %zuKB for match state, and %zuKB for buffers", neededSpace>>10, matchStateSize>>10, bufferSpace>>10); DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize); - if (workSpaceTooSmall || workSpaceWasteful) { - DEBUGLOG(4, "Resize workSpaceSize from %zuKB to %zuKB", - zc->workSpaceSize >> 10, + if (workspaceTooSmall || workspaceWasteful) { + DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB", + zc->workspaceSize >> 10, neededSpace >> 10); RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize"); - zc->workSpaceSize = 0; - ZSTD_free(zc->workSpace, zc->customMem); - zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem); - RETURN_ERROR_IF(zc->workSpace == NULL, memory_allocation); - zc->workSpaceSize = neededSpace; - zc->workSpaceOversizedDuration = 0; + zc->workspaceSize = 0; + ZSTD_free(zc->workspace, zc->customMem); + zc->workspace = ZSTD_malloc(neededSpace, zc->customMem); + RETURN_ERROR_IF(zc->workspace == NULL, memory_allocation); + zc->workspaceSize = neededSpace; + zc->workspaceOversizedDuration = 0; /* Statically sized space. * entropyWorkspace never moves, * though prev/next block swap places */ - assert(((size_t)zc->workSpace & 3) == 0); /* ensure correct alignment */ - assert(zc->workSpaceSize >= 2 * sizeof(ZSTD_compressedBlockState_t)); - zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)zc->workSpace; + assert(((size_t)zc->workspace & 3) == 0); /* ensure correct alignment */ + assert(zc->workspaceSize >= 2 * sizeof(ZSTD_compressedBlockState_t)); + zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)zc->workspace; zc->blockState.nextCBlock = zc->blockState.prevCBlock + 1; ptr = zc->blockState.nextCBlock + 1; zc->entropyWorkspace = (U32*)ptr; diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 262a6309..52d6f878 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -231,9 +231,9 @@ struct ZSTD_CCtx_s { ZSTD_CCtx_params appliedParams; U32 dictID; - int workSpaceOversizedDuration; - void* workSpace; - size_t workSpaceSize; + int workspaceOversizedDuration; + void* workspace; + size_t workspaceSize; size_t blockSize; unsigned long long pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */ unsigned long long consumedSrcSize; From c25283cf00283dbaa45aca49bef2f5bda60650f6 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 15 Jul 2019 17:14:51 -0400 Subject: [PATCH 022/144] Disambiguate 'workspace' and 'entropyWorkspace' --- lib/compress/zstd_compress.c | 59 +++++++++++++++++--------- lib/compress/zstd_compress_literals.c | 15 ++++--- lib/compress/zstd_compress_literals.h | 2 +- lib/compress/zstd_compress_sequences.c | 6 +-- lib/compress/zstd_compress_sequences.h | 2 +- 5 files changed, 53 insertions(+), 31 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 61dbebac..c4a659c3 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1960,7 +1960,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, ZSTD_entropyCTables_t* nextEntropy, const ZSTD_CCtx_params* cctxParams, void* dst, size_t dstCapacity, - void* workspace, size_t wkspSize, + void* entropyWorkspace, size_t entropyWkspSize, const int bmi2) { const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; @@ -1993,7 +1993,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, ZSTD_disableLiteralsCompression(cctxParams), op, dstCapacity, literals, litSize, - workspace, wkspSize, + entropyWorkspace, entropyWkspSize, bmi2); FORWARD_IF_ERROR(cSize); assert(cSize <= dstCapacity); @@ -2029,7 +2029,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, ZSTD_seqToCodes(seqStorePtr); /* build CTable for Literal Lengths */ { unsigned max = MaxLL; - size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ + size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ DEBUGLOG(5, "Building LL table"); nextEntropy->fse.litlength_repeatMode = prevEntropy->fse.litlength_repeatMode; LLtype = ZSTD_selectEncodingType(&nextEntropy->fse.litlength_repeatMode, @@ -2039,10 +2039,14 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, ZSTD_defaultAllowed, strategy); assert(set_basic < set_compressed && set_rle < set_compressed); assert(!(LLtype < set_compressed && nextEntropy->fse.litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable(op, (size_t)(oend - op), CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, - count, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL, - prevEntropy->fse.litlengthCTable, sizeof(prevEntropy->fse.litlengthCTable), - workspace, wkspSize); + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, + count, max, llCodeTable, nbSeq, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + prevEntropy->fse.litlengthCTable, + sizeof(prevEntropy->fse.litlengthCTable), + entropyWorkspace, entropyWkspSize); FORWARD_IF_ERROR(countSize); if (LLtype == set_compressed) lastNCount = op; @@ -2051,7 +2055,8 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, } } /* build CTable for Offsets */ { unsigned max = MaxOff; - size_t const mostFrequent = HIST_countFast_wksp(count, &max, ofCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ + size_t const mostFrequent = HIST_countFast_wksp( + count, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; DEBUGLOG(5, "Building OF table"); @@ -2062,10 +2067,14 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, OF_defaultNorm, OF_defaultNormLog, defaultPolicy, strategy); assert(!(Offtype < set_compressed && nextEntropy->fse.offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable(op, (size_t)(oend - op), CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, - count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, - prevEntropy->fse.offcodeCTable, sizeof(prevEntropy->fse.offcodeCTable), - workspace, wkspSize); + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, + count, max, ofCodeTable, nbSeq, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + prevEntropy->fse.offcodeCTable, + sizeof(prevEntropy->fse.offcodeCTable), + entropyWorkspace, entropyWkspSize); FORWARD_IF_ERROR(countSize); if (Offtype == set_compressed) lastNCount = op; @@ -2074,7 +2083,8 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, } } /* build CTable for MatchLengths */ { unsigned max = MaxML; - size_t const mostFrequent = HIST_countFast_wksp(count, &max, mlCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ + size_t const mostFrequent = HIST_countFast_wksp( + count, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); nextEntropy->fse.matchlength_repeatMode = prevEntropy->fse.matchlength_repeatMode; MLtype = ZSTD_selectEncodingType(&nextEntropy->fse.matchlength_repeatMode, @@ -2083,10 +2093,14 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, ML_defaultNorm, ML_defaultNormLog, ZSTD_defaultAllowed, strategy); assert(!(MLtype < set_compressed && nextEntropy->fse.matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable(op, (size_t)(oend - op), CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, - count, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML, - prevEntropy->fse.matchlengthCTable, sizeof(prevEntropy->fse.matchlengthCTable), - workspace, wkspSize); + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, + count, max, mlCodeTable, nbSeq, + ML_defaultNorm, ML_defaultNormLog, MaxML, + prevEntropy->fse.matchlengthCTable, + sizeof(prevEntropy->fse.matchlengthCTable), + entropyWorkspace, entropyWkspSize); FORWARD_IF_ERROR(countSize); if (MLtype == set_compressed) lastNCount = op; @@ -2134,13 +2148,13 @@ ZSTD_compressSequences(seqStore_t* seqStorePtr, const ZSTD_CCtx_params* cctxParams, void* dst, size_t dstCapacity, size_t srcSize, - void* workspace, size_t wkspSize, + void* entropyWorkspace, size_t entropyWkspSize, int bmi2) { size_t const cSize = ZSTD_compressSequences_internal( seqStorePtr, prevEntropy, nextEntropy, cctxParams, dst, dstCapacity, - workspace, wkspSize, bmi2); + entropyWorkspace, entropyWkspSize, bmi2); if (cSize == 0) return 0; /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block. * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block. @@ -2380,6 +2394,7 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, ZSTD_CCtx_params } } + /*! ZSTD_compress_frameChunk() : * Compress a chunk of data into one or multiple blocks. * All blocks will be terminated, all input will be consumed. @@ -2848,7 +2863,8 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, ZSTDcrp_continue, zbuff) ); { size_t const dictID = ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, - params, dict, dictSize, dictContentType, dtlm, cctx->entropyWorkspace); + params, dict, dictSize, dictContentType, dtlm, + cctx->entropyWorkspace); FORWARD_IF_ERROR(dictID); assert(dictID <= UINT_MAX); cctx->dictID = (U32)dictID; @@ -3393,7 +3409,8 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss) * Assumption 2 : either dict, or cdict, is defined, not both */ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, const void* dict, size_t dictSize, const ZSTD_CDict* cdict, - const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize) + const ZSTD_CCtx_params* params, + unsigned long long pledgedSrcSize) { DEBUGLOG(4, "ZSTD_initCStream_internal"); FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); diff --git a/lib/compress/zstd_compress_literals.c b/lib/compress/zstd_compress_literals.c index eb3e5a44..6c133311 100644 --- a/lib/compress/zstd_compress_literals.c +++ b/lib/compress/zstd_compress_literals.c @@ -70,7 +70,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, ZSTD_strategy strategy, int disableLiteralCompression, void* dst, size_t dstCapacity, const void* src, size_t srcSize, - void* workspace, size_t wkspSize, + void* entropyWorkspace, size_t entropyWorkspaceSize, const int bmi2) { size_t const minGain = ZSTD_minGain(srcSize, strategy); @@ -99,10 +99,15 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, { HUF_repeat repeat = prevHuf->repeatMode; int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0; if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; - cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, - workspace, wkspSize, (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) - : HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, - workspace, wkspSize, (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2); + cLitSize = singleStream ? + HUF_compress1X_repeat( + ostart+lhSize, dstCapacity-lhSize, src, srcSize, + 255, 11, entropyWorkspace, entropyWorkspaceSize, + (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) : + HUF_compress4X_repeat( + ostart+lhSize, dstCapacity-lhSize, src, srcSize, + 255, 11, entropyWorkspace, entropyWorkspaceSize, + (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2); if (repeat != HUF_repeat_none) { /* reused the existing table */ hType = set_repeat; diff --git a/lib/compress/zstd_compress_literals.h b/lib/compress/zstd_compress_literals.h index 7adbecc0..97273d7c 100644 --- a/lib/compress/zstd_compress_literals.h +++ b/lib/compress/zstd_compress_literals.h @@ -23,7 +23,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, ZSTD_strategy strategy, int disableLiteralCompression, void* dst, size_t dstCapacity, const void* src, size_t srcSize, - void* workspace, size_t wkspSize, + void* entropyWorkspace, size_t entropyWorkspaceSize, const int bmi2); #endif /* ZSTD_COMPRESS_LITERALS_H */ diff --git a/lib/compress/zstd_compress_sequences.c b/lib/compress/zstd_compress_sequences.c index 3c3deae0..0ff7a268 100644 --- a/lib/compress/zstd_compress_sequences.c +++ b/lib/compress/zstd_compress_sequences.c @@ -222,7 +222,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity, const BYTE* codeTable, size_t nbSeq, const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, const FSE_CTable* prevCTable, size_t prevCTableSize, - void* workspace, size_t workspaceSize) + void* entropyWorkspace, size_t entropyWorkspaceSize) { BYTE* op = (BYTE*)dst; const BYTE* const oend = op + dstCapacity; @@ -238,7 +238,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity, memcpy(nextCTable, prevCTable, prevCTableSize); return 0; case set_basic: - FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize)); /* note : could be pre-calculated */ + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize)); /* note : could be pre-calculated */ return 0; case set_compressed: { S16 norm[MaxSeq + 1]; @@ -252,7 +252,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity, FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max)); { size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ FORWARD_IF_ERROR(NCountSize); - FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, workspace, workspaceSize)); + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, entropyWorkspace, entropyWorkspaceSize)); return NCountSize; } } diff --git a/lib/compress/zstd_compress_sequences.h b/lib/compress/zstd_compress_sequences.h index f5234d94..57e8e367 100644 --- a/lib/compress/zstd_compress_sequences.h +++ b/lib/compress/zstd_compress_sequences.h @@ -35,7 +35,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity, const BYTE* codeTable, size_t nbSeq, const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, const FSE_CTable* prevCTable, size_t prevCTableSize, - void* workspace, size_t workspaceSize); + void* entropyWorkspace, size_t entropyWorkspaceSize); size_t ZSTD_encodeSequences( void* dst, size_t dstCapacity, From 786f2266bb77076054498241755b4ef6f42ab382 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 15 Jul 2019 15:08:49 -0400 Subject: [PATCH 023/144] TMP --- lib/compress/zstd_compress.c | 402 ++++++++++++++++++-------- lib/compress/zstd_compress_internal.h | 78 ++++- 2 files changed, 352 insertions(+), 128 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index c4a659c3..b0e57e4d 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -38,6 +38,159 @@ size_t ZSTD_compressBound(size_t srcSize) { } +/*-************************************* +* Workspace memory management +***************************************/ +#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 /* define "workspace is too large" as this number of times larger than needed */ +#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 /* when workspace is continuously too large + * during at least this number of times, + * context's memory usage is considered wasteful, + * because it's sized to handle a worst case scenario which rarely happens. + * In which case, resize it down to free some memory */ + +static size_t ZSTD_workspace_round_size(size_t size) { + // return size + sizeof(void*) - 1 - ((size - 1) & (sizeof(void*) - 1)); + return size + 3 - ((size - 1) & 3); +} + +/** + * Align must be a power of 2. + */ +static size_t ZSTD_workspace_align(size_t size, size_t align) { + return size + align - 1 - ((size - 1) & (align - 1)); + // return size + 3 - ((size - 1) & 3); +} + +static void* ZSTD_workspace_reserve(ZSTD_CCtx_workspace* ws, size_t bytes) { + /* TODO(felixh): alignment */ + void* alloc = ws->allocEnd; + void* newAllocEnd = (BYTE *)ws->allocEnd + bytes; + DEBUGLOG(3, "wksp: reserving %zd bytes, %zd bytes remaining", bytes, (BYTE *)ws->workspaceEnd - (BYTE *)newAllocEnd); + assert(newAllocEnd <= ws->workspaceEnd); + if (newAllocEnd > ws->workspaceEnd) { + ws->allocFailed = 1; + return NULL; + } + ws->allocEnd = newAllocEnd; + ws->staticAllocDone = 1; + return alloc; +} + +/** + * Aligned on sizeof(void*). + */ +static void* ZSTD_workspace_reserve_object(ZSTD_CCtx_workspace* ws, size_t bytes) { + size_t roundedBytes = ZSTD_workspace_align(bytes, sizeof(void*)); + void* start = ws->objectEnd; + void* end = (BYTE*)start + roundedBytes; + assert(ws->allocEnd == ws->objectEnd); + DEBUGLOG(3, "wksp: reserving %zd bytes object (rounded to %zd), %zd bytes remaining", bytes, roundedBytes, (BYTE *)ws->workspaceEnd - (BYTE *)end); + assert((bytes & (sizeof(void*)-1)) == 0); // TODO ??? + if (ws->staticAllocDone || end > ws->workspaceEnd) { + DEBUGLOG(3, "wksp: object alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->objectEnd = end; + ws->allocEnd = end; + return start; +} + +/** + * Aligned on sizeof(unsigned). These buffers have the special property that + * their values remain constrained, allowing us to re-use them without + * memset()-ing them. + */ +static void* ZSTD_workspace_reserve_table(ZSTD_CCtx_workspace* ws, size_t bytes) { + assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? + ws->staticAllocDone = 1; + return ZSTD_workspace_reserve(ws, ZSTD_workspace_align(bytes, sizeof(unsigned))); +} + +/** + * Aligned on sizeof(unsigned). + */ +static void* ZSTD_workspace_reserve_aligned(ZSTD_CCtx_workspace* ws, size_t bytes) { + assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? + ws->staticAllocDone = 1; + return ZSTD_workspace_reserve(ws, ZSTD_workspace_align(bytes, sizeof(unsigned))); +} + +/** + * Unaligned. + */ +static void* ZSTD_workspace_reserve_buffer(ZSTD_CCtx_workspace* ws, size_t bytes) { + ws->staticAllocDone = 1; + return ZSTD_workspace_reserve(ws, bytes); +} + +// TODO +static int ZSTD_workspace_bump_oversized_duration(ZSTD_CCtx_workspace* ws) { + // if (((BYTE*)ws->allocEnd - (BYTE*)ws->workspace) * ZSTD_WORKSPACETOOLARGE_FACTOR < (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace) { + // ws->workspaceOversizedDuration++; + // } else { + // ws->workspaceOversizedDuration = 0; + // } + // return ws->workspaceOversizedDuration; + return 0; +} + +static void ZSTD_workspace_clear(ZSTD_CCtx_workspace* ws) { + DEBUGLOG(3, "wksp: clearing!"); + ZSTD_workspace_bump_oversized_duration(ws); + ws->allocEnd = ws->objectEnd; + ws->allocFailed = 0; + + // ws->table = NULL; + // ws->tableEnd = NULL; + + // ws->bufferBegin = ws->workspaceEnd; +} + +static void ZSTD_workspace_init(ZSTD_CCtx_workspace* ws, void* start, size_t size) { + DEBUGLOG(3, "wksp: init'ing with %zd bytes", size); + assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ + ws->workspace = start; + ws->workspaceEnd = (BYTE*)start + size; + ws->objectEnd = ws->workspace; + ZSTD_workspace_clear(ws); + ws->workspaceOversizedDuration = 0; + ws->staticAllocDone = 0; +} + +static size_t ZSTD_workspace_create(ZSTD_CCtx_workspace* ws, size_t size, ZSTD_customMem customMem) { + void* workspace = ZSTD_malloc(size, customMem); + DEBUGLOG(3, "wksp: creating with %zd bytes", size); + RETURN_ERROR_IF(workspace == NULL, memory_allocation); + ZSTD_workspace_init(ws, workspace, size); + return 0; +} + +static void ZSTD_workspace_free(ZSTD_CCtx_workspace* ws, ZSTD_customMem customMem) { + DEBUGLOG(3, "wksp: freeing"); + ZSTD_free(ws->workspace, customMem); + ws->workspace = NULL; + ws->workspaceEnd = NULL; + ZSTD_workspace_clear(ws); +} + +static int ZSTD_workspace_check_available(ZSTD_CCtx_workspace* ws, size_t minFree) { + return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->allocEnd) >= minFree; +} + +static int ZSTD_workspace_check_wasteful(ZSTD_CCtx_workspace* ws, size_t minFree) { + return ZSTD_workspace_check_available(ws, minFree * ZSTD_WORKSPACETOOLARGE_FACTOR) && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} + +static size_t ZSTD_workspace_sizeof(const ZSTD_CCtx_workspace* ws) { + return (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace; +} + +static int ZSTD_workspace_reserve_failed(const ZSTD_CCtx_workspace* ws) { + return ws->allocFailed; +} + + /*-************************************* * Context memory management ***************************************/ @@ -45,8 +198,8 @@ struct ZSTD_CDict_s { void* dictBuffer; const void* dictContent; size_t dictContentSize; - void* workspace; - size_t workspaceSize; + U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */ + ZSTD_CCtx_workspace workspace; ZSTD_matchState_t matchState; ZSTD_compressedBlockState_t cBlockState; ZSTD_customMem customMem; @@ -87,20 +240,16 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize) ZSTD_CCtx* const cctx = (ZSTD_CCtx*) workspace; if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */ if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ - memset(workspace, 0, workspaceSize); /* may be a bit generous, could memset be smaller ? */ + memset(workspace, 0, workspaceSize); /* may be a bit generous, could memset be smaller ? */ /* TODO(felixh): investigate */ cctx->staticSize = workspaceSize; - cctx->workspace = (void*)(cctx+1); - cctx->workspaceSize = workspaceSize - sizeof(ZSTD_CCtx); + ZSTD_workspace_init(&cctx->workspace, (void*)(cctx+1), workspaceSize - sizeof(ZSTD_CCtx)); /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ - if (cctx->workspaceSize < HUF_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t)) return NULL; - assert(((size_t)cctx->workspace & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ - cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)cctx->workspace; - cctx->blockState.nextCBlock = cctx->blockState.prevCBlock + 1; - { - void* const ptr = cctx->blockState.nextCBlock + 1; - cctx->entropyWorkspace = (U32*)ptr; - } + if (!ZSTD_workspace_check_available(&cctx->workspace, HUF_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; + cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_workspace_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_workspace_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->entropyWorkspace = (U32*)ZSTD_workspace_reserve_object( + &cctx->workspace, HUF_WORKSPACE_SIZE); cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); return cctx; } @@ -128,7 +277,7 @@ static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) { assert(cctx != NULL); assert(cctx->staticSize == 0); - ZSTD_free(cctx->workspace, cctx->customMem); cctx->workspace = NULL; + ZSTD_workspace_free(&cctx->workspace, cctx->customMem); ZSTD_clearAllDicts(cctx); #ifdef ZSTD_MULTITHREAD ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; @@ -160,7 +309,7 @@ static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) { if (cctx==NULL) return 0; /* support sizeof on NULL */ - return sizeof(*cctx) + cctx->workspaceSize + return sizeof(*cctx) + ZSTD_workspace_sizeof(&cctx->workspace) + ZSTD_sizeof_localDict(cctx->localDict) + ZSTD_sizeof_mtctx(cctx); } @@ -1355,9 +1504,9 @@ typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset } ZSTD_compResetPolicy_e; typedef enum { ZSTD_resetTarget_CDict, ZSTD_resetTarget_CCtx } ZSTD_resetTarget_e; -static void* +static size_t ZSTD_reset_matchState(ZSTD_matchState_t* ms, - void* ptr, + ZSTD_CCtx_workspace* ws, const ZSTD_compressionParameters* cParams, ZSTD_compResetPolicy_e const crp, ZSTD_resetTarget_e const forWho) { @@ -1365,9 +1514,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, size_t const hSize = ((size_t)1) << cParams->hashLog; U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; size_t const h3Size = ((size_t)1) << hashLog3; - size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); - assert(((size_t)ptr & 3) == 0); ms->hashLog3 = hashLog3; memset(&ms->window, 0, sizeof(ms->window)); @@ -1376,33 +1523,40 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, ms->window.nextSrc = ms->window.base + 1; /* see issue #1241 */ ZSTD_invalidateMatchState(ms); + assert(!ZSTD_workspace_reserve_failed(ws)); /* check that allocation hasn't already failed */ + + DEBUGLOG(5, "reserving table space"); + /* table Space */ + ms->hashTable = (U32*)ZSTD_workspace_reserve_table(ws, hSize * sizeof(U32)); + ms->chainTable = (U32*)ZSTD_workspace_reserve_table(ws, chainSize * sizeof(U32)); + ms->hashTable3 = (U32*)ZSTD_workspace_reserve_table(ws, h3Size * sizeof(U32)); + RETURN_ERROR_IF(ZSTD_workspace_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); + DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_noMemset); + if (crp!=ZSTDcrp_noMemset) { + /* reset tables only */ + memset(ms->hashTable, 0, hSize * sizeof(U32)); + memset(ms->chainTable, 0, chainSize * sizeof(U32)); + memset(ms->hashTable3, 0, h3Size * sizeof(U32)); + } + /* opt parser space */ if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) { DEBUGLOG(4, "reserving optimal parser space"); - ms->opt.litFreq = (unsigned*)ptr; - ms->opt.litLengthFreq = ms->opt.litFreq + (1<opt.matchLengthFreq = ms->opt.litLengthFreq + (MaxLL+1); - ms->opt.offCodeFreq = ms->opt.matchLengthFreq + (MaxML+1); - ptr = ms->opt.offCodeFreq + (MaxOff+1); - ms->opt.matchTable = (ZSTD_match_t*)ptr; - ptr = ms->opt.matchTable + ZSTD_OPT_NUM+1; - ms->opt.priceTable = (ZSTD_optimal_t*)ptr; - ptr = ms->opt.priceTable + ZSTD_OPT_NUM+1; + ms->opt.litFreq = (unsigned*)ZSTD_workspace_reserve_aligned(ws, (1<opt.litLengthFreq = (unsigned*)ZSTD_workspace_reserve_aligned(ws, (MaxLL+1) * sizeof(unsigned)); + ms->opt.matchLengthFreq = (unsigned*)ZSTD_workspace_reserve_aligned(ws, (MaxML+1) * sizeof(unsigned)); + ms->opt.offCodeFreq = (unsigned*)ZSTD_workspace_reserve_aligned(ws, (MaxOff+1) * sizeof(unsigned)); + ms->opt.matchTable = (ZSTD_match_t*)ZSTD_workspace_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t)); + ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_workspace_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t)); } - /* table Space */ - DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_noMemset); - assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ - if (crp!=ZSTDcrp_noMemset) memset(ptr, 0, tableSpace); /* reset tables only */ - ms->hashTable = (U32*)(ptr); - ms->chainTable = ms->hashTable + hSize; - ms->hashTable3 = ms->chainTable + chainSize; - ptr = ms->hashTable3 + h3Size; - ms->cParams = *cParams; - assert(((size_t)ptr & 3) == 0); - return ptr; + RETURN_ERROR_IF(ZSTD_workspace_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); + + return 0; } /* ZSTD_indexTooCloseToMax() : @@ -1418,13 +1572,6 @@ static int ZSTD_indexTooCloseToMax(ZSTD_window_t w) return (size_t)(w.nextSrc - w.base) > (ZSTD_CURRENT_MAX - ZSTD_INDEXOVERFLOW_MARGIN); } -#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 /* define "workspace is too large" as this number of times larger than needed */ -#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 /* when workspace is continuously too large - * during at least this number of times, - * context's memory usage is considered wasteful, - * because it's sized to handle a worst case scenario which rarely happens. - * In which case, resize it down to free some memory */ - /*! ZSTD_resetCCtx_internal() : note : `params` are assumed fully validated at this stage */ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, @@ -1444,16 +1591,16 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->seqStore.maxNbSeq, zc->seqStore.maxNbLit, zbuff, pledgedSrcSize) ) { DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> consider continue mode"); - zc->workspaceOversizedDuration += (zc->workspaceOversizedDuration > 0); /* if it was too large, it still is */ - if (zc->workspaceOversizedDuration <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) { + if (ZSTD_workspace_bump_oversized_duration(&zc->workspace) <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) { DEBUGLOG(4, "continue mode confirmed (wLog1=%u, blockSize1=%zu)", zc->appliedParams.cParams.windowLog, zc->blockSize); if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) { /* prefer a reset, faster than a rescale */ - ZSTD_reset_matchState(&zc->blockState.matchState, - zc->entropyWorkspace + HUF_WORKSPACE_SIZE_U32, - ¶ms.cParams, - crp, ZSTD_resetTarget_CCtx); + FORWARD_IF_ERROR(ZSTD_reset_matchState( + &zc->blockState.matchState, + &zc->workspace, + ¶ms.cParams, + crp, ZSTD_resetTarget_CCtx)); } return ZSTD_continueCCtx(zc, ¶ms, pledgedSrcSize); } } } @@ -1476,7 +1623,6 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const buffInSize = (zbuff==ZSTDb_buffered) ? windowSize + blockSize : 0; size_t const matchStateSize = ZSTD_sizeof_matchState(¶ms.cParams, /* forCCtx */ 1); size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize); - void* ptr; /* used to partition workspace */ /* Check if workspace is large enough, alloc a new one if needed */ { size_t const entropySpace = HUF_WORKSPACE_SIZE; @@ -1485,14 +1631,17 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const ldmSpace = ZSTD_ldm_getTableSize(params.ldmParams); size_t const ldmSeqSpace = maxNbLdmSeq * sizeof(rawSeq); - size_t const neededSpace = entropySpace + blockStateSpace + ldmSpace + - ldmSeqSpace + matchStateSize + tokenSpace + - bufferSpace; + size_t const neededSpace = + entropySpace + + blockStateSpace + + ldmSpace + + ldmSeqSpace + + matchStateSize + + tokenSpace + + bufferSpace; - int const workspaceTooSmall = zc->workspaceSize < neededSpace; - int const workspaceTooLarge = zc->workspaceSize > ZSTD_WORKSPACETOOLARGE_FACTOR * neededSpace; - int const workspaceWasteful = workspaceTooLarge && (zc->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION); - zc->workspaceOversizedDuration = workspaceTooLarge ? zc->workspaceOversizedDuration+1 : 0; + int const workspaceTooSmall = !ZSTD_workspace_check_available(&zc->workspace, neededSpace); + int const workspaceWasteful = ZSTD_workspace_check_wasteful(&zc->workspace, neededSpace); DEBUGLOG(4, "Need %zuKB workspace, including %zuKB for match state, and %zuKB for buffers", neededSpace>>10, matchStateSize>>10, bufferSpace>>10); @@ -1500,29 +1649,30 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, if (workspaceTooSmall || workspaceWasteful) { DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB", - zc->workspaceSize >> 10, + ZSTD_workspace_sizeof(&zc->workspace) >> 10, neededSpace >> 10); RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize"); - zc->workspaceSize = 0; - ZSTD_free(zc->workspace, zc->customMem); - zc->workspace = ZSTD_malloc(neededSpace, zc->customMem); - RETURN_ERROR_IF(zc->workspace == NULL, memory_allocation); - zc->workspaceSize = neededSpace; - zc->workspaceOversizedDuration = 0; + ZSTD_workspace_free(&zc->workspace, zc->customMem); + FORWARD_IF_ERROR(ZSTD_workspace_create(&zc->workspace, neededSpace, zc->customMem)); + DEBUGLOG(5, "reserving object space"); /* Statically sized space. * entropyWorkspace never moves, * though prev/next block swap places */ - assert(((size_t)zc->workspace & 3) == 0); /* ensure correct alignment */ - assert(zc->workspaceSize >= 2 * sizeof(ZSTD_compressedBlockState_t)); - zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)zc->workspace; - zc->blockState.nextCBlock = zc->blockState.prevCBlock + 1; - ptr = zc->blockState.nextCBlock + 1; - zc->entropyWorkspace = (U32*)ptr; + /* assert(((size_t)zc->workspace.workspace & 3) == 0); */ /* ensure correct alignment */ /* TODO(felixh): check elsewhere */ + assert(ZSTD_workspace_check_available(&zc->workspace, 2 * sizeof(ZSTD_compressedBlockState_t))); + zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_workspace_reserve_object(&zc->workspace, sizeof(ZSTD_compressedBlockState_t)); + RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock"); + zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_workspace_reserve_object(&zc->workspace, sizeof(ZSTD_compressedBlockState_t)); + RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock"); + zc->entropyWorkspace = (U32*) ZSTD_workspace_reserve_object(&zc->workspace, HUF_WORKSPACE_SIZE); + RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate entropyWorkspace"); } } + ZSTD_workspace_clear(&zc->workspace); + /* init params */ zc->appliedParams = params; zc->blockState.matchState.cParams = params.cParams; @@ -1541,58 +1691,55 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock); - ptr = ZSTD_reset_matchState(&zc->blockState.matchState, - zc->entropyWorkspace + HUF_WORKSPACE_SIZE_U32, - ¶ms.cParams, - crp, ZSTD_resetTarget_CCtx); + FORWARD_IF_ERROR(ZSTD_reset_matchState( + &zc->blockState.matchState, + &zc->workspace, + ¶ms.cParams, + crp, ZSTD_resetTarget_CCtx)); + + DEBUGLOG(3, "Done allocating match state"); /* ldm hash table */ /* initialize bucketOffsets table later for pointer alignment */ if (params.ldmParams.enableLdm) { size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog; - memset(ptr, 0, ldmHSize * sizeof(ldmEntry_t)); - assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ - zc->ldmState.hashTable = (ldmEntry_t*)ptr; - ptr = zc->ldmState.hashTable + ldmHSize; - zc->ldmSequences = (rawSeq*)ptr; - ptr = zc->ldmSequences + maxNbLdmSeq; + zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_workspace_reserve_aligned(&zc->workspace, ldmHSize * sizeof(ldmEntry_t)); + memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); + zc->ldmSequences = (rawSeq*)ZSTD_workspace_reserve_aligned(&zc->workspace, maxNbLdmSeq * sizeof(rawSeq)); zc->maxNbLdmSequences = maxNbLdmSeq; memset(&zc->ldmState.window, 0, sizeof(zc->ldmState.window)); } - assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ - - /* sequences storage */ - zc->seqStore.maxNbSeq = maxNbSeq; - zc->seqStore.sequencesStart = (seqDef*)ptr; - ptr = zc->seqStore.sequencesStart + maxNbSeq; - zc->seqStore.llCode = (BYTE*) ptr; - zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq; - zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq; - zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq; - /* ZSTD_wildcopy() is used to copy into the literals buffer, - * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. - */ - zc->seqStore.maxNbLit = blockSize; - ptr = zc->seqStore.litStart + blockSize + WILDCOPY_OVERLENGTH; /* ldm bucketOffsets table */ if (params.ldmParams.enableLdm) { size_t const ldmBucketSize = ((size_t)1) << (params.ldmParams.hashLog - params.ldmParams.bucketSizeLog); - memset(ptr, 0, ldmBucketSize); - zc->ldmState.bucketOffsets = (BYTE*)ptr; - ptr = zc->ldmState.bucketOffsets + ldmBucketSize; + zc->ldmState.bucketOffsets = (BYTE*)ZSTD_workspace_reserve_aligned(&zc->workspace, ldmBucketSize); + memset(zc->ldmState.bucketOffsets, 0, ldmBucketSize); ZSTD_window_clear(&zc->ldmState.window); } ZSTD_referenceExternalSequences(zc, NULL, 0); + /* sequences storage */ + zc->seqStore.maxNbSeq = maxNbSeq; + zc->seqStore.sequencesStart = (seqDef*)ZSTD_workspace_reserve_aligned(&zc->workspace, maxNbSeq * sizeof(seqDef)); + zc->seqStore.llCode = (BYTE*)ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); + zc->seqStore.mlCode = (BYTE*)ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); + zc->seqStore.ofCode = (BYTE*)ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); + + /* ZSTD_wildcopy() is used to copy into the literals buffer, + * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. + */ + zc->seqStore.litStart = (BYTE*)ZSTD_workspace_reserve_buffer(&zc->workspace, blockSize + WILDCOPY_OVERLENGTH); + zc->seqStore.maxNbLit = blockSize; + /* buffers */ zc->inBuffSize = buffInSize; - zc->inBuff = (char*)ptr; + zc->inBuff = (char*)ZSTD_workspace_reserve_buffer(&zc->workspace, buffInSize); zc->outBuffSize = buffOutSize; - zc->outBuff = zc->inBuff + buffInSize; + zc->outBuff = (char*)ZSTD_workspace_reserve_buffer(&zc->workspace, buffOutSize); return 0; } @@ -3092,7 +3239,7 @@ size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) { if (cdict==NULL) return 0; /* support sizeof on NULL */ DEBUGLOG(5, "sizeof(*cdict) : %u", (unsigned)sizeof(*cdict)); - return cdict->workspaceSize + (cdict->dictBuffer ? cdict->dictContentSize : 0) + sizeof(*cdict); + return ZSTD_workspace_sizeof(&cdict->workspace) + (cdict->dictBuffer ? cdict->dictContentSize : 0) + sizeof(*cdict); } static size_t ZSTD_initCDict_internal( @@ -3117,15 +3264,16 @@ static size_t ZSTD_initCDict_internal( } cdict->dictContentSize = dictSize; + cdict->entropyWorkspace = (U32*)ZSTD_workspace_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE); + + /* Reset the state to no dictionary */ ZSTD_reset_compressedBlockState(&cdict->cBlockState); - { void* const end = ZSTD_reset_matchState(&cdict->matchState, - (U32*)cdict->workspace + HUF_WORKSPACE_SIZE_U32, - &cParams, - ZSTDcrp_continue, ZSTD_resetTarget_CDict); - assert(end == (char*)cdict->workspace + cdict->workspaceSize); - (void)end; - } + FORWARD_IF_ERROR(ZSTD_reset_matchState( + &cdict->matchState, + &cdict->workspace, + &cParams, + ZSTDcrp_continue, ZSTD_resetTarget_CDict)); /* (Maybe) load the dictionary * Skips loading the dictionary if it is <= 8 bytes. */ @@ -3137,7 +3285,7 @@ static size_t ZSTD_initCDict_internal( { size_t const dictID = ZSTD_compress_insertDictionary( &cdict->cBlockState, &cdict->matchState, ¶ms, cdict->dictContent, cdict->dictContentSize, - dictContentType, ZSTD_dtlm_full, cdict->workspace); + dictContentType, ZSTD_dtlm_full, cdict->entropyWorkspace); FORWARD_IF_ERROR(dictID); assert(dictID <= (size_t)(U32)-1); cdict->dictID = (U32)dictID; @@ -3165,8 +3313,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, return NULL; } cdict->customMem = customMem; - cdict->workspace = workspace; - cdict->workspaceSize = workspaceSize; + ZSTD_workspace_init(&cdict->workspace, workspace, workspaceSize); if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dictBuffer, dictSize, dictLoadMethod, dictContentType, @@ -3199,7 +3346,7 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) { if (cdict==NULL) return 0; /* support free on NULL */ { ZSTD_customMem const cMem = cdict->customMem; - ZSTD_free(cdict->workspace, cMem); + ZSTD_workspace_free(&cdict->workspace, cMem); ZSTD_free(cdict->dictBuffer, cMem); ZSTD_free(cdict, cMem); return 0; @@ -3227,24 +3374,29 @@ const ZSTD_CDict* ZSTD_initStaticCDict( ZSTD_compressionParameters cParams) { size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0); - size_t const neededSize = sizeof(ZSTD_CDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize) + size_t const neededSize = sizeof(ZSTD_CDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_workspace_align(dictSize, sizeof(void*))) + HUF_WORKSPACE_SIZE + matchStateSize; - ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace; - void* ptr; + ZSTD_CDict* cdict; + if ((size_t)workspace & 7) return NULL; /* 8-aligned */ + + { + ZSTD_CCtx_workspace ws; + ZSTD_workspace_init(&ws, workspace, workspaceSize); + cdict = (ZSTD_CDict*)ZSTD_workspace_reserve_object(&ws, sizeof(ZSTD_CDict)); + if (cdict == NULL) return NULL; + cdict->workspace = ws; + } + DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u", (unsigned)workspaceSize, (unsigned)neededSize, (unsigned)(workspaceSize < neededSize)); if (workspaceSize < neededSize) return NULL; if (dictLoadMethod == ZSTD_dlm_byCopy) { - memcpy(cdict+1, dict, dictSize); - dict = cdict+1; - ptr = (char*)workspace + sizeof(ZSTD_CDict) + dictSize; - } else { - ptr = cdict+1; + void *dictCopy = ZSTD_workspace_reserve_object(&cdict->workspace, ZSTD_workspace_align(dictSize, sizeof(void*))); + memcpy(dictCopy, dict, dictSize); + dict = dictCopy; } - cdict->workspace = ptr; - cdict->workspaceSize = HUF_WORKSPACE_SIZE + matchStateSize; if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dict, dictSize, diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 52d6f878..287be309 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -223,6 +223,80 @@ struct ZSTD_CCtx_params_s { ZSTD_customMem customMem; }; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */ +/** + * Zstd fits all its internal datastructures into a single continuous buffer, + * so that it only needs to perform a single OS allocation (or so that a buffer + * can be provided to it and it can perform no allocations at all). This buffer + * is called the workspace. + * + * Several optimizations complicate that process of allocating memory ranges + * from this workspace for each datastructure: + * + * - These different internal datastructures have different setup requirements. + * Some (e.g., the window buffer) don't care, and are happy to accept + * uninitialized memory. Others (e.g., the matchstate tables) can accept + * memory filled with unknown but bounded values (i.e., a memory area whose + * values are known to be constrained between 0 and some upper bound). If + * that constraint isn't known to be satisfied, the area has to be cleared. + * + * - We would like to reuse the objects in the workspace for multiple + * compressions without having to perform any expensive reallocation or + * reinitialization work. + * + * - We would like to be able to efficiently reuse the workspace across + * multiple compressions **even when the compression parameters change** and + * we need to resize some of the objects (where possible). + * + * Workspace Layout: + * + * In order to accomplish this, the various objects that live in the workspace + * are divided into the following categories: + * + * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict, + * so that literally everything fits in a single buffer. + * + * - Fixed size objects: these are fixed-size, fixed-count objects that are + * nonetheless "dynamically" allocated in the workspace so that we can + * control how they're initialized separately from the broader ZSTD_CCtx. + * Examples: + * - Entropy Workspace + * - 2 x ZSTD_compressedBlockState_t + * + * - Tables: these are any of several different datastructures (hash tables, + * chain tables, binary trees) that all respect a common format: they are + * uint32_t arrays, all of whose values are between 0 and (nextSrc - base). + * Their sizes depend on the cparams. + * + * - Uninitialized memory: these buffers are used for various purposes that + * don't require any initialization before they're used. This means they can + * be moved around at no cost for a new compression. + * - I/O Buffers + * + * [workspace, workspace + workspaceSize) + * [] + */ +typedef struct { + void* workspace; + void* workspaceEnd; + + void* objectEnd; + + // // void* tableZoneStart; + // void* tableAllocStart; + // void* tableAllocEnd; + // // void* tableZoneEnd; + + // void* seqEnd; + + // void* bufferBegin; + + void* allocEnd; + int allocFailed; + + int workspaceOversizedDuration; + int staticAllocDone; +} ZSTD_CCtx_workspace; + struct ZSTD_CCtx_s { ZSTD_compressionStage_e stage; int cParamsChanged; /* == 1 if cParams(except wlog) or compression level are changed in requestedParams. Triggers transmission of new params to ZSTDMT (if available) then reset to 0. */ @@ -231,9 +305,7 @@ struct ZSTD_CCtx_s { ZSTD_CCtx_params appliedParams; U32 dictID; - int workspaceOversizedDuration; - void* workspace; - size_t workspaceSize; + ZSTD_CCtx_workspace workspace; /* manages buffer for dynamic allocations */ size_t blockSize; unsigned long long pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */ unsigned long long consumedSrcSize; From 6177354b362e56afb22220b67a25f32ddf6a9d81 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 12 Aug 2019 14:22:54 -0400 Subject: [PATCH 024/144] Begin Introducing Phases --- lib/compress/zstd_compress.c | 129 +++++++++++++------------- lib/compress/zstd_compress_internal.h | 8 +- 2 files changed, 73 insertions(+), 64 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index b0e57e4d..8d7d4439 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -48,11 +48,6 @@ size_t ZSTD_compressBound(size_t srcSize) { * because it's sized to handle a worst case scenario which rarely happens. * In which case, resize it down to free some memory */ -static size_t ZSTD_workspace_round_size(size_t size) { - // return size + sizeof(void*) - 1 - ((size - 1) & (sizeof(void*) - 1)); - return size + 3 - ((size - 1) & 3); -} - /** * Align must be a power of 2. */ @@ -61,18 +56,24 @@ static size_t ZSTD_workspace_align(size_t size, size_t align) { // return size + 3 - ((size - 1) & 3); } -static void* ZSTD_workspace_reserve(ZSTD_CCtx_workspace* ws, size_t bytes) { +static void* ZSTD_workspace_reserve(ZSTD_CCtx_workspace* ws, size_t bytes, ZSTD_workspace_alloc_phase_e phase) { /* TODO(felixh): alignment */ void* alloc = ws->allocEnd; void* newAllocEnd = (BYTE *)ws->allocEnd + bytes; DEBUGLOG(3, "wksp: reserving %zd bytes, %zd bytes remaining", bytes, (BYTE *)ws->workspaceEnd - (BYTE *)newAllocEnd); + assert(phase >= ws->phase); + if (phase > ws->phase) { + if (ws->phase <= ZSTD_workspace_alloc_buffers) { + + } + ws->phase = phase; + } assert(newAllocEnd <= ws->workspaceEnd); if (newAllocEnd > ws->workspaceEnd) { ws->allocFailed = 1; return NULL; } ws->allocEnd = newAllocEnd; - ws->staticAllocDone = 1; return alloc; } @@ -86,7 +87,7 @@ static void* ZSTD_workspace_reserve_object(ZSTD_CCtx_workspace* ws, size_t bytes assert(ws->allocEnd == ws->objectEnd); DEBUGLOG(3, "wksp: reserving %zd bytes object (rounded to %zd), %zd bytes remaining", bytes, roundedBytes, (BYTE *)ws->workspaceEnd - (BYTE *)end); assert((bytes & (sizeof(void*)-1)) == 0); // TODO ??? - if (ws->staticAllocDone || end > ws->workspaceEnd) { + if (ws->phase != ZSTD_workspace_alloc_objects || end > ws->workspaceEnd) { DEBUGLOG(3, "wksp: object alloc failed!"); ws->allocFailed = 1; return NULL; @@ -103,8 +104,7 @@ static void* ZSTD_workspace_reserve_object(ZSTD_CCtx_workspace* ws, size_t bytes */ static void* ZSTD_workspace_reserve_table(ZSTD_CCtx_workspace* ws, size_t bytes) { assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? - ws->staticAllocDone = 1; - return ZSTD_workspace_reserve(ws, ZSTD_workspace_align(bytes, sizeof(unsigned))); + return ZSTD_workspace_reserve(ws, ZSTD_workspace_align(bytes, sizeof(unsigned)), ZSTD_workspace_alloc_aligned); } /** @@ -112,20 +112,19 @@ static void* ZSTD_workspace_reserve_table(ZSTD_CCtx_workspace* ws, size_t bytes) */ static void* ZSTD_workspace_reserve_aligned(ZSTD_CCtx_workspace* ws, size_t bytes) { assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? - ws->staticAllocDone = 1; - return ZSTD_workspace_reserve(ws, ZSTD_workspace_align(bytes, sizeof(unsigned))); + return ZSTD_workspace_reserve(ws, ZSTD_workspace_align(bytes, sizeof(unsigned)), ZSTD_workspace_alloc_aligned); } /** * Unaligned. */ -static void* ZSTD_workspace_reserve_buffer(ZSTD_CCtx_workspace* ws, size_t bytes) { - ws->staticAllocDone = 1; - return ZSTD_workspace_reserve(ws, bytes); +static BYTE* ZSTD_workspace_reserve_buffer(ZSTD_CCtx_workspace* ws, size_t bytes) { + return (BYTE*)ZSTD_workspace_reserve(ws, bytes, ZSTD_workspace_alloc_buffers); } // TODO static int ZSTD_workspace_bump_oversized_duration(ZSTD_CCtx_workspace* ws) { + (void)ws; // if (((BYTE*)ws->allocEnd - (BYTE*)ws->workspace) * ZSTD_WORKSPACETOOLARGE_FACTOR < (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace) { // ws->workspaceOversizedDuration++; // } else { @@ -140,6 +139,9 @@ static void ZSTD_workspace_clear(ZSTD_CCtx_workspace* ws) { ZSTD_workspace_bump_oversized_duration(ws); ws->allocEnd = ws->objectEnd; ws->allocFailed = 0; + if (ws->phase > ZSTD_workspace_alloc_buffers) { + ws->phase = ZSTD_workspace_alloc_buffers; + } // ws->table = NULL; // ws->tableEnd = NULL; @@ -153,9 +155,9 @@ static void ZSTD_workspace_init(ZSTD_CCtx_workspace* ws, void* start, size_t siz ws->workspace = start; ws->workspaceEnd = (BYTE*)start + size; ws->objectEnd = ws->workspace; + ws->phase = ZSTD_workspace_alloc_objects; ZSTD_workspace_clear(ws); ws->workspaceOversizedDuration = 0; - ws->staticAllocDone = 0; } static size_t ZSTD_workspace_create(ZSTD_CCtx_workspace* ws, size_t size, ZSTD_customMem customMem) { @@ -1500,7 +1502,7 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params, return 0; } -typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset } ZSTD_compResetPolicy_e; +typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_noRealloc } ZSTD_compResetPolicy_e; typedef enum { ZSTD_resetTarget_CDict, ZSTD_resetTarget_CCtx } ZSTD_resetTarget_e; @@ -1525,19 +1527,22 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, assert(!ZSTD_workspace_reserve_failed(ws)); /* check that allocation hasn't already failed */ - DEBUGLOG(5, "reserving table space"); - /* table Space */ - ms->hashTable = (U32*)ZSTD_workspace_reserve_table(ws, hSize * sizeof(U32)); - ms->chainTable = (U32*)ZSTD_workspace_reserve_table(ws, chainSize * sizeof(U32)); - ms->hashTable3 = (U32*)ZSTD_workspace_reserve_table(ws, h3Size * sizeof(U32)); - RETURN_ERROR_IF(ZSTD_workspace_reserve_failed(ws), memory_allocation, - "failed a workspace allocation in ZSTD_reset_matchState"); - DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_noMemset); - if (crp!=ZSTDcrp_noMemset) { - /* reset tables only */ - memset(ms->hashTable, 0, hSize * sizeof(U32)); - memset(ms->chainTable, 0, chainSize * sizeof(U32)); - memset(ms->hashTable3, 0, h3Size * sizeof(U32)); + if (crp!=ZSTDcrp_noRealloc) { + DEBUGLOG(5, "reserving table space"); + /* table Space */ + ms->hashTable = (U32*)ZSTD_workspace_reserve_table(ws, hSize * sizeof(U32)); + ms->chainTable = (U32*)ZSTD_workspace_reserve_table(ws, chainSize * sizeof(U32)); + ms->hashTable3 = (U32*)ZSTD_workspace_reserve_table(ws, h3Size * sizeof(U32)); + RETURN_ERROR_IF(ZSTD_workspace_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); + + DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_noMemset); + if (crp!=ZSTDcrp_noMemset) { + /* reset tables only */ + memset(ms->hashTable, 0, hSize * sizeof(U32)); + memset(ms->chainTable, 0, chainSize * sizeof(U32)); + memset(ms->hashTable3, 0, h3Size * sizeof(U32)); + } } /* opt parser space */ @@ -1600,7 +1605,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, &zc->blockState.matchState, &zc->workspace, ¶ms.cParams, - crp, ZSTD_resetTarget_CCtx)); + ZSTDcrp_noRealloc, ZSTD_resetTarget_CCtx)); } return ZSTD_continueCCtx(zc, ¶ms, pledgedSrcSize); } } } @@ -1691,16 +1696,43 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock); + /* ZSTD_wildcopy() is used to copy into the literals buffer, + * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. + */ + zc->seqStore.litStart = ZSTD_workspace_reserve_buffer(&zc->workspace, blockSize + WILDCOPY_OVERLENGTH); + zc->seqStore.maxNbLit = blockSize; + + /* buffers */ + zc->inBuffSize = buffInSize; + zc->inBuff = (char*)ZSTD_workspace_reserve_buffer(&zc->workspace, buffInSize); + zc->outBuffSize = buffOutSize; + zc->outBuff = (char*)ZSTD_workspace_reserve_buffer(&zc->workspace, buffOutSize); + + /* ldm bucketOffsets table */ + if (params.ldmParams.enableLdm) { + size_t const ldmBucketSize = + ((size_t)1) << (params.ldmParams.hashLog - + params.ldmParams.bucketSizeLog); + zc->ldmState.bucketOffsets = ZSTD_workspace_reserve_buffer(&zc->workspace, ldmBucketSize); + memset(zc->ldmState.bucketOffsets, 0, ldmBucketSize); + } + + /* sequences storage */ + ZSTD_referenceExternalSequences(zc, NULL, 0); + zc->seqStore.maxNbSeq = maxNbSeq; + zc->seqStore.llCode = ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); + zc->seqStore.mlCode = ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); + zc->seqStore.ofCode = ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); + zc->seqStore.sequencesStart = (seqDef*)ZSTD_workspace_reserve_aligned(&zc->workspace, maxNbSeq * sizeof(seqDef)); + FORWARD_IF_ERROR(ZSTD_reset_matchState( &zc->blockState.matchState, &zc->workspace, ¶ms.cParams, crp, ZSTD_resetTarget_CCtx)); - DEBUGLOG(3, "Done allocating match state"); - /* ldm hash table */ - /* initialize bucketOffsets table later for pointer alignment */ + /* initialize bucketOffsets table separately for pointer alignment */ if (params.ldmParams.enableLdm) { size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog; zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_workspace_reserve_aligned(&zc->workspace, ldmHSize * sizeof(ldmEntry_t)); @@ -1709,37 +1741,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->maxNbLdmSequences = maxNbLdmSeq; memset(&zc->ldmState.window, 0, sizeof(zc->ldmState.window)); - } - - /* ldm bucketOffsets table */ - if (params.ldmParams.enableLdm) { - size_t const ldmBucketSize = - ((size_t)1) << (params.ldmParams.hashLog - - params.ldmParams.bucketSizeLog); - zc->ldmState.bucketOffsets = (BYTE*)ZSTD_workspace_reserve_aligned(&zc->workspace, ldmBucketSize); - memset(zc->ldmState.bucketOffsets, 0, ldmBucketSize); ZSTD_window_clear(&zc->ldmState.window); } - ZSTD_referenceExternalSequences(zc, NULL, 0); - - /* sequences storage */ - zc->seqStore.maxNbSeq = maxNbSeq; - zc->seqStore.sequencesStart = (seqDef*)ZSTD_workspace_reserve_aligned(&zc->workspace, maxNbSeq * sizeof(seqDef)); - zc->seqStore.llCode = (BYTE*)ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); - zc->seqStore.mlCode = (BYTE*)ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); - zc->seqStore.ofCode = (BYTE*)ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); - - /* ZSTD_wildcopy() is used to copy into the literals buffer, - * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. - */ - zc->seqStore.litStart = (BYTE*)ZSTD_workspace_reserve_buffer(&zc->workspace, blockSize + WILDCOPY_OVERLENGTH); - zc->seqStore.maxNbLit = blockSize; - - /* buffers */ - zc->inBuffSize = buffInSize; - zc->inBuff = (char*)ZSTD_workspace_reserve_buffer(&zc->workspace, buffInSize); - zc->outBuffSize = buffOutSize; - zc->outBuff = (char*)ZSTD_workspace_reserve_buffer(&zc->workspace, buffOutSize); return 0; } diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 287be309..60c00799 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -223,6 +223,12 @@ struct ZSTD_CCtx_params_s { ZSTD_customMem customMem; }; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */ +typedef enum { + ZSTD_workspace_alloc_objects, + ZSTD_workspace_alloc_buffers, + ZSTD_workspace_alloc_aligned +} ZSTD_workspace_alloc_phase_e; + /** * Zstd fits all its internal datastructures into a single continuous buffer, * so that it only needs to perform a single OS allocation (or so that a buffer @@ -294,7 +300,7 @@ typedef struct { int allocFailed; int workspaceOversizedDuration; - int staticAllocDone; + ZSTD_workspace_alloc_phase_e phase; } ZSTD_CCtx_workspace; struct ZSTD_CCtx_s { From e69b67e33ad26b0250e039da089762c8a19a44d6 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 12 Aug 2019 18:36:48 -0400 Subject: [PATCH 025/144] Alloc Tables Separately --- lib/compress/zstd_compress.c | 99 +++++++++++++++++---------- lib/compress/zstd_compress_internal.h | 14 ++-- 2 files changed, 68 insertions(+), 45 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 8d7d4439..012ea0cb 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -58,9 +58,10 @@ static size_t ZSTD_workspace_align(size_t size, size_t align) { static void* ZSTD_workspace_reserve(ZSTD_CCtx_workspace* ws, size_t bytes, ZSTD_workspace_alloc_phase_e phase) { /* TODO(felixh): alignment */ - void* alloc = ws->allocEnd; - void* newAllocEnd = (BYTE *)ws->allocEnd + bytes; - DEBUGLOG(3, "wksp: reserving %zd bytes, %zd bytes remaining", bytes, (BYTE *)ws->workspaceEnd - (BYTE *)newAllocEnd); + void* alloc = (BYTE *)ws->allocStart - bytes; + void* bottom = ws->tableEnd; + DEBUGLOG(3, "wksp: reserving align %zd bytes, %zd bytes remaining", + bytes, (BYTE *)alloc - (BYTE *)bottom); assert(phase >= ws->phase); if (phase > ws->phase) { if (ws->phase <= ZSTD_workspace_alloc_buffers) { @@ -68,12 +69,42 @@ static void* ZSTD_workspace_reserve(ZSTD_CCtx_workspace* ws, size_t bytes, ZSTD_ } ws->phase = phase; } - assert(newAllocEnd <= ws->workspaceEnd); - if (newAllocEnd > ws->workspaceEnd) { + assert(alloc >= bottom); + if (alloc < bottom) { ws->allocFailed = 1; return NULL; } - ws->allocEnd = newAllocEnd; + ws->allocStart = alloc; + return alloc; +} + +/** + * Aligned on sizeof(unsigned). These buffers have the special property that + * their values remain constrained, allowing us to re-use them without + * memset()-ing them. + */ +static void* ZSTD_workspace_reserve_table(ZSTD_CCtx_workspace* ws, size_t bytes) { + /* TODO(felixh): alignment */ + const ZSTD_workspace_alloc_phase_e phase = ZSTD_workspace_alloc_aligned; + void* alloc = ws->tableEnd; + void* end = (BYTE *)alloc + bytes; + void* top = ws->allocStart; + DEBUGLOG(3, "wksp: reserving table %zd bytes, %zd bytes remaining", + bytes, (BYTE *)top - (BYTE *)end); + assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? + assert(phase >= ws->phase); + if (phase > ws->phase) { + if (ws->phase <= ZSTD_workspace_alloc_buffers) { + + } + ws->phase = phase; + } + assert(end <= top); + if (end > top) { + ws->allocFailed = 1; + return NULL; + } + ws->tableEnd = end; return alloc; } @@ -84,7 +115,6 @@ static void* ZSTD_workspace_reserve_object(ZSTD_CCtx_workspace* ws, size_t bytes size_t roundedBytes = ZSTD_workspace_align(bytes, sizeof(void*)); void* start = ws->objectEnd; void* end = (BYTE*)start + roundedBytes; - assert(ws->allocEnd == ws->objectEnd); DEBUGLOG(3, "wksp: reserving %zd bytes object (rounded to %zd), %zd bytes remaining", bytes, roundedBytes, (BYTE *)ws->workspaceEnd - (BYTE *)end); assert((bytes & (sizeof(void*)-1)) == 0); // TODO ??? if (ws->phase != ZSTD_workspace_alloc_objects || end > ws->workspaceEnd) { @@ -93,20 +123,10 @@ static void* ZSTD_workspace_reserve_object(ZSTD_CCtx_workspace* ws, size_t bytes return NULL; } ws->objectEnd = end; - ws->allocEnd = end; + ws->tableEnd = end; return start; } -/** - * Aligned on sizeof(unsigned). These buffers have the special property that - * their values remain constrained, allowing us to re-use them without - * memset()-ing them. - */ -static void* ZSTD_workspace_reserve_table(ZSTD_CCtx_workspace* ws, size_t bytes) { - assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? - return ZSTD_workspace_reserve(ws, ZSTD_workspace_align(bytes, sizeof(unsigned)), ZSTD_workspace_alloc_aligned); -} - /** * Aligned on sizeof(unsigned). */ @@ -134,10 +154,15 @@ static int ZSTD_workspace_bump_oversized_duration(ZSTD_CCtx_workspace* ws) { return 0; } +static void ZSTD_workspace_clear_tables(ZSTD_CCtx_workspace* ws) { + ws->tableEnd = ws->objectEnd; +} + static void ZSTD_workspace_clear(ZSTD_CCtx_workspace* ws) { DEBUGLOG(3, "wksp: clearing!"); ZSTD_workspace_bump_oversized_duration(ws); - ws->allocEnd = ws->objectEnd; + ws->tableEnd = ws->objectEnd; + ws->allocStart = ws->workspaceEnd; ws->allocFailed = 0; if (ws->phase > ZSTD_workspace_alloc_buffers) { ws->phase = ZSTD_workspace_alloc_buffers; @@ -177,7 +202,7 @@ static void ZSTD_workspace_free(ZSTD_CCtx_workspace* ws, ZSTD_customMem customMe } static int ZSTD_workspace_check_available(ZSTD_CCtx_workspace* ws, size_t minFree) { - return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->allocEnd) >= minFree; + return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd) >= minFree; } static int ZSTD_workspace_check_wasteful(ZSTD_CCtx_workspace* ws, size_t minFree) { @@ -1527,22 +1552,22 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, assert(!ZSTD_workspace_reserve_failed(ws)); /* check that allocation hasn't already failed */ - if (crp!=ZSTDcrp_noRealloc) { - DEBUGLOG(5, "reserving table space"); - /* table Space */ - ms->hashTable = (U32*)ZSTD_workspace_reserve_table(ws, hSize * sizeof(U32)); - ms->chainTable = (U32*)ZSTD_workspace_reserve_table(ws, chainSize * sizeof(U32)); - ms->hashTable3 = (U32*)ZSTD_workspace_reserve_table(ws, h3Size * sizeof(U32)); - RETURN_ERROR_IF(ZSTD_workspace_reserve_failed(ws), memory_allocation, - "failed a workspace allocation in ZSTD_reset_matchState"); - - DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_noMemset); - if (crp!=ZSTDcrp_noMemset) { - /* reset tables only */ - memset(ms->hashTable, 0, hSize * sizeof(U32)); - memset(ms->chainTable, 0, chainSize * sizeof(U32)); - memset(ms->hashTable3, 0, h3Size * sizeof(U32)); - } + ZSTD_workspace_clear_tables(ws); + + DEBUGLOG(5, "reserving table space"); + /* table Space */ + ms->hashTable = (U32*)ZSTD_workspace_reserve_table(ws, hSize * sizeof(U32)); + ms->chainTable = (U32*)ZSTD_workspace_reserve_table(ws, chainSize * sizeof(U32)); + ms->hashTable3 = (U32*)ZSTD_workspace_reserve_table(ws, h3Size * sizeof(U32)); + RETURN_ERROR_IF(ZSTD_workspace_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); + + DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_noMemset); + if (crp!=ZSTDcrp_noMemset) { + /* reset tables only */ + memset(ms->hashTable, 0, hSize * sizeof(U32)); + memset(ms->chainTable, 0, chainSize * sizeof(U32)); + memset(ms->hashTable3, 0, h3Size * sizeof(U32)); } /* opt parser space */ @@ -1744,6 +1769,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_window_clear(&zc->ldmState.window); } + assert(!ZSTD_workspace_check_available(&zc->workspace, 1)); + return 0; } } diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 60c00799..d067c095 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -255,6 +255,9 @@ typedef enum { * * Workspace Layout: * + * [ ... workspace ... ] + * [objects][tables ... ->] free space [<- ... aligned][<- ... buffers] + * * In order to accomplish this, the various objects that live in the workspace * are divided into the following categories: * @@ -287,18 +290,11 @@ typedef struct { void* objectEnd; - // // void* tableZoneStart; - // void* tableAllocStart; - // void* tableAllocEnd; - // // void* tableZoneEnd; + void* tableEnd; - // void* seqEnd; + void* allocStart; - // void* bufferBegin; - - void* allocEnd; int allocFailed; - int workspaceOversizedDuration; ZSTD_workspace_alloc_phase_e phase; } ZSTD_CCtx_workspace; From 75d574368b4ca876a4bfca884e13cb22d97b075a Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 12 Aug 2019 19:24:24 -0400 Subject: [PATCH 026/144] When Loading Dict By Copy, Always Put it in the Workspace --- lib/compress/zstd_compress.c | 24 +++++++----------------- lib/compress/zstd_compress_internal.h | 1 + 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 012ea0cb..b975efda 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -132,7 +132,7 @@ static void* ZSTD_workspace_reserve_object(ZSTD_CCtx_workspace* ws, size_t bytes */ static void* ZSTD_workspace_reserve_aligned(ZSTD_CCtx_workspace* ws, size_t bytes) { assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? - return ZSTD_workspace_reserve(ws, ZSTD_workspace_align(bytes, sizeof(unsigned)), ZSTD_workspace_alloc_aligned); + return ZSTD_workspace_reserve(ws, ZSTD_workspace_align(bytes, sizeof(U32)), ZSTD_workspace_alloc_aligned); } /** @@ -222,7 +222,6 @@ static int ZSTD_workspace_reserve_failed(const ZSTD_CCtx_workspace* ws) { * Context memory management ***************************************/ struct ZSTD_CDict_s { - void* dictBuffer; const void* dictContent; size_t dictContentSize; U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */ @@ -3256,7 +3255,7 @@ size_t ZSTD_estimateCDictSize_advanced( { DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict)); return sizeof(ZSTD_CDict) + HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) - + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_workspace_align(dictSize, sizeof(void *))); } size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel) @@ -3269,7 +3268,7 @@ size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) { if (cdict==NULL) return 0; /* support sizeof on NULL */ DEBUGLOG(5, "sizeof(*cdict) : %u", (unsigned)sizeof(*cdict)); - return ZSTD_workspace_sizeof(&cdict->workspace) + (cdict->dictBuffer ? cdict->dictContentSize : 0) + sizeof(*cdict); + return ZSTD_workspace_sizeof(&cdict->workspace) + sizeof(*cdict); } static size_t ZSTD_initCDict_internal( @@ -3283,13 +3282,11 @@ static size_t ZSTD_initCDict_internal( assert(!ZSTD_checkCParams(cParams)); cdict->matchState.cParams = cParams; if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) { - cdict->dictBuffer = NULL; cdict->dictContent = dictBuffer; } else { - void* const internalBuffer = ZSTD_malloc(dictSize, cdict->customMem); - cdict->dictBuffer = internalBuffer; - cdict->dictContent = internalBuffer; + void *internalBuffer = ZSTD_workspace_reserve_object(&cdict->workspace, ZSTD_workspace_align(dictSize, sizeof(void*))); RETURN_ERROR_IF(!internalBuffer, memory_allocation); + cdict->dictContent = internalBuffer; memcpy(internalBuffer, dictBuffer, dictSize); } cdict->dictContentSize = dictSize; @@ -3334,7 +3331,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, if (!customMem.customAlloc ^ !customMem.customFree) return NULL; { ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); - size_t const workspaceSize = HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0); + size_t const workspaceSize = HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_workspace_align(dictSize, sizeof(void*))); void* const workspace = ZSTD_malloc(workspaceSize, customMem); if (!cdict || !workspace) { @@ -3377,7 +3374,6 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) if (cdict==NULL) return 0; /* support free on NULL */ { ZSTD_customMem const cMem = cdict->customMem; ZSTD_workspace_free(&cdict->workspace, cMem); - ZSTD_free(cdict->dictBuffer, cMem); ZSTD_free(cdict, cMem); return 0; } @@ -3422,15 +3418,9 @@ const ZSTD_CDict* ZSTD_initStaticCDict( (unsigned)workspaceSize, (unsigned)neededSize, (unsigned)(workspaceSize < neededSize)); if (workspaceSize < neededSize) return NULL; - if (dictLoadMethod == ZSTD_dlm_byCopy) { - void *dictCopy = ZSTD_workspace_reserve_object(&cdict->workspace, ZSTD_workspace_align(dictSize, sizeof(void*))); - memcpy(dictCopy, dict, dictSize); - dict = dictCopy; - } - if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dict, dictSize, - ZSTD_dlm_byRef, dictContentType, + dictLoadMethod, dictContentType, cParams) )) return NULL; diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index d067c095..71226757 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -270,6 +270,7 @@ typedef enum { * Examples: * - Entropy Workspace * - 2 x ZSTD_compressedBlockState_t + * - CDict dictionary contents sometimes??? // TODO * * - Tables: these are any of several different datastructures (hash tables, * chain tables, binary trees) that all respect a common format: they are From e936b73889395cdadd6a5855dc001520f91dddfc Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 12 Aug 2019 20:01:57 -0400 Subject: [PATCH 027/144] Remove Overly-Restrictive Assert --- lib/compress/zstd_compress.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index b975efda..f081f838 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -201,8 +201,12 @@ static void ZSTD_workspace_free(ZSTD_CCtx_workspace* ws, ZSTD_customMem customMe ZSTD_workspace_clear(ws); } +static size_t ZSTD_workspace_available_space(ZSTD_CCtx_workspace* ws) { + return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); +} + static int ZSTD_workspace_check_available(ZSTD_CCtx_workspace* ws, size_t minFree) { - return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd) >= minFree; + return ZSTD_workspace_available_space(ws) >= minFree; } static int ZSTD_workspace_check_wasteful(ZSTD_CCtx_workspace* ws, size_t minFree) { @@ -1768,7 +1772,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_window_clear(&zc->ldmState.window); } - assert(!ZSTD_workspace_check_available(&zc->workspace, 1)); + DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_workspace_available_space(&zc->workspace)); return 0; } From 88c2fcd0ee13d566fd34886bd3a8ff921eb246db Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 13 Aug 2019 11:57:20 -0400 Subject: [PATCH 028/144] Align Alloc Pointer When Transitioning from Buffers to Aligned Allocs --- lib/compress/zstd_compress.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index f081f838..78a57945 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -64,8 +64,19 @@ static void* ZSTD_workspace_reserve(ZSTD_CCtx_workspace* ws, size_t bytes, ZSTD_ bytes, (BYTE *)alloc - (BYTE *)bottom); assert(phase >= ws->phase); if (phase > ws->phase) { - if (ws->phase <= ZSTD_workspace_alloc_buffers) { - + if (ws->phase < ZSTD_workspace_alloc_buffers && + phase >= ZSTD_workspace_alloc_buffers) { + } + if (ws->phase < ZSTD_workspace_alloc_aligned && + phase >= ZSTD_workspace_alloc_aligned) { + /* If unaligned allocations down from a too-large top have left us + * unaligned, we need to realign our alloc ptr. Technically, this + * can consume space that is unaccounted for in the neededSpace + * calculation. However, I believe this can only happen when the + * workspace is too large, and specifically when it is too large + * by a larger margin than the space that will be consumed. */ + /* TODO: cleaner, compiler warning friendly way to do this??? */ + alloc = (BYTE*)alloc - ((size_t)alloc & (sizeof(U32)-1)); } ws->phase = phase; } From 58b69ab15c9088a7c8afbc01431f3e33add54ed5 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 14 Aug 2019 11:35:53 -0400 Subject: [PATCH 029/144] Only the CCtx Itself Needs to be Cleared during Static CCtx Init --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 78a57945..a8bf0d92 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -281,7 +281,7 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize) ZSTD_CCtx* const cctx = (ZSTD_CCtx*) workspace; if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */ if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ - memset(workspace, 0, workspaceSize); /* may be a bit generous, could memset be smaller ? */ /* TODO(felixh): investigate */ + memset(cctx, 0, sizeof(ZSTD_CCtx)); cctx->staticSize = workspaceSize; ZSTD_workspace_init(&cctx->workspace, (void*)(cctx+1), workspaceSize - sizeof(ZSTD_CCtx)); From 65057cf009a749e66b045b0a709f5e95c6c8abd1 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 14 Aug 2019 14:44:17 -0400 Subject: [PATCH 030/144] Rewrite ZSTD_initStaticCCtx to Alloc CCtx in Workspace --- lib/compress/zstd_compress.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index a8bf0d92..f27698c0 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -278,12 +278,19 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize) { - ZSTD_CCtx* const cctx = (ZSTD_CCtx*) workspace; + ZSTD_CCtx_workspace tmpWorkspace; + ZSTD_CCtx* cctx; if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */ if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ + ZSTD_workspace_init(&tmpWorkspace, workspace, workspaceSize); + + cctx = (ZSTD_CCtx*)ZSTD_workspace_reserve_object(&tmpWorkspace, sizeof(ZSTD_CCtx)); + if (cctx == NULL) { + return NULL; + } memset(cctx, 0, sizeof(ZSTD_CCtx)); + cctx->workspace = tmpWorkspace; cctx->staticSize = workspaceSize; - ZSTD_workspace_init(&cctx->workspace, (void*)(cctx+1), workspaceSize - sizeof(ZSTD_CCtx)); /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ if (!ZSTD_workspace_check_available(&cctx->workspace, HUF_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; From 7a2416a863803956cd64979f04f46858d369cfbc Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 14 Aug 2019 16:48:01 -0400 Subject: [PATCH 031/144] Allocate CDict in Workspace (Rather than in Separate Allocation) --- lib/compress/zstd_compress.c | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index f27698c0..4e0b76b1 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -325,7 +325,11 @@ static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) { assert(cctx != NULL); assert(cctx->staticSize == 0); - ZSTD_workspace_free(&cctx->workspace, cctx->customMem); + /* Only free workspace if cctx not in workspace, otherwise the workspace + * will be freed when the cctx itself is freed. */ + if ((void*)cctx->workspace.workspace != (void*)cctx) { + ZSTD_workspace_free(&cctx->workspace, cctx->customMem); + } ZSTD_clearAllDicts(cctx); #ifdef ZSTD_MULTITHREAD ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; @@ -3352,17 +3356,27 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, DEBUGLOG(3, "ZSTD_createCDict_advanced, mode %u", (unsigned)dictContentType); if (!customMem.customAlloc ^ !customMem.customFree) return NULL; - { ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); - size_t const workspaceSize = HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_workspace_align(dictSize, sizeof(void*))); + { size_t const workspaceSize = + sizeof(ZSTD_CDict) + + HUF_WORKSPACE_SIZE + + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_workspace_align(dictSize, sizeof(void*))); void* const workspace = ZSTD_malloc(workspaceSize, customMem); + ZSTD_CCtx_workspace tmpWorkspace; + ZSTD_CDict* cdict; - if (!cdict || !workspace) { - ZSTD_free(cdict, customMem); + if (!workspace) { ZSTD_free(workspace, customMem); return NULL; } + + ZSTD_workspace_init(&tmpWorkspace, workspace, workspaceSize); + + cdict = (ZSTD_CDict*)ZSTD_workspace_reserve_object(&tmpWorkspace, sizeof(ZSTD_CDict)); + assert(cdict != NULL); + cdict->workspace = tmpWorkspace; cdict->customMem = customMem; - ZSTD_workspace_init(&cdict->workspace, workspace, workspaceSize); if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dictBuffer, dictSize, dictLoadMethod, dictContentType, @@ -3395,7 +3409,11 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) { if (cdict==NULL) return 0; /* support free on NULL */ { ZSTD_customMem const cMem = cdict->customMem; - ZSTD_workspace_free(&cdict->workspace, cMem); + /* Only free workspace if cdict not in workspace, otherwise the + * workspace will be freed when the cdict itself is freed. */ + if ((void*)cdict->workspace.workspace != (void*)cdict) { + ZSTD_workspace_free(&cdict->workspace, cMem); + } ZSTD_free(cdict, cMem); return 0; } From 2abe0145b18ec5aba2367e35ebacd12c5150eac5 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 14 Aug 2019 16:55:12 -0400 Subject: [PATCH 032/144] Improve Comments a Bit --- lib/compress/zstd_compress.c | 9 ++++++++- lib/compress/zstd_compress_internal.h | 20 ++++++++++++++++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 4e0b76b1..262efead 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -53,7 +53,6 @@ size_t ZSTD_compressBound(size_t srcSize) { */ static size_t ZSTD_workspace_align(size_t size, size_t align) { return size + align - 1 - ((size - 1) & (align - 1)); - // return size + 3 - ((size - 1) & 3); } static void* ZSTD_workspace_reserve(ZSTD_CCtx_workspace* ws, size_t bytes, ZSTD_workspace_alloc_phase_e phase) { @@ -165,10 +164,18 @@ static int ZSTD_workspace_bump_oversized_duration(ZSTD_CCtx_workspace* ws) { return 0; } +/** + * Invalidates table allocations. + * All other allocations remain valid. + */ static void ZSTD_workspace_clear_tables(ZSTD_CCtx_workspace* ws) { ws->tableEnd = ws->objectEnd; } +/** + * Invalidates all buffer, aligned, and table allocations. + * Object allocations remain valid. + */ static void ZSTD_workspace_clear(ZSTD_CCtx_workspace* ws) { DEBUGLOG(3, "wksp: clearing!"); ZSTD_workspace_bump_oversized_duration(ws); diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 71226757..52d15544 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -262,7 +262,10 @@ typedef enum { * are divided into the following categories: * * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict, - * so that literally everything fits in a single buffer. + * so that literally everything fits in a single buffer. Note: if present, + * this must be the first object in the workspace, since ZSTD_free{CCtx, + * CDict}() rely on a pointer comparison to see whether one or two frees are + * required. * * - Fixed size objects: these are fixed-size, fixed-count objects that are * nonetheless "dynamically" allocated in the workspace so that we can @@ -277,13 +280,22 @@ typedef enum { * uint32_t arrays, all of whose values are between 0 and (nextSrc - base). * Their sizes depend on the cparams. * + * - Aligned: these buffers are used for various purposes that don't require + * any initialization before they're used. + * * - Uninitialized memory: these buffers are used for various purposes that * don't require any initialization before they're used. This means they can * be moved around at no cost for a new compression. - * - I/O Buffers * - * [workspace, workspace + workspaceSize) - * [] + * Allocating Memory: + * + * The various types of objects must be allocated in order, so they can be + * correctly packed into the workspace buffer. That order is: + * + * 1. Objects + * 2. Buffers + * 3. Aligned + * 4. Tables */ typedef struct { void* workspace; From ebd162194fb7624162154206dea9bc3824e33ffe Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 14 Aug 2019 17:11:16 -0400 Subject: [PATCH 033/144] Clean Up TODOs and Comments --- lib/compress/zstd_compress.c | 52 +++++++++++++-------------- lib/compress/zstd_compress_internal.h | 8 +++-- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 262efead..7002e0d4 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -51,11 +51,16 @@ size_t ZSTD_compressBound(size_t srcSize) { /** * Align must be a power of 2. */ -static size_t ZSTD_workspace_align(size_t size, size_t align) { - return size + align - 1 - ((size - 1) & (align - 1)); +static size_t ZSTD_workspace_align(size_t size, size_t const align) { + size_t const mask = align - 1; + assert((align & mask) == 0); + return (size + mask) & ~mask; } -static void* ZSTD_workspace_reserve(ZSTD_CCtx_workspace* ws, size_t bytes, ZSTD_workspace_alloc_phase_e phase) { +/** + * Internal function, use wrappers instead. + */ +static void* ZSTD_workspace_reserve_internal(ZSTD_CCtx_workspace* ws, size_t bytes, ZSTD_workspace_alloc_phase_e phase) { /* TODO(felixh): alignment */ void* alloc = (BYTE *)ws->allocStart - bytes; void* bottom = ws->tableEnd; @@ -88,6 +93,21 @@ static void* ZSTD_workspace_reserve(ZSTD_CCtx_workspace* ws, size_t bytes, ZSTD_ return alloc; } +/** + * Unaligned. + */ +static BYTE* ZSTD_workspace_reserve_buffer(ZSTD_CCtx_workspace* ws, size_t bytes) { + return (BYTE*)ZSTD_workspace_reserve_internal(ws, bytes, ZSTD_workspace_alloc_buffers); +} + +/** + * Aligned on sizeof(unsigned). + */ +static void* ZSTD_workspace_reserve_aligned(ZSTD_CCtx_workspace* ws, size_t bytes) { + assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? + return ZSTD_workspace_reserve_internal(ws, ZSTD_workspace_align(bytes, sizeof(U32)), ZSTD_workspace_alloc_aligned); +} + /** * Aligned on sizeof(unsigned). These buffers have the special property that * their values remain constrained, allowing us to re-use them without @@ -126,7 +146,8 @@ static void* ZSTD_workspace_reserve_object(ZSTD_CCtx_workspace* ws, size_t bytes void* start = ws->objectEnd; void* end = (BYTE*)start + roundedBytes; DEBUGLOG(3, "wksp: reserving %zd bytes object (rounded to %zd), %zd bytes remaining", bytes, roundedBytes, (BYTE *)ws->workspaceEnd - (BYTE *)end); - assert((bytes & (sizeof(void*)-1)) == 0); // TODO ??? + assert(((size_t)start & (sizeof(void*)-1)) == 0); + assert((bytes & (sizeof(void*)-1)) == 0); if (ws->phase != ZSTD_workspace_alloc_objects || end > ws->workspaceEnd) { DEBUGLOG(3, "wksp: object alloc failed!"); ws->allocFailed = 1; @@ -137,21 +158,6 @@ static void* ZSTD_workspace_reserve_object(ZSTD_CCtx_workspace* ws, size_t bytes return start; } -/** - * Aligned on sizeof(unsigned). - */ -static void* ZSTD_workspace_reserve_aligned(ZSTD_CCtx_workspace* ws, size_t bytes) { - assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? - return ZSTD_workspace_reserve(ws, ZSTD_workspace_align(bytes, sizeof(U32)), ZSTD_workspace_alloc_aligned); -} - -/** - * Unaligned. - */ -static BYTE* ZSTD_workspace_reserve_buffer(ZSTD_CCtx_workspace* ws, size_t bytes) { - return (BYTE*)ZSTD_workspace_reserve(ws, bytes, ZSTD_workspace_alloc_buffers); -} - // TODO static int ZSTD_workspace_bump_oversized_duration(ZSTD_CCtx_workspace* ws) { (void)ws; @@ -185,16 +191,11 @@ static void ZSTD_workspace_clear(ZSTD_CCtx_workspace* ws) { if (ws->phase > ZSTD_workspace_alloc_buffers) { ws->phase = ZSTD_workspace_alloc_buffers; } - - // ws->table = NULL; - // ws->tableEnd = NULL; - - // ws->bufferBegin = ws->workspaceEnd; } static void ZSTD_workspace_init(ZSTD_CCtx_workspace* ws, void* start, size_t size) { DEBUGLOG(3, "wksp: init'ing with %zd bytes", size); - assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ + assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ ws->workspace = start; ws->workspaceEnd = (BYTE*)start + size; ws->objectEnd = ws->workspace; @@ -1723,7 +1724,6 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, /* Statically sized space. * entropyWorkspace never moves, * though prev/next block swap places */ - /* assert(((size_t)zc->workspace.workspace & 3) == 0); */ /* ensure correct alignment */ /* TODO(felixh): check elsewhere */ assert(ZSTD_workspace_check_available(&zc->workspace, 2 * sizeof(ZSTD_compressedBlockState_t))); zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_workspace_reserve_object(&zc->workspace, sizeof(ZSTD_compressedBlockState_t)); RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock"); diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 52d15544..fd3d030d 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -273,7 +273,7 @@ typedef enum { * Examples: * - Entropy Workspace * - 2 x ZSTD_compressedBlockState_t - * - CDict dictionary contents sometimes??? // TODO + * - CDict dictionary contents * * - Tables: these are any of several different datastructures (hash tables, * chain tables, binary trees) that all respect a common format: they are @@ -296,15 +296,17 @@ typedef enum { * 2. Buffers * 3. Aligned * 4. Tables + * + * Reusing Table Space: + * + * TODO(felixh): ... */ typedef struct { void* workspace; void* workspaceEnd; void* objectEnd; - void* tableEnd; - void* allocStart; int allocFailed; From 077a2d7dc92a3009beccb128cc809105b43f6e40 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 15 Aug 2019 12:51:24 -0400 Subject: [PATCH 034/144] Rename --- lib/compress/zstd_compress.c | 194 +++++++++++++------------- lib/compress/zstd_compress_internal.h | 14 +- 2 files changed, 104 insertions(+), 104 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 7002e0d4..45720e1d 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -51,7 +51,7 @@ size_t ZSTD_compressBound(size_t srcSize) { /** * Align must be a power of 2. */ -static size_t ZSTD_workspace_align(size_t size, size_t const align) { +static size_t ZSTD_cwksp_align(size_t size, size_t const align) { size_t const mask = align - 1; assert((align & mask) == 0); return (size + mask) & ~mask; @@ -60,7 +60,7 @@ static size_t ZSTD_workspace_align(size_t size, size_t const align) { /** * Internal function, use wrappers instead. */ -static void* ZSTD_workspace_reserve_internal(ZSTD_CCtx_workspace* ws, size_t bytes, ZSTD_workspace_alloc_phase_e phase) { +static void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { /* TODO(felixh): alignment */ void* alloc = (BYTE *)ws->allocStart - bytes; void* bottom = ws->tableEnd; @@ -68,11 +68,11 @@ static void* ZSTD_workspace_reserve_internal(ZSTD_CCtx_workspace* ws, size_t byt bytes, (BYTE *)alloc - (BYTE *)bottom); assert(phase >= ws->phase); if (phase > ws->phase) { - if (ws->phase < ZSTD_workspace_alloc_buffers && - phase >= ZSTD_workspace_alloc_buffers) { + if (ws->phase < ZSTD_cwksp_alloc_buffers && + phase >= ZSTD_cwksp_alloc_buffers) { } - if (ws->phase < ZSTD_workspace_alloc_aligned && - phase >= ZSTD_workspace_alloc_aligned) { + if (ws->phase < ZSTD_cwksp_alloc_aligned && + phase >= ZSTD_cwksp_alloc_aligned) { /* If unaligned allocations down from a too-large top have left us * unaligned, we need to realign our alloc ptr. Technically, this * can consume space that is unaccounted for in the neededSpace @@ -96,16 +96,16 @@ static void* ZSTD_workspace_reserve_internal(ZSTD_CCtx_workspace* ws, size_t byt /** * Unaligned. */ -static BYTE* ZSTD_workspace_reserve_buffer(ZSTD_CCtx_workspace* ws, size_t bytes) { - return (BYTE*)ZSTD_workspace_reserve_internal(ws, bytes, ZSTD_workspace_alloc_buffers); +static BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) { + return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers); } /** * Aligned on sizeof(unsigned). */ -static void* ZSTD_workspace_reserve_aligned(ZSTD_CCtx_workspace* ws, size_t bytes) { +static void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) { assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? - return ZSTD_workspace_reserve_internal(ws, ZSTD_workspace_align(bytes, sizeof(U32)), ZSTD_workspace_alloc_aligned); + return ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, sizeof(U32)), ZSTD_cwksp_alloc_aligned); } /** @@ -113,9 +113,9 @@ static void* ZSTD_workspace_reserve_aligned(ZSTD_CCtx_workspace* ws, size_t byte * their values remain constrained, allowing us to re-use them without * memset()-ing them. */ -static void* ZSTD_workspace_reserve_table(ZSTD_CCtx_workspace* ws, size_t bytes) { +static void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { /* TODO(felixh): alignment */ - const ZSTD_workspace_alloc_phase_e phase = ZSTD_workspace_alloc_aligned; + const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned; void* alloc = ws->tableEnd; void* end = (BYTE *)alloc + bytes; void* top = ws->allocStart; @@ -124,7 +124,7 @@ static void* ZSTD_workspace_reserve_table(ZSTD_CCtx_workspace* ws, size_t bytes) assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? assert(phase >= ws->phase); if (phase > ws->phase) { - if (ws->phase <= ZSTD_workspace_alloc_buffers) { + if (ws->phase <= ZSTD_cwksp_alloc_buffers) { } ws->phase = phase; @@ -141,14 +141,14 @@ static void* ZSTD_workspace_reserve_table(ZSTD_CCtx_workspace* ws, size_t bytes) /** * Aligned on sizeof(void*). */ -static void* ZSTD_workspace_reserve_object(ZSTD_CCtx_workspace* ws, size_t bytes) { - size_t roundedBytes = ZSTD_workspace_align(bytes, sizeof(void*)); +static void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { + size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); void* start = ws->objectEnd; void* end = (BYTE*)start + roundedBytes; DEBUGLOG(3, "wksp: reserving %zd bytes object (rounded to %zd), %zd bytes remaining", bytes, roundedBytes, (BYTE *)ws->workspaceEnd - (BYTE *)end); assert(((size_t)start & (sizeof(void*)-1)) == 0); assert((bytes & (sizeof(void*)-1)) == 0); - if (ws->phase != ZSTD_workspace_alloc_objects || end > ws->workspaceEnd) { + if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { DEBUGLOG(3, "wksp: object alloc failed!"); ws->allocFailed = 1; return NULL; @@ -159,7 +159,7 @@ static void* ZSTD_workspace_reserve_object(ZSTD_CCtx_workspace* ws, size_t bytes } // TODO -static int ZSTD_workspace_bump_oversized_duration(ZSTD_CCtx_workspace* ws) { +static int ZSTD_cwksp_bump_oversized_duration(ZSTD_cwksp* ws) { (void)ws; // if (((BYTE*)ws->allocEnd - (BYTE*)ws->workspace) * ZSTD_WORKSPACETOOLARGE_FACTOR < (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace) { // ws->workspaceOversizedDuration++; @@ -174,7 +174,7 @@ static int ZSTD_workspace_bump_oversized_duration(ZSTD_CCtx_workspace* ws) { * Invalidates table allocations. * All other allocations remain valid. */ -static void ZSTD_workspace_clear_tables(ZSTD_CCtx_workspace* ws) { +static void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { ws->tableEnd = ws->objectEnd; } @@ -182,61 +182,61 @@ static void ZSTD_workspace_clear_tables(ZSTD_CCtx_workspace* ws) { * Invalidates all buffer, aligned, and table allocations. * Object allocations remain valid. */ -static void ZSTD_workspace_clear(ZSTD_CCtx_workspace* ws) { +static void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { DEBUGLOG(3, "wksp: clearing!"); - ZSTD_workspace_bump_oversized_duration(ws); + ZSTD_cwksp_bump_oversized_duration(ws); ws->tableEnd = ws->objectEnd; ws->allocStart = ws->workspaceEnd; ws->allocFailed = 0; - if (ws->phase > ZSTD_workspace_alloc_buffers) { - ws->phase = ZSTD_workspace_alloc_buffers; + if (ws->phase > ZSTD_cwksp_alloc_buffers) { + ws->phase = ZSTD_cwksp_alloc_buffers; } } -static void ZSTD_workspace_init(ZSTD_CCtx_workspace* ws, void* start, size_t size) { +static void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) { DEBUGLOG(3, "wksp: init'ing with %zd bytes", size); assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ ws->workspace = start; ws->workspaceEnd = (BYTE*)start + size; ws->objectEnd = ws->workspace; - ws->phase = ZSTD_workspace_alloc_objects; - ZSTD_workspace_clear(ws); + ws->phase = ZSTD_cwksp_alloc_objects; + ZSTD_cwksp_clear(ws); ws->workspaceOversizedDuration = 0; } -static size_t ZSTD_workspace_create(ZSTD_CCtx_workspace* ws, size_t size, ZSTD_customMem customMem) { +static size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { void* workspace = ZSTD_malloc(size, customMem); DEBUGLOG(3, "wksp: creating with %zd bytes", size); RETURN_ERROR_IF(workspace == NULL, memory_allocation); - ZSTD_workspace_init(ws, workspace, size); + ZSTD_cwksp_init(ws, workspace, size); return 0; } -static void ZSTD_workspace_free(ZSTD_CCtx_workspace* ws, ZSTD_customMem customMem) { +static void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { DEBUGLOG(3, "wksp: freeing"); ZSTD_free(ws->workspace, customMem); ws->workspace = NULL; ws->workspaceEnd = NULL; - ZSTD_workspace_clear(ws); + ZSTD_cwksp_clear(ws); } -static size_t ZSTD_workspace_available_space(ZSTD_CCtx_workspace* ws) { +static size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); } -static int ZSTD_workspace_check_available(ZSTD_CCtx_workspace* ws, size_t minFree) { - return ZSTD_workspace_available_space(ws) >= minFree; +static int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t minFree) { + return ZSTD_cwksp_available_space(ws) >= minFree; } -static int ZSTD_workspace_check_wasteful(ZSTD_CCtx_workspace* ws, size_t minFree) { - return ZSTD_workspace_check_available(ws, minFree * ZSTD_WORKSPACETOOLARGE_FACTOR) && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; +static int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t minFree) { + return ZSTD_cwksp_check_available(ws, minFree * ZSTD_WORKSPACETOOLARGE_FACTOR) && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; } -static size_t ZSTD_workspace_sizeof(const ZSTD_CCtx_workspace* ws) { +static size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { return (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace; } -static int ZSTD_workspace_reserve_failed(const ZSTD_CCtx_workspace* ws) { +static int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { return ws->allocFailed; } @@ -248,7 +248,7 @@ struct ZSTD_CDict_s { const void* dictContent; size_t dictContentSize; U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */ - ZSTD_CCtx_workspace workspace; + ZSTD_cwksp workspace; ZSTD_matchState_t matchState; ZSTD_compressedBlockState_t cBlockState; ZSTD_customMem customMem; @@ -286,25 +286,25 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize) { - ZSTD_CCtx_workspace tmpWorkspace; + ZSTD_cwksp ws; ZSTD_CCtx* cctx; if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */ if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ - ZSTD_workspace_init(&tmpWorkspace, workspace, workspaceSize); + ZSTD_cwksp_init(&ws, workspace, workspaceSize); - cctx = (ZSTD_CCtx*)ZSTD_workspace_reserve_object(&tmpWorkspace, sizeof(ZSTD_CCtx)); + cctx = (ZSTD_CCtx*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CCtx)); if (cctx == NULL) { return NULL; } memset(cctx, 0, sizeof(ZSTD_CCtx)); - cctx->workspace = tmpWorkspace; + cctx->workspace = ws; cctx->staticSize = workspaceSize; /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ - if (!ZSTD_workspace_check_available(&cctx->workspace, HUF_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; - cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_workspace_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); - cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_workspace_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); - cctx->entropyWorkspace = (U32*)ZSTD_workspace_reserve_object( + if (!ZSTD_cwksp_check_available(&cctx->workspace, HUF_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; + cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object( &cctx->workspace, HUF_WORKSPACE_SIZE); cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); return cctx; @@ -336,7 +336,7 @@ static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) /* Only free workspace if cctx not in workspace, otherwise the workspace * will be freed when the cctx itself is freed. */ if ((void*)cctx->workspace.workspace != (void*)cctx) { - ZSTD_workspace_free(&cctx->workspace, cctx->customMem); + ZSTD_cwksp_free(&cctx->workspace, cctx->customMem); } ZSTD_clearAllDicts(cctx); #ifdef ZSTD_MULTITHREAD @@ -369,7 +369,7 @@ static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) { if (cctx==NULL) return 0; /* support sizeof on NULL */ - return sizeof(*cctx) + ZSTD_workspace_sizeof(&cctx->workspace) + return sizeof(*cctx) + ZSTD_cwksp_sizeof(&cctx->workspace) + ZSTD_sizeof_localDict(cctx->localDict) + ZSTD_sizeof_mtctx(cctx); } @@ -1566,7 +1566,7 @@ typedef enum { ZSTD_resetTarget_CDict, ZSTD_resetTarget_CCtx } ZSTD_resetTarget_ static size_t ZSTD_reset_matchState(ZSTD_matchState_t* ms, - ZSTD_CCtx_workspace* ws, + ZSTD_cwksp* ws, const ZSTD_compressionParameters* cParams, ZSTD_compResetPolicy_e const crp, ZSTD_resetTarget_e const forWho) { @@ -1583,16 +1583,16 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, ms->window.nextSrc = ms->window.base + 1; /* see issue #1241 */ ZSTD_invalidateMatchState(ms); - assert(!ZSTD_workspace_reserve_failed(ws)); /* check that allocation hasn't already failed */ + assert(!ZSTD_cwksp_reserve_failed(ws)); /* check that allocation hasn't already failed */ - ZSTD_workspace_clear_tables(ws); + ZSTD_cwksp_clear_tables(ws); DEBUGLOG(5, "reserving table space"); /* table Space */ - ms->hashTable = (U32*)ZSTD_workspace_reserve_table(ws, hSize * sizeof(U32)); - ms->chainTable = (U32*)ZSTD_workspace_reserve_table(ws, chainSize * sizeof(U32)); - ms->hashTable3 = (U32*)ZSTD_workspace_reserve_table(ws, h3Size * sizeof(U32)); - RETURN_ERROR_IF(ZSTD_workspace_reserve_failed(ws), memory_allocation, + ms->hashTable = (U32*)ZSTD_cwksp_reserve_table(ws, hSize * sizeof(U32)); + ms->chainTable = (U32*)ZSTD_cwksp_reserve_table(ws, chainSize * sizeof(U32)); + ms->hashTable3 = (U32*)ZSTD_cwksp_reserve_table(ws, h3Size * sizeof(U32)); + RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, "failed a workspace allocation in ZSTD_reset_matchState"); DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_noMemset); @@ -1606,17 +1606,17 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, /* opt parser space */ if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) { DEBUGLOG(4, "reserving optimal parser space"); - ms->opt.litFreq = (unsigned*)ZSTD_workspace_reserve_aligned(ws, (1<opt.litLengthFreq = (unsigned*)ZSTD_workspace_reserve_aligned(ws, (MaxLL+1) * sizeof(unsigned)); - ms->opt.matchLengthFreq = (unsigned*)ZSTD_workspace_reserve_aligned(ws, (MaxML+1) * sizeof(unsigned)); - ms->opt.offCodeFreq = (unsigned*)ZSTD_workspace_reserve_aligned(ws, (MaxOff+1) * sizeof(unsigned)); - ms->opt.matchTable = (ZSTD_match_t*)ZSTD_workspace_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t)); - ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_workspace_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t)); + ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (1<opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxLL+1) * sizeof(unsigned)); + ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxML+1) * sizeof(unsigned)); + ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxOff+1) * sizeof(unsigned)); + ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t)); + ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t)); } ms->cParams = *cParams; - RETURN_ERROR_IF(ZSTD_workspace_reserve_failed(ws), memory_allocation, + RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, "failed a workspace allocation in ZSTD_reset_matchState"); return 0; @@ -1654,7 +1654,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->seqStore.maxNbSeq, zc->seqStore.maxNbLit, zbuff, pledgedSrcSize) ) { DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> consider continue mode"); - if (ZSTD_workspace_bump_oversized_duration(&zc->workspace) <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) { + if (ZSTD_cwksp_bump_oversized_duration(&zc->workspace) <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) { DEBUGLOG(4, "continue mode confirmed (wLog1=%u, blockSize1=%zu)", zc->appliedParams.cParams.windowLog, zc->blockSize); if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) { @@ -1703,8 +1703,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, tokenSpace + bufferSpace; - int const workspaceTooSmall = !ZSTD_workspace_check_available(&zc->workspace, neededSpace); - int const workspaceWasteful = ZSTD_workspace_check_wasteful(&zc->workspace, neededSpace); + int const workspaceTooSmall = !ZSTD_cwksp_check_available(&zc->workspace, neededSpace); + int const workspaceWasteful = ZSTD_cwksp_check_wasteful(&zc->workspace, neededSpace); DEBUGLOG(4, "Need %zuKB workspace, including %zuKB for match state, and %zuKB for buffers", neededSpace>>10, matchStateSize>>10, bufferSpace>>10); @@ -1712,28 +1712,28 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, if (workspaceTooSmall || workspaceWasteful) { DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB", - ZSTD_workspace_sizeof(&zc->workspace) >> 10, + ZSTD_cwksp_sizeof(&zc->workspace) >> 10, neededSpace >> 10); RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize"); - ZSTD_workspace_free(&zc->workspace, zc->customMem); - FORWARD_IF_ERROR(ZSTD_workspace_create(&zc->workspace, neededSpace, zc->customMem)); + ZSTD_cwksp_free(&zc->workspace, zc->customMem); + FORWARD_IF_ERROR(ZSTD_cwksp_create(&zc->workspace, neededSpace, zc->customMem)); DEBUGLOG(5, "reserving object space"); /* Statically sized space. * entropyWorkspace never moves, * though prev/next block swap places */ - assert(ZSTD_workspace_check_available(&zc->workspace, 2 * sizeof(ZSTD_compressedBlockState_t))); - zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_workspace_reserve_object(&zc->workspace, sizeof(ZSTD_compressedBlockState_t)); + assert(ZSTD_cwksp_check_available(&zc->workspace, 2 * sizeof(ZSTD_compressedBlockState_t))); + zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(&zc->workspace, sizeof(ZSTD_compressedBlockState_t)); RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock"); - zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_workspace_reserve_object(&zc->workspace, sizeof(ZSTD_compressedBlockState_t)); + zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(&zc->workspace, sizeof(ZSTD_compressedBlockState_t)); RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock"); - zc->entropyWorkspace = (U32*) ZSTD_workspace_reserve_object(&zc->workspace, HUF_WORKSPACE_SIZE); + zc->entropyWorkspace = (U32*) ZSTD_cwksp_reserve_object(&zc->workspace, HUF_WORKSPACE_SIZE); RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate entropyWorkspace"); } } - ZSTD_workspace_clear(&zc->workspace); + ZSTD_cwksp_clear(&zc->workspace); /* init params */ zc->appliedParams = params; @@ -1756,31 +1756,31 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, /* ZSTD_wildcopy() is used to copy into the literals buffer, * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. */ - zc->seqStore.litStart = ZSTD_workspace_reserve_buffer(&zc->workspace, blockSize + WILDCOPY_OVERLENGTH); + zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(&zc->workspace, blockSize + WILDCOPY_OVERLENGTH); zc->seqStore.maxNbLit = blockSize; /* buffers */ zc->inBuffSize = buffInSize; - zc->inBuff = (char*)ZSTD_workspace_reserve_buffer(&zc->workspace, buffInSize); + zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(&zc->workspace, buffInSize); zc->outBuffSize = buffOutSize; - zc->outBuff = (char*)ZSTD_workspace_reserve_buffer(&zc->workspace, buffOutSize); + zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(&zc->workspace, buffOutSize); /* ldm bucketOffsets table */ if (params.ldmParams.enableLdm) { size_t const ldmBucketSize = ((size_t)1) << (params.ldmParams.hashLog - params.ldmParams.bucketSizeLog); - zc->ldmState.bucketOffsets = ZSTD_workspace_reserve_buffer(&zc->workspace, ldmBucketSize); + zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(&zc->workspace, ldmBucketSize); memset(zc->ldmState.bucketOffsets, 0, ldmBucketSize); } /* sequences storage */ ZSTD_referenceExternalSequences(zc, NULL, 0); zc->seqStore.maxNbSeq = maxNbSeq; - zc->seqStore.llCode = ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); - zc->seqStore.mlCode = ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); - zc->seqStore.ofCode = ZSTD_workspace_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); - zc->seqStore.sequencesStart = (seqDef*)ZSTD_workspace_reserve_aligned(&zc->workspace, maxNbSeq * sizeof(seqDef)); + zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); + zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); + zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); + zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(&zc->workspace, maxNbSeq * sizeof(seqDef)); FORWARD_IF_ERROR(ZSTD_reset_matchState( &zc->blockState.matchState, @@ -1792,16 +1792,16 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, /* initialize bucketOffsets table separately for pointer alignment */ if (params.ldmParams.enableLdm) { size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog; - zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_workspace_reserve_aligned(&zc->workspace, ldmHSize * sizeof(ldmEntry_t)); + zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(&zc->workspace, ldmHSize * sizeof(ldmEntry_t)); memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); - zc->ldmSequences = (rawSeq*)ZSTD_workspace_reserve_aligned(&zc->workspace, maxNbLdmSeq * sizeof(rawSeq)); + zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(&zc->workspace, maxNbLdmSeq * sizeof(rawSeq)); zc->maxNbLdmSequences = maxNbLdmSeq; memset(&zc->ldmState.window, 0, sizeof(zc->ldmState.window)); ZSTD_window_clear(&zc->ldmState.window); } - DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_workspace_available_space(&zc->workspace)); + DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(&zc->workspace)); return 0; } @@ -3288,7 +3288,7 @@ size_t ZSTD_estimateCDictSize_advanced( { DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict)); return sizeof(ZSTD_CDict) + HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) - + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_workspace_align(dictSize, sizeof(void *))); + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_cwksp_align(dictSize, sizeof(void *))); } size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel) @@ -3301,7 +3301,7 @@ size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) { if (cdict==NULL) return 0; /* support sizeof on NULL */ DEBUGLOG(5, "sizeof(*cdict) : %u", (unsigned)sizeof(*cdict)); - return ZSTD_workspace_sizeof(&cdict->workspace) + sizeof(*cdict); + return ZSTD_cwksp_sizeof(&cdict->workspace) + sizeof(*cdict); } static size_t ZSTD_initCDict_internal( @@ -3317,14 +3317,14 @@ static size_t ZSTD_initCDict_internal( if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) { cdict->dictContent = dictBuffer; } else { - void *internalBuffer = ZSTD_workspace_reserve_object(&cdict->workspace, ZSTD_workspace_align(dictSize, sizeof(void*))); + void *internalBuffer = ZSTD_cwksp_reserve_object(&cdict->workspace, ZSTD_cwksp_align(dictSize, sizeof(void*))); RETURN_ERROR_IF(!internalBuffer, memory_allocation); cdict->dictContent = internalBuffer; memcpy(internalBuffer, dictBuffer, dictSize); } cdict->dictContentSize = dictSize; - cdict->entropyWorkspace = (U32*)ZSTD_workspace_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE); + cdict->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE); /* Reset the state to no dictionary */ @@ -3368,9 +3368,9 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 - : ZSTD_workspace_align(dictSize, sizeof(void*))); + : ZSTD_cwksp_align(dictSize, sizeof(void*))); void* const workspace = ZSTD_malloc(workspaceSize, customMem); - ZSTD_CCtx_workspace tmpWorkspace; + ZSTD_cwksp ws; ZSTD_CDict* cdict; if (!workspace) { @@ -3378,11 +3378,11 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, return NULL; } - ZSTD_workspace_init(&tmpWorkspace, workspace, workspaceSize); + ZSTD_cwksp_init(&ws, workspace, workspaceSize); - cdict = (ZSTD_CDict*)ZSTD_workspace_reserve_object(&tmpWorkspace, sizeof(ZSTD_CDict)); + cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); assert(cdict != NULL); - cdict->workspace = tmpWorkspace; + cdict->workspace = ws; cdict->customMem = customMem; if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dictBuffer, dictSize, @@ -3419,7 +3419,7 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) /* Only free workspace if cdict not in workspace, otherwise the * workspace will be freed when the cdict itself is freed. */ if ((void*)cdict->workspace.workspace != (void*)cdict) { - ZSTD_workspace_free(&cdict->workspace, cMem); + ZSTD_cwksp_free(&cdict->workspace, cMem); } ZSTD_free(cdict, cMem); return 0; @@ -3447,16 +3447,16 @@ const ZSTD_CDict* ZSTD_initStaticCDict( ZSTD_compressionParameters cParams) { size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0); - size_t const neededSize = sizeof(ZSTD_CDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_workspace_align(dictSize, sizeof(void*))) + size_t const neededSize = sizeof(ZSTD_CDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_cwksp_align(dictSize, sizeof(void*))) + HUF_WORKSPACE_SIZE + matchStateSize; ZSTD_CDict* cdict; if ((size_t)workspace & 7) return NULL; /* 8-aligned */ { - ZSTD_CCtx_workspace ws; - ZSTD_workspace_init(&ws, workspace, workspaceSize); - cdict = (ZSTD_CDict*)ZSTD_workspace_reserve_object(&ws, sizeof(ZSTD_CDict)); + ZSTD_cwksp ws; + ZSTD_cwksp_init(&ws, workspace, workspaceSize); + cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); if (cdict == NULL) return NULL; cdict->workspace = ws; } diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index fd3d030d..806ce849 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -224,10 +224,10 @@ struct ZSTD_CCtx_params_s { }; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */ typedef enum { - ZSTD_workspace_alloc_objects, - ZSTD_workspace_alloc_buffers, - ZSTD_workspace_alloc_aligned -} ZSTD_workspace_alloc_phase_e; + ZSTD_cwksp_alloc_objects, + ZSTD_cwksp_alloc_buffers, + ZSTD_cwksp_alloc_aligned +} ZSTD_cwksp_alloc_phase_e; /** * Zstd fits all its internal datastructures into a single continuous buffer, @@ -311,8 +311,8 @@ typedef struct { int allocFailed; int workspaceOversizedDuration; - ZSTD_workspace_alloc_phase_e phase; -} ZSTD_CCtx_workspace; + ZSTD_cwksp_alloc_phase_e phase; +} ZSTD_cwksp; struct ZSTD_CCtx_s { ZSTD_compressionStage_e stage; @@ -322,7 +322,7 @@ struct ZSTD_CCtx_s { ZSTD_CCtx_params appliedParams; U32 dictID; - ZSTD_CCtx_workspace workspace; /* manages buffer for dynamic allocations */ + ZSTD_cwksp workspace; /* manages buffer for dynamic allocations */ size_t blockSize; unsigned long long pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */ unsigned long long consumedSrcSize; From b511a84adc733fb505f7bbec6f7cc063dee8ba36 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 20 Aug 2019 14:02:50 -0400 Subject: [PATCH 035/144] Move Workspace Functions to Their Own File --- lib/compress/zstd_compress.c | 203 -------------------------- lib/compress/zstd_compress_internal.h | 92 +----------- lib/compress/zstd_cwksp.c | 203 ++++++++++++++++++++++++++ lib/compress/zstd_cwksp.h | 175 ++++++++++++++++++++++ 4 files changed, 379 insertions(+), 294 deletions(-) create mode 100644 lib/compress/zstd_cwksp.c create mode 100644 lib/compress/zstd_cwksp.h diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 45720e1d..f88f3b7c 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -38,209 +38,6 @@ size_t ZSTD_compressBound(size_t srcSize) { } -/*-************************************* -* Workspace memory management -***************************************/ -#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 /* define "workspace is too large" as this number of times larger than needed */ -#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 /* when workspace is continuously too large - * during at least this number of times, - * context's memory usage is considered wasteful, - * because it's sized to handle a worst case scenario which rarely happens. - * In which case, resize it down to free some memory */ - -/** - * Align must be a power of 2. - */ -static size_t ZSTD_cwksp_align(size_t size, size_t const align) { - size_t const mask = align - 1; - assert((align & mask) == 0); - return (size + mask) & ~mask; -} - -/** - * Internal function, use wrappers instead. - */ -static void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { - /* TODO(felixh): alignment */ - void* alloc = (BYTE *)ws->allocStart - bytes; - void* bottom = ws->tableEnd; - DEBUGLOG(3, "wksp: reserving align %zd bytes, %zd bytes remaining", - bytes, (BYTE *)alloc - (BYTE *)bottom); - assert(phase >= ws->phase); - if (phase > ws->phase) { - if (ws->phase < ZSTD_cwksp_alloc_buffers && - phase >= ZSTD_cwksp_alloc_buffers) { - } - if (ws->phase < ZSTD_cwksp_alloc_aligned && - phase >= ZSTD_cwksp_alloc_aligned) { - /* If unaligned allocations down from a too-large top have left us - * unaligned, we need to realign our alloc ptr. Technically, this - * can consume space that is unaccounted for in the neededSpace - * calculation. However, I believe this can only happen when the - * workspace is too large, and specifically when it is too large - * by a larger margin than the space that will be consumed. */ - /* TODO: cleaner, compiler warning friendly way to do this??? */ - alloc = (BYTE*)alloc - ((size_t)alloc & (sizeof(U32)-1)); - } - ws->phase = phase; - } - assert(alloc >= bottom); - if (alloc < bottom) { - ws->allocFailed = 1; - return NULL; - } - ws->allocStart = alloc; - return alloc; -} - -/** - * Unaligned. - */ -static BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) { - return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers); -} - -/** - * Aligned on sizeof(unsigned). - */ -static void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) { - assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? - return ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, sizeof(U32)), ZSTD_cwksp_alloc_aligned); -} - -/** - * Aligned on sizeof(unsigned). These buffers have the special property that - * their values remain constrained, allowing us to re-use them without - * memset()-ing them. - */ -static void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { - /* TODO(felixh): alignment */ - const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned; - void* alloc = ws->tableEnd; - void* end = (BYTE *)alloc + bytes; - void* top = ws->allocStart; - DEBUGLOG(3, "wksp: reserving table %zd bytes, %zd bytes remaining", - bytes, (BYTE *)top - (BYTE *)end); - assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? - assert(phase >= ws->phase); - if (phase > ws->phase) { - if (ws->phase <= ZSTD_cwksp_alloc_buffers) { - - } - ws->phase = phase; - } - assert(end <= top); - if (end > top) { - ws->allocFailed = 1; - return NULL; - } - ws->tableEnd = end; - return alloc; -} - -/** - * Aligned on sizeof(void*). - */ -static void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { - size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); - void* start = ws->objectEnd; - void* end = (BYTE*)start + roundedBytes; - DEBUGLOG(3, "wksp: reserving %zd bytes object (rounded to %zd), %zd bytes remaining", bytes, roundedBytes, (BYTE *)ws->workspaceEnd - (BYTE *)end); - assert(((size_t)start & (sizeof(void*)-1)) == 0); - assert((bytes & (sizeof(void*)-1)) == 0); - if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { - DEBUGLOG(3, "wksp: object alloc failed!"); - ws->allocFailed = 1; - return NULL; - } - ws->objectEnd = end; - ws->tableEnd = end; - return start; -} - -// TODO -static int ZSTD_cwksp_bump_oversized_duration(ZSTD_cwksp* ws) { - (void)ws; - // if (((BYTE*)ws->allocEnd - (BYTE*)ws->workspace) * ZSTD_WORKSPACETOOLARGE_FACTOR < (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace) { - // ws->workspaceOversizedDuration++; - // } else { - // ws->workspaceOversizedDuration = 0; - // } - // return ws->workspaceOversizedDuration; - return 0; -} - -/** - * Invalidates table allocations. - * All other allocations remain valid. - */ -static void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { - ws->tableEnd = ws->objectEnd; -} - -/** - * Invalidates all buffer, aligned, and table allocations. - * Object allocations remain valid. - */ -static void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { - DEBUGLOG(3, "wksp: clearing!"); - ZSTD_cwksp_bump_oversized_duration(ws); - ws->tableEnd = ws->objectEnd; - ws->allocStart = ws->workspaceEnd; - ws->allocFailed = 0; - if (ws->phase > ZSTD_cwksp_alloc_buffers) { - ws->phase = ZSTD_cwksp_alloc_buffers; - } -} - -static void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) { - DEBUGLOG(3, "wksp: init'ing with %zd bytes", size); - assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ - ws->workspace = start; - ws->workspaceEnd = (BYTE*)start + size; - ws->objectEnd = ws->workspace; - ws->phase = ZSTD_cwksp_alloc_objects; - ZSTD_cwksp_clear(ws); - ws->workspaceOversizedDuration = 0; -} - -static size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { - void* workspace = ZSTD_malloc(size, customMem); - DEBUGLOG(3, "wksp: creating with %zd bytes", size); - RETURN_ERROR_IF(workspace == NULL, memory_allocation); - ZSTD_cwksp_init(ws, workspace, size); - return 0; -} - -static void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { - DEBUGLOG(3, "wksp: freeing"); - ZSTD_free(ws->workspace, customMem); - ws->workspace = NULL; - ws->workspaceEnd = NULL; - ZSTD_cwksp_clear(ws); -} - -static size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { - return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); -} - -static int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t minFree) { - return ZSTD_cwksp_available_space(ws) >= minFree; -} - -static int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t minFree) { - return ZSTD_cwksp_check_available(ws, minFree * ZSTD_WORKSPACETOOLARGE_FACTOR) && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; -} - -static size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { - return (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace; -} - -static int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { - return ws->allocFailed; -} - - /*-************************************* * Context memory management ***************************************/ diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 806ce849..ae106e02 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -19,6 +19,7 @@ * Dependencies ***************************************/ #include "zstd_internal.h" +#include "zstd_cwksp.h" #ifdef ZSTD_MULTITHREAD # include "zstdmt_compress.h" #endif @@ -223,97 +224,6 @@ struct ZSTD_CCtx_params_s { ZSTD_customMem customMem; }; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */ -typedef enum { - ZSTD_cwksp_alloc_objects, - ZSTD_cwksp_alloc_buffers, - ZSTD_cwksp_alloc_aligned -} ZSTD_cwksp_alloc_phase_e; - -/** - * Zstd fits all its internal datastructures into a single continuous buffer, - * so that it only needs to perform a single OS allocation (or so that a buffer - * can be provided to it and it can perform no allocations at all). This buffer - * is called the workspace. - * - * Several optimizations complicate that process of allocating memory ranges - * from this workspace for each datastructure: - * - * - These different internal datastructures have different setup requirements. - * Some (e.g., the window buffer) don't care, and are happy to accept - * uninitialized memory. Others (e.g., the matchstate tables) can accept - * memory filled with unknown but bounded values (i.e., a memory area whose - * values are known to be constrained between 0 and some upper bound). If - * that constraint isn't known to be satisfied, the area has to be cleared. - * - * - We would like to reuse the objects in the workspace for multiple - * compressions without having to perform any expensive reallocation or - * reinitialization work. - * - * - We would like to be able to efficiently reuse the workspace across - * multiple compressions **even when the compression parameters change** and - * we need to resize some of the objects (where possible). - * - * Workspace Layout: - * - * [ ... workspace ... ] - * [objects][tables ... ->] free space [<- ... aligned][<- ... buffers] - * - * In order to accomplish this, the various objects that live in the workspace - * are divided into the following categories: - * - * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict, - * so that literally everything fits in a single buffer. Note: if present, - * this must be the first object in the workspace, since ZSTD_free{CCtx, - * CDict}() rely on a pointer comparison to see whether one or two frees are - * required. - * - * - Fixed size objects: these are fixed-size, fixed-count objects that are - * nonetheless "dynamically" allocated in the workspace so that we can - * control how they're initialized separately from the broader ZSTD_CCtx. - * Examples: - * - Entropy Workspace - * - 2 x ZSTD_compressedBlockState_t - * - CDict dictionary contents - * - * - Tables: these are any of several different datastructures (hash tables, - * chain tables, binary trees) that all respect a common format: they are - * uint32_t arrays, all of whose values are between 0 and (nextSrc - base). - * Their sizes depend on the cparams. - * - * - Aligned: these buffers are used for various purposes that don't require - * any initialization before they're used. - * - * - Uninitialized memory: these buffers are used for various purposes that - * don't require any initialization before they're used. This means they can - * be moved around at no cost for a new compression. - * - * Allocating Memory: - * - * The various types of objects must be allocated in order, so they can be - * correctly packed into the workspace buffer. That order is: - * - * 1. Objects - * 2. Buffers - * 3. Aligned - * 4. Tables - * - * Reusing Table Space: - * - * TODO(felixh): ... - */ -typedef struct { - void* workspace; - void* workspaceEnd; - - void* objectEnd; - void* tableEnd; - void* allocStart; - - int allocFailed; - int workspaceOversizedDuration; - ZSTD_cwksp_alloc_phase_e phase; -} ZSTD_cwksp; - struct ZSTD_CCtx_s { ZSTD_compressionStage_e stage; int cParamsChanged; /* == 1 if cParams(except wlog) or compression level are changed in requestedParams. Triggers transmission of new params to ZSTDMT (if available) then reset to 0. */ diff --git a/lib/compress/zstd_cwksp.c b/lib/compress/zstd_cwksp.c new file mode 100644 index 00000000..3b7341eb --- /dev/null +++ b/lib/compress/zstd_cwksp.c @@ -0,0 +1,203 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_cwksp.h" + +/** + * Align must be a power of 2. + */ +size_t ZSTD_cwksp_align(size_t size, size_t const align) { + size_t const mask = align - 1; + assert((align & mask) == 0); + return (size + mask) & ~mask; +} + +/** + * Internal function, use wrappers instead. + */ +void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { + /* TODO(felixh): alignment */ + void* alloc = (BYTE *)ws->allocStart - bytes; + void* bottom = ws->tableEnd; + DEBUGLOG(3, "wksp: reserving align %zd bytes, %zd bytes remaining", + bytes, (BYTE *)alloc - (BYTE *)bottom); + assert(phase >= ws->phase); + if (phase > ws->phase) { + if (ws->phase < ZSTD_cwksp_alloc_buffers && + phase >= ZSTD_cwksp_alloc_buffers) { + } + if (ws->phase < ZSTD_cwksp_alloc_aligned && + phase >= ZSTD_cwksp_alloc_aligned) { + /* If unaligned allocations down from a too-large top have left us + * unaligned, we need to realign our alloc ptr. Technically, this + * can consume space that is unaccounted for in the neededSpace + * calculation. However, I believe this can only happen when the + * workspace is too large, and specifically when it is too large + * by a larger margin than the space that will be consumed. */ + /* TODO: cleaner, compiler warning friendly way to do this??? */ + alloc = (BYTE*)alloc - ((size_t)alloc & (sizeof(U32)-1)); + } + ws->phase = phase; + } + assert(alloc >= bottom); + if (alloc < bottom) { + ws->allocFailed = 1; + return NULL; + } + ws->allocStart = alloc; + return alloc; +} + +/** + * Unaligned. + */ +BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) { + return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers); +} + +/** + * Aligned on sizeof(unsigned). + */ +void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) { + assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? + return ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, sizeof(U32)), ZSTD_cwksp_alloc_aligned); +} + +/** + * Aligned on sizeof(unsigned). These buffers have the special property that + * their values remain constrained, allowing us to re-use them without + * memset()-ing them. + */ +void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { + /* TODO(felixh): alignment */ + const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned; + void* alloc = ws->tableEnd; + void* end = (BYTE *)alloc + bytes; + void* top = ws->allocStart; + DEBUGLOG(3, "wksp: reserving table %zd bytes, %zd bytes remaining", + bytes, (BYTE *)top - (BYTE *)end); + assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? + assert(phase >= ws->phase); + if (phase > ws->phase) { + if (ws->phase <= ZSTD_cwksp_alloc_buffers) { + + } + ws->phase = phase; + } + assert(end <= top); + if (end > top) { + ws->allocFailed = 1; + return NULL; + } + ws->tableEnd = end; + return alloc; +} + +/** + * Aligned on sizeof(void*). + */ +void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { + size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); + void* start = ws->objectEnd; + void* end = (BYTE*)start + roundedBytes; + DEBUGLOG(3, "wksp: reserving %zd bytes object (rounded to %zd), %zd bytes remaining", bytes, roundedBytes, (BYTE *)ws->workspaceEnd - (BYTE *)end); + assert(((size_t)start & (sizeof(void*)-1)) == 0); + assert((bytes & (sizeof(void*)-1)) == 0); + if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { + DEBUGLOG(3, "wksp: object alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->objectEnd = end; + ws->tableEnd = end; + return start; +} + +// TODO +int ZSTD_cwksp_bump_oversized_duration(ZSTD_cwksp* ws) { + (void)ws; + // if (((BYTE*)ws->allocEnd - (BYTE*)ws->workspace) * ZSTD_WORKSPACETOOLARGE_FACTOR < (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace) { + // ws->workspaceOversizedDuration++; + // } else { + // ws->workspaceOversizedDuration = 0; + // } + // return ws->workspaceOversizedDuration; + return 0; +} + +/** + * Invalidates table allocations. + * All other allocations remain valid. + */ +void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { + ws->tableEnd = ws->objectEnd; +} + +/** + * Invalidates all buffer, aligned, and table allocations. + * Object allocations remain valid. + */ +void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { + DEBUGLOG(3, "wksp: clearing!"); + ZSTD_cwksp_bump_oversized_duration(ws); + ws->tableEnd = ws->objectEnd; + ws->allocStart = ws->workspaceEnd; + ws->allocFailed = 0; + if (ws->phase > ZSTD_cwksp_alloc_buffers) { + ws->phase = ZSTD_cwksp_alloc_buffers; + } +} + +void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) { + DEBUGLOG(3, "wksp: init'ing with %zd bytes", size); + assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ + ws->workspace = start; + ws->workspaceEnd = (BYTE*)start + size; + ws->objectEnd = ws->workspace; + ws->phase = ZSTD_cwksp_alloc_objects; + ZSTD_cwksp_clear(ws); + ws->workspaceOversizedDuration = 0; +} + +size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { + void* workspace = ZSTD_malloc(size, customMem); + DEBUGLOG(3, "wksp: creating with %zd bytes", size); + RETURN_ERROR_IF(workspace == NULL, memory_allocation); + ZSTD_cwksp_init(ws, workspace, size); + return 0; +} + +void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { + DEBUGLOG(3, "wksp: freeing"); + ZSTD_free(ws->workspace, customMem); + ws->workspace = NULL; + ws->workspaceEnd = NULL; + ZSTD_cwksp_clear(ws); +} + +size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); +} + +int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t minFree) { + return ZSTD_cwksp_available_space(ws) >= minFree; +} + +int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t minFree) { + return ZSTD_cwksp_check_available(ws, minFree * ZSTD_WORKSPACETOOLARGE_FACTOR) && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} + +size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { + return (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace; +} + +int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { + return ws->allocFailed; +} \ No newline at end of file diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h new file mode 100644 index 00000000..b5b35849 --- /dev/null +++ b/lib/compress/zstd_cwksp.h @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CWKSP_H +#define ZSTD_CWKSP_H + +#include "zstd_internal.h" + +#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 /* define "workspace is too large" as this number of times larger than needed */ +#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 /* when workspace is continuously too large + * during at least this number of times, + * context's memory usage is considered wasteful, + * because it's sized to handle a worst case scenario which rarely happens. + * In which case, resize it down to free some memory */ +typedef enum { + ZSTD_cwksp_alloc_objects, + ZSTD_cwksp_alloc_buffers, + ZSTD_cwksp_alloc_aligned +} ZSTD_cwksp_alloc_phase_e; + +/** + * Zstd fits all its internal datastructures into a single continuous buffer, + * so that it only needs to perform a single OS allocation (or so that a buffer + * can be provided to it and it can perform no allocations at all). This buffer + * is called the workspace. + * + * Several optimizations complicate that process of allocating memory ranges + * from this workspace for each datastructure: + * + * - These different internal datastructures have different setup requirements. + * Some (e.g., the window buffer) don't care, and are happy to accept + * uninitialized memory. Others (e.g., the matchstate tables) can accept + * memory filled with unknown but bounded values (i.e., a memory area whose + * values are known to be constrained between 0 and some upper bound). If + * that constraint isn't known to be satisfied, the area has to be cleared. + * + * - We would like to reuse the objects in the workspace for multiple + * compressions without having to perform any expensive reallocation or + * reinitialization work. + * + * - We would like to be able to efficiently reuse the workspace across + * multiple compressions **even when the compression parameters change** and + * we need to resize some of the objects (where possible). + * + * Workspace Layout: + * + * [ ... workspace ... ] + * [objects][tables ... ->] free space [<- ... aligned][<- ... buffers] + * + * In order to accomplish this, the various objects that live in the workspace + * are divided into the following categories: + * + * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict, + * so that literally everything fits in a single buffer. Note: if present, + * this must be the first object in the workspace, since ZSTD_free{CCtx, + * CDict}() rely on a pointer comparison to see whether one or two frees are + * required. + * + * - Fixed size objects: these are fixed-size, fixed-count objects that are + * nonetheless "dynamically" allocated in the workspace so that we can + * control how they're initialized separately from the broader ZSTD_CCtx. + * Examples: + * - Entropy Workspace + * - 2 x ZSTD_compressedBlockState_t + * - CDict dictionary contents + * + * - Tables: these are any of several different datastructures (hash tables, + * chain tables, binary trees) that all respect a common format: they are + * uint32_t arrays, all of whose values are between 0 and (nextSrc - base). + * Their sizes depend on the cparams. + * + * - Aligned: these buffers are used for various purposes that don't require + * any initialization before they're used. + * + * - Uninitialized memory: these buffers are used for various purposes that + * don't require any initialization before they're used. This means they can + * be moved around at no cost for a new compression. + * + * Allocating Memory: + * + * The various types of objects must be allocated in order, so they can be + * correctly packed into the workspace buffer. That order is: + * + * 1. Objects + * 2. Buffers + * 3. Aligned + * 4. Tables + * + * Reusing Table Space: + * + * TODO(felixh): ... + */ +typedef struct { + void* workspace; + void* workspaceEnd; + + void* objectEnd; + void* tableEnd; + void* allocStart; + + int allocFailed; + int workspaceOversizedDuration; + ZSTD_cwksp_alloc_phase_e phase; +} ZSTD_cwksp; + +/** + * Align must be a power of 2. + */ +size_t ZSTD_cwksp_align(size_t size, size_t const align); + +/** + * Internal function, use wrappers instead. + */ +void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase); + +/** + * Unaligned. + */ +BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes); + +/** + * Aligned on sizeof(unsigned). + */ +void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes); + +/** + * Aligned on sizeof(unsigned). These buffers have the special property that + * their values remain constrained, allowing us to re-use them without + * memset()-ing them. + */ +void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes); + +/** + * Aligned on sizeof(void*). + */ +void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes); + +int ZSTD_cwksp_bump_oversized_duration(ZSTD_cwksp* ws); + +/** + * Invalidates table allocations. + * All other allocations remain valid. + */ +void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws); + +/** + * Invalidates all buffer, aligned, and table allocations. + * Object allocations remain valid. + */ +void ZSTD_cwksp_clear(ZSTD_cwksp* ws); + +void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size); + +size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem); + +void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem); + +size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws); + +int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t minFree); + +int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t minFree); + +size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws); + +int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws); + +#endif /* ZSTD_CWKSP_H */ From 881bcd80cacaecdc457654ba47ff13f8b27752fe Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 20 Aug 2019 14:15:06 -0400 Subject: [PATCH 036/144] Cleanup from Move --- lib/compress/zstd_cwksp.c | 12 +++++++++-- lib/compress/zstd_cwksp.h | 43 +++++++++++++++++++++++++++++---------- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/lib/compress/zstd_cwksp.c b/lib/compress/zstd_cwksp.c index 3b7341eb..a117ad09 100644 --- a/lib/compress/zstd_cwksp.c +++ b/lib/compress/zstd_cwksp.c @@ -10,6 +10,10 @@ #include "zstd_cwksp.h" +#if defined (__cplusplus) +extern "C" { +#endif + /** * Align must be a power of 2. */ @@ -22,7 +26,7 @@ size_t ZSTD_cwksp_align(size_t size, size_t const align) { /** * Internal function, use wrappers instead. */ -void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { +static void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { /* TODO(felixh): alignment */ void* alloc = (BYTE *)ws->allocStart - bytes; void* bottom = ws->tableEnd; @@ -200,4 +204,8 @@ size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { return ws->allocFailed; -} \ No newline at end of file +} + +#if defined (__cplusplus) +} +#endif diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index b5b35849..965ca6b0 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -11,14 +11,32 @@ #ifndef ZSTD_CWKSP_H #define ZSTD_CWKSP_H +/*-************************************* +* Dependencies +***************************************/ #include "zstd_internal.h" -#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 /* define "workspace is too large" as this number of times larger than needed */ -#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 /* when workspace is continuously too large - * during at least this number of times, - * context's memory usage is considered wasteful, - * because it's sized to handle a worst case scenario which rarely happens. - * In which case, resize it down to free some memory */ +#if defined (__cplusplus) +extern "C" { +#endif + +/*-************************************* +* Constants +***************************************/ + +/* define "workspace is too large" as this number of times larger than needed */ +#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 + +/* when workspace is continuously too large + * during at least this number of times, + * context's memory usage is considered wasteful, + * because it's sized to handle a worst case scenario which rarely happens. + * In which case, resize it down to free some memory */ +#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 + +/*-************************************* +* Structures +***************************************/ typedef enum { ZSTD_cwksp_alloc_objects, ZSTD_cwksp_alloc_buffers, @@ -110,16 +128,15 @@ typedef struct { ZSTD_cwksp_alloc_phase_e phase; } ZSTD_cwksp; +/*-************************************* +* Functions +***************************************/ + /** * Align must be a power of 2. */ size_t ZSTD_cwksp_align(size_t size, size_t const align); -/** - * Internal function, use wrappers instead. - */ -void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase); - /** * Unaligned. */ @@ -172,4 +189,8 @@ size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws); int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws); +#if defined (__cplusplus) +} +#endif + #endif /* ZSTD_CWKSP_H */ From 4e9b1341cfa573f369239c02402a71fe14ac1e52 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 20 Aug 2019 15:14:12 -0400 Subject: [PATCH 037/144] Add New File to Meson Build --- build/meson/lib/meson.build | 1 + 1 file changed, 1 insertion(+) diff --git a/build/meson/lib/meson.build b/build/meson/lib/meson.build index ef669327..48f1e7cc 100644 --- a/build/meson/lib/meson.build +++ b/build/meson/lib/meson.build @@ -30,6 +30,7 @@ libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_compress.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_compress_literals.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_compress_sequences.c'), + join_paths(zstd_rootdir, 'lib/compress/zstd_cwksp.c'), join_paths(zstd_rootdir, 'lib/compress/zstdmt_compress.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_fast.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_double_fast.c'), From e8cc1374770f2631bbea2e4f9b7c8cd0d8128005 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 20 Aug 2019 15:19:58 -0400 Subject: [PATCH 038/144] Add New File to Visual Studio Projects --- build/VS2008/fullbench/fullbench.vcproj | 8 ++++++++ build/VS2008/fuzzer/fuzzer.vcproj | 8 ++++++++ build/VS2008/zstd/zstd.vcproj | 8 ++++++++ build/VS2008/zstdlib/zstdlib.vcproj | 8 ++++++++ build/VS2010/fullbench/fullbench.vcxproj | 2 ++ build/VS2010/fuzzer/fuzzer.vcxproj | 2 ++ build/VS2010/libzstd-dll/libzstd-dll.vcxproj | 2 ++ build/VS2010/libzstd/libzstd.vcxproj | 2 ++ build/VS2010/zstd/zstd.vcxproj | 2 ++ 9 files changed, 42 insertions(+) diff --git a/build/VS2008/fullbench/fullbench.vcproj b/build/VS2008/fullbench/fullbench.vcproj index cd9391ec..ffb0a015 100644 --- a/build/VS2008/fullbench/fullbench.vcproj +++ b/build/VS2008/fullbench/fullbench.vcproj @@ -372,6 +372,10 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c" > + + @@ -510,6 +514,10 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h" > + + diff --git a/build/VS2008/fuzzer/fuzzer.vcproj b/build/VS2008/fuzzer/fuzzer.vcproj index 5f10bbe7..c3875ec7 100644 --- a/build/VS2008/fuzzer/fuzzer.vcproj +++ b/build/VS2008/fuzzer/fuzzer.vcproj @@ -420,6 +420,10 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c" > + + @@ -546,6 +550,10 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h" > + + diff --git a/build/VS2008/zstd/zstd.vcproj b/build/VS2008/zstd/zstd.vcproj index 34ccb6f4..5b57dde5 100644 --- a/build/VS2008/zstd/zstd.vcproj +++ b/build/VS2008/zstd/zstd.vcproj @@ -432,6 +432,10 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c" > + + @@ -626,6 +630,10 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h" > + + diff --git a/build/VS2008/zstdlib/zstdlib.vcproj b/build/VS2008/zstdlib/zstdlib.vcproj index be46d351..e864b5fb 100644 --- a/build/VS2008/zstdlib/zstdlib.vcproj +++ b/build/VS2008/zstdlib/zstdlib.vcproj @@ -404,6 +404,10 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c" > + + @@ -558,6 +562,10 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.h" > + + diff --git a/build/VS2010/fullbench/fullbench.vcxproj b/build/VS2010/fullbench/fullbench.vcxproj index 105c692b..65e7ff78 100644 --- a/build/VS2010/fullbench/fullbench.vcxproj +++ b/build/VS2010/fullbench/fullbench.vcxproj @@ -169,6 +169,7 @@ + @@ -197,6 +198,7 @@ + diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj index 7f7e404d..e9b88a8a 100644 --- a/build/VS2010/fuzzer/fuzzer.vcxproj +++ b/build/VS2010/fuzzer/fuzzer.vcxproj @@ -169,6 +169,7 @@ + @@ -200,6 +201,7 @@ + diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj index 840d1ec3..4140879d 100644 --- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj +++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj @@ -33,6 +33,7 @@ + @@ -82,6 +83,7 @@ + diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj index b84d45b9..02690cef 100644 --- a/build/VS2010/libzstd/libzstd.vcxproj +++ b/build/VS2010/libzstd/libzstd.vcxproj @@ -33,6 +33,7 @@ + @@ -82,6 +83,7 @@ + diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj index d65f55b3..2a21ba4c 100644 --- a/build/VS2010/zstd/zstd.vcxproj +++ b/build/VS2010/zstd/zstd.vcxproj @@ -34,6 +34,7 @@ + @@ -79,6 +80,7 @@ + From 901bba4ca6b4486aaa3e1bc827e13c804fbb2260 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 22 Aug 2019 16:09:00 -0400 Subject: [PATCH 039/144] Re-Implement Workspace Shrinking when Oversized --- lib/compress/zstd_compress.c | 3 ++- lib/compress/zstd_cwksp.c | 52 +++++++++++++++++++----------------- lib/compress/zstd_cwksp.h | 22 +++++++++------ 3 files changed, 43 insertions(+), 34 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index f88f3b7c..8abb7a48 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1451,7 +1451,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->seqStore.maxNbSeq, zc->seqStore.maxNbLit, zbuff, pledgedSrcSize) ) { DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> consider continue mode"); - if (ZSTD_cwksp_bump_oversized_duration(&zc->workspace) <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) { + ZSTD_cwksp_bump_oversized_duration(&zc->workspace, 0); + if (!ZSTD_cwksp_check_wasteful(&zc->workspace, 0)) { DEBUGLOG(4, "continue mode confirmed (wLog1=%u, blockSize1=%zu)", zc->appliedParams.cParams.windowLog, zc->blockSize); if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) { diff --git a/lib/compress/zstd_cwksp.c b/lib/compress/zstd_cwksp.c index a117ad09..51df4ee3 100644 --- a/lib/compress/zstd_cwksp.c +++ b/lib/compress/zstd_cwksp.c @@ -124,18 +124,6 @@ void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { return start; } -// TODO -int ZSTD_cwksp_bump_oversized_duration(ZSTD_cwksp* ws) { - (void)ws; - // if (((BYTE*)ws->allocEnd - (BYTE*)ws->workspace) * ZSTD_WORKSPACETOOLARGE_FACTOR < (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace) { - // ws->workspaceOversizedDuration++; - // } else { - // ws->workspaceOversizedDuration = 0; - // } - // return ws->workspaceOversizedDuration; - return 0; -} - /** * Invalidates table allocations. * All other allocations remain valid. @@ -150,7 +138,6 @@ void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { */ void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { DEBUGLOG(3, "wksp: clearing!"); - ZSTD_cwksp_bump_oversized_duration(ws); ws->tableEnd = ws->objectEnd; ws->allocStart = ws->workspaceEnd; ws->allocFailed = 0; @@ -186,18 +173,6 @@ void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { ZSTD_cwksp_clear(ws); } -size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { - return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); -} - -int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t minFree) { - return ZSTD_cwksp_available_space(ws) >= minFree; -} - -int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t minFree) { - return ZSTD_cwksp_check_available(ws, minFree * ZSTD_WORKSPACETOOLARGE_FACTOR) && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; -} - size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { return (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace; } @@ -206,6 +181,33 @@ int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { return ws->allocFailed; } +size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); +} + +void ZSTD_cwksp_bump_oversized_duration( + ZSTD_cwksp* ws, size_t additionalNeededSpace) { + if (ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)) { + ws->workspaceOversizedDuration++; + } else { + ws->workspaceOversizedDuration = 0; + } +} + +int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_available_space(ws) >= additionalNeededSpace; +} + +int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_available( + ws, additionalNeededSpace * ZSTD_WORKSPACETOOLARGE_FACTOR); +} + +int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_too_large(ws, additionalNeededSpace) + && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} + #if defined (__cplusplus) } #endif diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 965ca6b0..5abc97be 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -159,8 +159,6 @@ void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes); */ void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes); -int ZSTD_cwksp_bump_oversized_duration(ZSTD_cwksp* ws); - /** * Invalidates table allocations. * All other allocations remain valid. @@ -179,16 +177,24 @@ size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem); void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem); -size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws); - -int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t minFree); - -int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t minFree); - size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws); int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws); +/*-************************************* +* Functions Checking Free Space +***************************************/ + +size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws); + +int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeededSpace); + +int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace); + +int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace); + +void ZSTD_cwksp_bump_oversized_duration(ZSTD_cwksp* ws, size_t additionalNeededSpace); + #if defined (__cplusplus) } #endif From 7321e4c9f3c79f88b0e60eaf23615e03a751719b Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 27 Aug 2019 15:21:59 -0400 Subject: [PATCH 040/144] Remove Unused noRealloc CRP Value --- lib/compress/zstd_compress.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 8abb7a48..663bfeac 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1357,7 +1357,7 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params, return 0; } -typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_noRealloc } ZSTD_compResetPolicy_e; +typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset } ZSTD_compResetPolicy_e; typedef enum { ZSTD_resetTarget_CDict, ZSTD_resetTarget_CCtx } ZSTD_resetTarget_e; @@ -1461,7 +1461,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, &zc->blockState.matchState, &zc->workspace, ¶ms.cParams, - ZSTDcrp_noRealloc, ZSTD_resetTarget_CCtx)); + ZSTDcrp_noMemset, ZSTD_resetTarget_CCtx)); } return ZSTD_continueCCtx(zc, ¶ms, pledgedSrcSize); } } } From 7100d24221b0cc4235bfdc67a8783924903ed055 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 29 Aug 2019 16:33:45 -0400 Subject: [PATCH 041/144] Fix Rescale Continue Special Case --- lib/compress/zstd_compress.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 663bfeac..d2ff96b5 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1461,7 +1461,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, &zc->blockState.matchState, &zc->workspace, ¶ms.cParams, - ZSTDcrp_noMemset, ZSTD_resetTarget_CCtx)); + crp, + ZSTD_resetTarget_CCtx)); } return ZSTD_continueCCtx(zc, ¶ms, pledgedSrcSize); } } } From 2405c03bcd048caf30e2e6549320840b289de3de Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 3 Sep 2019 12:41:35 -0400 Subject: [PATCH 042/144] Fix DEBUGLOG Statement Levels --- lib/compress/zstd_cwksp.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/lib/compress/zstd_cwksp.c b/lib/compress/zstd_cwksp.c index 51df4ee3..f8503975 100644 --- a/lib/compress/zstd_cwksp.c +++ b/lib/compress/zstd_cwksp.c @@ -30,7 +30,7 @@ static void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwks /* TODO(felixh): alignment */ void* alloc = (BYTE *)ws->allocStart - bytes; void* bottom = ws->tableEnd; - DEBUGLOG(3, "wksp: reserving align %zd bytes, %zd bytes remaining", + DEBUGLOG(4, "cwksp: reserving align %zd bytes, %zd bytes remaining", bytes, (BYTE *)alloc - (BYTE *)bottom); assert(phase >= ws->phase); if (phase > ws->phase) { @@ -80,14 +80,13 @@ void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) { * memset()-ing them. */ void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { - /* TODO(felixh): alignment */ const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned; void* alloc = ws->tableEnd; void* end = (BYTE *)alloc + bytes; void* top = ws->allocStart; - DEBUGLOG(3, "wksp: reserving table %zd bytes, %zd bytes remaining", - bytes, (BYTE *)top - (BYTE *)end); - assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? + DEBUGLOG(4, "cwksp: reserving table %zd bytes, %zd bytes remaining", + bytes, ZSTD_cwksp_available_space(ws) - bytes); + assert((bytes & (sizeof(U32)-1)) == 0); assert(phase >= ws->phase); if (phase > ws->phase) { if (ws->phase <= ZSTD_cwksp_alloc_buffers) { @@ -97,6 +96,7 @@ void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { } assert(end <= top); if (end > top) { + DEBUGLOG(4, "cwksp: object alloc failed!"); ws->allocFailed = 1; return NULL; } @@ -111,11 +111,13 @@ void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); void* start = ws->objectEnd; void* end = (BYTE*)start + roundedBytes; - DEBUGLOG(3, "wksp: reserving %zd bytes object (rounded to %zd), %zd bytes remaining", bytes, roundedBytes, (BYTE *)ws->workspaceEnd - (BYTE *)end); + DEBUGLOG(4, + "cwksp: reserving %zd bytes object (rounded to %zd), %zd bytes remaining", + bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); assert(((size_t)start & (sizeof(void*)-1)) == 0); assert((bytes & (sizeof(void*)-1)) == 0); if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { - DEBUGLOG(3, "wksp: object alloc failed!"); + DEBUGLOG(4, "cwksp: object alloc failed!"); ws->allocFailed = 1; return NULL; } @@ -129,6 +131,7 @@ void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { * All other allocations remain valid. */ void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: clearing tables!"); ws->tableEnd = ws->objectEnd; } @@ -137,7 +140,7 @@ void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { * Object allocations remain valid. */ void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { - DEBUGLOG(3, "wksp: clearing!"); + DEBUGLOG(4, "cwksp: clearing!"); ws->tableEnd = ws->objectEnd; ws->allocStart = ws->workspaceEnd; ws->allocFailed = 0; @@ -147,7 +150,7 @@ void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { } void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) { - DEBUGLOG(3, "wksp: init'ing with %zd bytes", size); + DEBUGLOG(4, "cwksp: init'ing workspace with %zd bytes", size); assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ ws->workspace = start; ws->workspaceEnd = (BYTE*)start + size; @@ -159,14 +162,14 @@ void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) { size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { void* workspace = ZSTD_malloc(size, customMem); - DEBUGLOG(3, "wksp: creating with %zd bytes", size); + DEBUGLOG(4, "cwksp: creating new workspace with %zd bytes", size); RETURN_ERROR_IF(workspace == NULL, memory_allocation); ZSTD_cwksp_init(ws, workspace, size); return 0; } void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { - DEBUGLOG(3, "wksp: freeing"); + DEBUGLOG(4, "cwksp: freeing workspace"); ZSTD_free(ws->workspace, customMem); ws->workspace = NULL; ws->workspaceEnd = NULL; From 8549ae9f1da6e01c90cbac674b5495a49467f7d9 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 3 Sep 2019 12:48:45 -0400 Subject: [PATCH 043/144] Hide Workspace Movement Behind Helper Function --- lib/compress/zstd_compress.c | 6 +++--- lib/compress/zstd_cwksp.c | 5 +++++ lib/compress/zstd_cwksp.h | 6 ++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index d2ff96b5..99c8a606 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -94,7 +94,7 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize) return NULL; } memset(cctx, 0, sizeof(ZSTD_CCtx)); - cctx->workspace = ws; + ZSTD_cwksp_move(&cctx->workspace, &ws); cctx->staticSize = workspaceSize; /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ @@ -3181,7 +3181,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); assert(cdict != NULL); - cdict->workspace = ws; + ZSTD_cwksp_move(&cdict->workspace, &ws); cdict->customMem = customMem; if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dictBuffer, dictSize, @@ -3257,7 +3257,7 @@ const ZSTD_CDict* ZSTD_initStaticCDict( ZSTD_cwksp_init(&ws, workspace, workspaceSize); cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); if (cdict == NULL) return NULL; - cdict->workspace = ws; + ZSTD_cwksp_move(&cdict->workspace, &ws); } DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u", diff --git a/lib/compress/zstd_cwksp.c b/lib/compress/zstd_cwksp.c index f8503975..0ff6bec0 100644 --- a/lib/compress/zstd_cwksp.c +++ b/lib/compress/zstd_cwksp.c @@ -176,6 +176,11 @@ void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { ZSTD_cwksp_clear(ws); } +void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) { + *dst = *src; + memset(src, 0, sizeof(ZSTD_cwksp)); +} + size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { return (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace; } diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 5abc97be..630ad3ba 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -177,6 +177,12 @@ size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem); void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem); +/** + * Moves the management of a workspace from one cwksp to another. The src cwksp + * is left in an invalid state (src must be re-init()'ed before its used again). + */ +void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src); + size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws); int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws); From 7d7b665c9041d6ee53e0aab73a487e7f61001064 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 3 Sep 2019 12:59:10 -0400 Subject: [PATCH 044/144] Pull Phase Advance Logic Out into Internal Function --- lib/compress/zstd_cwksp.c | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/lib/compress/zstd_cwksp.c b/lib/compress/zstd_cwksp.c index 0ff6bec0..36ba4bbe 100644 --- a/lib/compress/zstd_cwksp.c +++ b/lib/compress/zstd_cwksp.c @@ -23,15 +23,8 @@ size_t ZSTD_cwksp_align(size_t size, size_t const align) { return (size + mask) & ~mask; } -/** - * Internal function, use wrappers instead. - */ -static void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { - /* TODO(felixh): alignment */ - void* alloc = (BYTE *)ws->allocStart - bytes; - void* bottom = ws->tableEnd; - DEBUGLOG(4, "cwksp: reserving align %zd bytes, %zd bytes remaining", - bytes, (BYTE *)alloc - (BYTE *)bottom); +static void ZSTD_cwksp_internal_advance_phase( + ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) { assert(phase >= ws->phase); if (phase > ws->phase) { if (ws->phase < ZSTD_cwksp_alloc_buffers && @@ -46,10 +39,23 @@ static void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwks * workspace is too large, and specifically when it is too large * by a larger margin than the space that will be consumed. */ /* TODO: cleaner, compiler warning friendly way to do this??? */ - alloc = (BYTE*)alloc - ((size_t)alloc & (sizeof(U32)-1)); + ws->allocStart = (BYTE*)ws->allocStart - ((size_t)ws->allocStart & (sizeof(U32)-1)); } ws->phase = phase; } +} + +/** + * Internal function, use wrappers instead. + */ +static void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { + /* TODO(felixh): alignment */ + void* alloc; + void* bottom = ws->tableEnd; + ZSTD_cwksp_internal_advance_phase(ws, phase); + alloc = (BYTE *)ws->allocStart - bytes; + DEBUGLOG(4, "cwksp: reserving align %zd bytes, %zd bytes remaining", + bytes, ZSTD_cwksp_available_space(ws) - bytes); assert(alloc >= bottom); if (alloc < bottom) { ws->allocFailed = 1; @@ -87,13 +93,7 @@ void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { DEBUGLOG(4, "cwksp: reserving table %zd bytes, %zd bytes remaining", bytes, ZSTD_cwksp_available_space(ws) - bytes); assert((bytes & (sizeof(U32)-1)) == 0); - assert(phase >= ws->phase); - if (phase > ws->phase) { - if (ws->phase <= ZSTD_cwksp_alloc_buffers) { - - } - ws->phase = phase; - } + ZSTD_cwksp_internal_advance_phase(ws, phase); assert(end <= top); if (end > top) { DEBUGLOG(4, "cwksp: object alloc failed!"); @@ -116,6 +116,7 @@ void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); assert(((size_t)start & (sizeof(void*)-1)) == 0); assert((bytes & (sizeof(void*)-1)) == 0); + /* we must be in the first phase, no advance is possible */ if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { DEBUGLOG(4, "cwksp: object alloc failed!"); ws->allocFailed = 1; From c60e1c3be5d4dd73040f5b16c0672bf758186503 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 3 Sep 2019 13:00:59 -0400 Subject: [PATCH 045/144] Nit --- lib/compress/zstd_cwksp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_cwksp.c b/lib/compress/zstd_cwksp.c index 36ba4bbe..a9471639 100644 --- a/lib/compress/zstd_cwksp.c +++ b/lib/compress/zstd_cwksp.c @@ -112,7 +112,7 @@ void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { void* start = ws->objectEnd; void* end = (BYTE*)start + roundedBytes; DEBUGLOG(4, - "cwksp: reserving %zd bytes object (rounded to %zd), %zd bytes remaining", + "cwksp: reserving object %zd bytes (rounded to %zd), %zd bytes remaining", bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); assert(((size_t)start & (sizeof(void*)-1)) == 0); assert((bytes & (sizeof(void*)-1)) == 0); From 1120e4d9624a6ccc6ae68530011a4dbb0548bda1 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 9 Sep 2019 14:04:39 -0400 Subject: [PATCH 046/144] Clean Up TODOs and Comments pt. II --- lib/compress/zstd_cwksp.c | 11 ++++----- lib/compress/zstd_cwksp.h | 49 ++++++++++++++++++++++++++------------- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/lib/compress/zstd_cwksp.c b/lib/compress/zstd_cwksp.c index a9471639..08765e40 100644 --- a/lib/compress/zstd_cwksp.c +++ b/lib/compress/zstd_cwksp.c @@ -45,16 +45,13 @@ static void ZSTD_cwksp_internal_advance_phase( } } -/** - * Internal function, use wrappers instead. - */ -static void* ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { - /* TODO(felixh): alignment */ +static void* ZSTD_cwksp_reserve_internal( + ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { void* alloc; void* bottom = ws->tableEnd; ZSTD_cwksp_internal_advance_phase(ws, phase); alloc = (BYTE *)ws->allocStart - bytes; - DEBUGLOG(4, "cwksp: reserving align %zd bytes, %zd bytes remaining", + DEBUGLOG(4, "cwksp: reserving %zd bytes, %zd bytes remaining", bytes, ZSTD_cwksp_available_space(ws) - bytes); assert(alloc >= bottom); if (alloc < bottom) { @@ -76,7 +73,7 @@ BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) { * Aligned on sizeof(unsigned). */ void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) { - assert((bytes & (sizeof(U32)-1)) == 0); // TODO ??? + assert((bytes & (sizeof(U32)-1)) == 0); return ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, sizeof(U32)), ZSTD_cwksp_alloc_aligned); } diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 630ad3ba..80d174f0 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -50,14 +50,25 @@ typedef enum { * is called the workspace. * * Several optimizations complicate that process of allocating memory ranges - * from this workspace for each datastructure: + * from this workspace for each internal datastructure: * - * - These different internal datastructures have different setup requirements. - * Some (e.g., the window buffer) don't care, and are happy to accept - * uninitialized memory. Others (e.g., the matchstate tables) can accept - * memory filled with unknown but bounded values (i.e., a memory area whose - * values are known to be constrained between 0 and some upper bound). If - * that constraint isn't known to be satisfied, the area has to be cleared. + * - These different internal datastructures have different setup requirements: + * + * - The static objects need to be cleared once and can then be trivially + * reused for each compression. + * + * - Various buffers don't need to be initialized at all--they are always + * written into before they're read. + * + * - The matchstate tables have a unique requirement that they don't need + * their memory to be totally cleared, but they do need the memory to have + * some bound, i.e., a guarantee that all values in the memory they've been + * allocated is less than some maximum value (which is the starting value + * for the indices that they will then use for compression). When this + * guarantee is provided to them, they can use the memory without any setup + * work. When it can't, they have to clear the area. + * + * - These buffers also have different alignment requirements. * * - We would like to reuse the objects in the workspace for multiple * compressions without having to perform any expensive reallocation or @@ -67,13 +78,16 @@ typedef enum { * multiple compressions **even when the compression parameters change** and * we need to resize some of the objects (where possible). * + * To attempt to manage this buffer, given these constraints, the ZSTD_cwksp + * abstraction was created. It works as follows: + * * Workspace Layout: * * [ ... workspace ... ] * [objects][tables ... ->] free space [<- ... aligned][<- ... buffers] * - * In order to accomplish this, the various objects that live in the workspace - * are divided into the following categories: + * The various objects that live in the workspace are divided into the + * following categories, and are allocated separately: * * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict, * so that literally everything fits in a single buffer. Note: if present, @@ -94,11 +108,11 @@ typedef enum { * uint32_t arrays, all of whose values are between 0 and (nextSrc - base). * Their sizes depend on the cparams. * - * - Aligned: these buffers are used for various purposes that don't require - * any initialization before they're used. + * - Aligned: these buffers are used for various purposes that require 4 byte + * alignment, but don't require any initialization before they're used. * - * - Uninitialized memory: these buffers are used for various purposes that - * don't require any initialization before they're used. This means they can + * - Buffers: these buffers are used for various purposes that don't require + * any alignment or initialization before they're used. This means they can * be moved around at no cost for a new compression. * * Allocating Memory: @@ -111,9 +125,7 @@ typedef enum { * 3. Aligned * 4. Tables * - * Reusing Table Space: - * - * TODO(felixh): ... + * Attempts to reserve objects of different types out of order will fail. */ typedef struct { void* workspace; @@ -171,6 +183,11 @@ void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws); */ void ZSTD_cwksp_clear(ZSTD_cwksp* ws); +/** + * The provided workspace takes ownership of the buffer [start, start+size). + * Any existing values in the workspace are ignored (the previously managed + * buffer, if present, must be separately freed). + */ void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size); size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem); From 0b25ab2202fec7db59ccab8d73dc1a5fc1d679c3 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Mon, 9 Sep 2019 11:54:43 -0700 Subject: [PATCH 047/144] Testing first block decompression cli --- tests/files/rle-first-block.zst | Bin 0 -> 45 bytes tests/golden/huffman-compressed-larger | Bin 0 -> 143 bytes tests/playTests.sh | 9 +++++++-- 3 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 tests/files/rle-first-block.zst create mode 100644 tests/golden/huffman-compressed-larger diff --git a/tests/files/rle-first-block.zst b/tests/files/rle-first-block.zst new file mode 100644 index 0000000000000000000000000000000000000000..fd067edd74ef9bab1dcf9af83baa7fee24f73287 GIT binary patch literal 45 acmdPcs{eNh1A_nq6CTVAl>2BW_7DJtPX%=V literal 0 HcmV?d00001 diff --git a/tests/golden/huffman-compressed-larger b/tests/golden/huffman-compressed-larger new file mode 100644 index 0000000000000000000000000000000000000000..f594f1ae9816a52054935aab96eec94c4ffe14b7 GIT binary patch literal 143 zcmV;A0C4{(wJ-eyP!$9KW`-Ueka7hx0Dvr|0RR9101)1^uIswqo=F4%0Du4hfEEB3 z02Tl|2JCX%cWtRD`Y@9(#KBM?3Vi8VgP*Qj(lfvmE1c0RaG879=~%<1Ug5Gim?; literal 0 HcmV?d00001 diff --git a/tests/playTests.sh b/tests/playTests.sh index 69387321..34f4dfcc 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -242,6 +242,11 @@ $ZSTD -f tmp && die "attempt to compress a non existing file" test -f tmp.zst # destination file should still be present rm tmp* +println "\n===> decompression only tests " +head -c 1M /dev/zero > tmp +$ZSTD -d -o tmp1 "$TESTDIR/files/rle-first-block.zst" +$DIFF -s tmp1 tmp +rm tmp* println "test : compress multiple files" println hello > tmp1 @@ -583,8 +588,8 @@ $ZSTD -t tmpSplit.* && die "bad file not detected !" println "\n===> golden files tests " -$ZSTD -t -r "$TESTDIR/files" -$ZSTD -c -r "$TESTDIR/files" | $ZSTD -t +$ZSTD -t -r "$TESTDIR/golden" +$ZSTD -c -r "$TESTDIR/golden" | $ZSTD -t println "\n===> benchmark mode tests " From 0a65a67901f6cc3d12f1b42f111680d0419a5c5c Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 9 Sep 2019 14:59:09 -0400 Subject: [PATCH 048/144] Shorten `&zc->workspace` -> `ws` in `ZSTD_resetCCtx_internal()` --- lib/compress/zstd_compress.c | 51 ++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 99c8a606..cbda17b8 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1440,6 +1440,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_compResetPolicy_e const crp, ZSTD_buffered_policy_e const zbuff) { + ZSTD_cwksp* const ws = &zc->workspace; DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u", (U32)pledgedSrcSize, params.cParams.windowLog); assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); @@ -1451,15 +1452,15 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->seqStore.maxNbSeq, zc->seqStore.maxNbLit, zbuff, pledgedSrcSize) ) { DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> consider continue mode"); - ZSTD_cwksp_bump_oversized_duration(&zc->workspace, 0); - if (!ZSTD_cwksp_check_wasteful(&zc->workspace, 0)) { + ZSTD_cwksp_bump_oversized_duration(ws, 0); + if (!ZSTD_cwksp_check_wasteful(ws, 0)) { DEBUGLOG(4, "continue mode confirmed (wLog1=%u, blockSize1=%zu)", zc->appliedParams.cParams.windowLog, zc->blockSize); if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) { /* prefer a reset, faster than a rescale */ FORWARD_IF_ERROR(ZSTD_reset_matchState( &zc->blockState.matchState, - &zc->workspace, + ws, ¶ms.cParams, crp, ZSTD_resetTarget_CCtx)); @@ -1502,8 +1503,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, tokenSpace + bufferSpace; - int const workspaceTooSmall = !ZSTD_cwksp_check_available(&zc->workspace, neededSpace); - int const workspaceWasteful = ZSTD_cwksp_check_wasteful(&zc->workspace, neededSpace); + int const workspaceTooSmall = !ZSTD_cwksp_check_available(ws, neededSpace); + int const workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace); DEBUGLOG(4, "Need %zuKB workspace, including %zuKB for match state, and %zuKB for buffers", neededSpace>>10, matchStateSize>>10, bufferSpace>>10); @@ -1511,28 +1512,28 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, if (workspaceTooSmall || workspaceWasteful) { DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB", - ZSTD_cwksp_sizeof(&zc->workspace) >> 10, + ZSTD_cwksp_sizeof(ws) >> 10, neededSpace >> 10); RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize"); - ZSTD_cwksp_free(&zc->workspace, zc->customMem); - FORWARD_IF_ERROR(ZSTD_cwksp_create(&zc->workspace, neededSpace, zc->customMem)); + ZSTD_cwksp_free(ws, zc->customMem); + FORWARD_IF_ERROR(ZSTD_cwksp_create(ws, neededSpace, zc->customMem)); DEBUGLOG(5, "reserving object space"); /* Statically sized space. * entropyWorkspace never moves, * though prev/next block swap places */ - assert(ZSTD_cwksp_check_available(&zc->workspace, 2 * sizeof(ZSTD_compressedBlockState_t))); - zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(&zc->workspace, sizeof(ZSTD_compressedBlockState_t)); + assert(ZSTD_cwksp_check_available(ws, 2 * sizeof(ZSTD_compressedBlockState_t))); + zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock"); - zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(&zc->workspace, sizeof(ZSTD_compressedBlockState_t)); + zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock"); - zc->entropyWorkspace = (U32*) ZSTD_cwksp_reserve_object(&zc->workspace, HUF_WORKSPACE_SIZE); + zc->entropyWorkspace = (U32*) ZSTD_cwksp_reserve_object(ws, HUF_WORKSPACE_SIZE); RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate entropyWorkspace"); } } - ZSTD_cwksp_clear(&zc->workspace); + ZSTD_cwksp_clear(ws); /* init params */ zc->appliedParams = params; @@ -1555,35 +1556,35 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, /* ZSTD_wildcopy() is used to copy into the literals buffer, * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. */ - zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(&zc->workspace, blockSize + WILDCOPY_OVERLENGTH); + zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(ws, blockSize + WILDCOPY_OVERLENGTH); zc->seqStore.maxNbLit = blockSize; /* buffers */ zc->inBuffSize = buffInSize; - zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(&zc->workspace, buffInSize); + zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize); zc->outBuffSize = buffOutSize; - zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(&zc->workspace, buffOutSize); + zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffOutSize); /* ldm bucketOffsets table */ if (params.ldmParams.enableLdm) { size_t const ldmBucketSize = ((size_t)1) << (params.ldmParams.hashLog - params.ldmParams.bucketSizeLog); - zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(&zc->workspace, ldmBucketSize); + zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, ldmBucketSize); memset(zc->ldmState.bucketOffsets, 0, ldmBucketSize); } /* sequences storage */ ZSTD_referenceExternalSequences(zc, NULL, 0); zc->seqStore.maxNbSeq = maxNbSeq; - zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); - zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); - zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(&zc->workspace, maxNbSeq * sizeof(BYTE)); - zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(&zc->workspace, maxNbSeq * sizeof(seqDef)); + zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef)); FORWARD_IF_ERROR(ZSTD_reset_matchState( &zc->blockState.matchState, - &zc->workspace, + ws, ¶ms.cParams, crp, ZSTD_resetTarget_CCtx)); @@ -1591,16 +1592,16 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, /* initialize bucketOffsets table separately for pointer alignment */ if (params.ldmParams.enableLdm) { size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog; - zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(&zc->workspace, ldmHSize * sizeof(ldmEntry_t)); + zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t)); memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); - zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(&zc->workspace, maxNbLdmSeq * sizeof(rawSeq)); + zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq)); zc->maxNbLdmSequences = maxNbLdmSeq; memset(&zc->ldmState.window, 0, sizeof(zc->ldmState.window)); ZSTD_window_clear(&zc->ldmState.window); } - DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(&zc->workspace)); + DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws)); return 0; } From e6be4cf4eb16b6840b558fded58d834067bccfcc Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Mon, 9 Sep 2019 12:08:33 -0700 Subject: [PATCH 049/144] Changing test file directory names to be more descriptive --- .../huffman-compressed-larger | Bin .../rle-first-block.zst | Bin tests/golden/huffman-compressed-larger | Bin 143 -> 0 bytes tests/playTests.sh | 6 +++--- 4 files changed, 3 insertions(+), 3 deletions(-) rename tests/{files => golden-compression}/huffman-compressed-larger (100%) rename tests/{files => golden-decompression}/rle-first-block.zst (100%) delete mode 100644 tests/golden/huffman-compressed-larger diff --git a/tests/files/huffman-compressed-larger b/tests/golden-compression/huffman-compressed-larger similarity index 100% rename from tests/files/huffman-compressed-larger rename to tests/golden-compression/huffman-compressed-larger diff --git a/tests/files/rle-first-block.zst b/tests/golden-decompression/rle-first-block.zst similarity index 100% rename from tests/files/rle-first-block.zst rename to tests/golden-decompression/rle-first-block.zst diff --git a/tests/golden/huffman-compressed-larger b/tests/golden/huffman-compressed-larger deleted file mode 100644 index f594f1ae9816a52054935aab96eec94c4ffe14b7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 143 zcmV;A0C4{(wJ-eyP!$9KW`-Ueka7hx0Dvr|0RR9101)1^uIswqo=F4%0Du4hfEEB3 z02Tl|2JCX%cWtRD`Y@9(#KBM?3Vi8VgP*Qj(lfvmE1c0RaG879=~%<1Ug5Gim?; diff --git a/tests/playTests.sh b/tests/playTests.sh index 34f4dfcc..5d9373a0 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -244,7 +244,7 @@ rm tmp* println "\n===> decompression only tests " head -c 1M /dev/zero > tmp -$ZSTD -d -o tmp1 "$TESTDIR/files/rle-first-block.zst" +$ZSTD -d -o tmp1 "$TESTDIR/golden-decompression/rle-first-block.zst" $DIFF -s tmp1 tmp rm tmp* @@ -588,8 +588,8 @@ $ZSTD -t tmpSplit.* && die "bad file not detected !" println "\n===> golden files tests " -$ZSTD -t -r "$TESTDIR/golden" -$ZSTD -c -r "$TESTDIR/golden" | $ZSTD -t +$ZSTD -t -r "$TESTDIR/golden-compression" +$ZSTD -c -r "$TESTDIR/golden-compression" | $ZSTD -t println "\n===> benchmark mode tests " From e3703825a843b04be0aff3ff38336ee5188ea84c Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 9 Sep 2019 15:12:14 -0400 Subject: [PATCH 050/144] Fix workspaceTooSmall Calculation --- lib/compress/zstd_compress.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index cbda17b8..7adcb7c1 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1488,13 +1488,15 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize); /* Check if workspace is large enough, alloc a new one if needed */ - { size_t const entropySpace = HUF_WORKSPACE_SIZE; + { size_t const cctxSpace = zc->staticSize ? sizeof(ZSTD_CCtx) : 0; + size_t const entropySpace = HUF_WORKSPACE_SIZE; size_t const blockStateSpace = 2 * sizeof(ZSTD_compressedBlockState_t); size_t const bufferSpace = buffInSize + buffOutSize; size_t const ldmSpace = ZSTD_ldm_getTableSize(params.ldmParams); size_t const ldmSeqSpace = maxNbLdmSeq * sizeof(rawSeq); size_t const neededSpace = + cctxSpace + entropySpace + blockStateSpace + ldmSpace + @@ -1503,7 +1505,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, tokenSpace + bufferSpace; - int const workspaceTooSmall = !ZSTD_cwksp_check_available(ws, neededSpace); + int const workspaceTooSmall = ZSTD_cwksp_sizeof(ws) < neededSpace; int const workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace); DEBUGLOG(4, "Need %zuKB workspace, including %zuKB for match state, and %zuKB for buffers", From eb6f69d97838add9d713140999900acdcb8b4c9b Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 9 Sep 2019 16:45:17 -0400 Subject: [PATCH 051/144] Fix sizeof_CCtx and sizeof_CDict Calculations for Statically Init'ed Objects --- lib/compress/zstd_compress.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 7adcb7c1..c8b031f8 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -166,7 +166,9 @@ static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) { if (cctx==NULL) return 0; /* support sizeof on NULL */ - return sizeof(*cctx) + ZSTD_cwksp_sizeof(&cctx->workspace) + /* cctx may be in the workspace */ + return (cctx->workspace.workspace == cctx ? 0 : sizeof(*cctx)) + + ZSTD_cwksp_sizeof(&cctx->workspace) + ZSTD_sizeof_localDict(cctx->localDict) + ZSTD_sizeof_mtctx(cctx); } @@ -3103,7 +3105,9 @@ size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) { if (cdict==NULL) return 0; /* support sizeof on NULL */ DEBUGLOG(5, "sizeof(*cdict) : %u", (unsigned)sizeof(*cdict)); - return ZSTD_cwksp_sizeof(&cdict->workspace) + sizeof(*cdict); + /* cdict may be in the workspace */ + return (cdict->workspace.workspace == cdict ? 0 : sizeof(*cdict)) + + ZSTD_cwksp_sizeof(&cdict->workspace); } static size_t ZSTD_initCDict_internal( From 0db3ffe7eea3836d3d54c876281a8e9533d5ea22 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 3 Sep 2019 13:13:16 -0400 Subject: [PATCH 052/144] Forward resetCCtx Errors when Using CDict --- lib/compress/zstd_compress.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index c8b031f8..2cf508f3 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1665,8 +1665,8 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, * has its own tables. */ params.cParams = ZSTD_adjustCParams_internal(*cdict_cParams, pledgedSrcSize, 0); params.cParams.windowLog = windowLog; - ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, - ZSTDcrp_continue, zbuff); + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + ZSTDcrp_continue, zbuff)); assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); } @@ -1714,8 +1714,8 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, /* Copy only compression parameters related to tables. */ params.cParams = *cdict_cParams; params.cParams.windowLog = windowLog; - ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, - ZSTDcrp_noMemset, zbuff); + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + ZSTDcrp_noMemset, zbuff)); assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog); assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog); From 3932fcfebc077876bf57eab7d7ee2f1a542c4303 Mon Sep 17 00:00:00 2001 From: Dario Pavlovic Date: Mon, 9 Sep 2019 15:39:04 -0700 Subject: [PATCH 053/144] Fixing issues with double usage of data. --- tests/fuzz/simple_decompress.c | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/tests/fuzz/simple_decompress.c b/tests/fuzz/simple_decompress.c index 56ebb93e..803f7f86 100644 --- a/tests/fuzz/simple_decompress.c +++ b/tests/fuzz/simple_decompress.c @@ -23,23 +23,26 @@ static ZSTD_DCtx *dctx = NULL; int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); - FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); - int i; - if (!dctx) { - dctx = ZSTD_createDCtx(); - FUZZ_ASSERT(dctx); - } - /* Run it 10 times over 10 output sizes. Reuse the context. */ - for (i = 0; i < 10; ++i) { - size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 2 * size); - void* rBuf = malloc(bufSize); - FUZZ_ASSERT(rBuf); - ZSTD_decompressDCtx(dctx, rBuf, bufSize, src, size); - free(rBuf); - } + int i; + if (!dctx) { + dctx = ZSTD_createDCtx(); + FUZZ_ASSERT(dctx); + } - FUZZ_dataProducer_free(producer); + size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 2 * size); + void* rBuf = malloc(bufSize); + FUZZ_ASSERT(rBuf); + + /* Restrict to remaining data. If we run out of data while generating params, + we should still continue and let decompression happen on empty data. */ + size = FUZZ_dataProducer_remainingBytes(producer); + + ZSTD_decompressDCtx(dctx, rBuf, bufSize, src, size); + free(rBuf); + + FUZZ_dataProducer_free(producer); #ifndef STATEFUL_FUZZING ZSTD_freeDCtx(dctx); dctx = NULL; From 91bf1babd1f3be9607c5bb917fe2442c20a0c2ba Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 9 Sep 2019 18:30:53 -0400 Subject: [PATCH 054/144] Inline Workspace Functions --- lib/compress/zstd_cwksp.c | 200 +------------------------------------- lib/compress/zstd_cwksp.h | 182 ++++++++++++++++++++++++++++++---- 2 files changed, 163 insertions(+), 219 deletions(-) diff --git a/lib/compress/zstd_cwksp.c b/lib/compress/zstd_cwksp.c index 08765e40..a757910d 100644 --- a/lib/compress/zstd_cwksp.c +++ b/lib/compress/zstd_cwksp.c @@ -14,205 +14,7 @@ extern "C" { #endif -/** - * Align must be a power of 2. - */ -size_t ZSTD_cwksp_align(size_t size, size_t const align) { - size_t const mask = align - 1; - assert((align & mask) == 0); - return (size + mask) & ~mask; -} - -static void ZSTD_cwksp_internal_advance_phase( - ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) { - assert(phase >= ws->phase); - if (phase > ws->phase) { - if (ws->phase < ZSTD_cwksp_alloc_buffers && - phase >= ZSTD_cwksp_alloc_buffers) { - } - if (ws->phase < ZSTD_cwksp_alloc_aligned && - phase >= ZSTD_cwksp_alloc_aligned) { - /* If unaligned allocations down from a too-large top have left us - * unaligned, we need to realign our alloc ptr. Technically, this - * can consume space that is unaccounted for in the neededSpace - * calculation. However, I believe this can only happen when the - * workspace is too large, and specifically when it is too large - * by a larger margin than the space that will be consumed. */ - /* TODO: cleaner, compiler warning friendly way to do this??? */ - ws->allocStart = (BYTE*)ws->allocStart - ((size_t)ws->allocStart & (sizeof(U32)-1)); - } - ws->phase = phase; - } -} - -static void* ZSTD_cwksp_reserve_internal( - ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { - void* alloc; - void* bottom = ws->tableEnd; - ZSTD_cwksp_internal_advance_phase(ws, phase); - alloc = (BYTE *)ws->allocStart - bytes; - DEBUGLOG(4, "cwksp: reserving %zd bytes, %zd bytes remaining", - bytes, ZSTD_cwksp_available_space(ws) - bytes); - assert(alloc >= bottom); - if (alloc < bottom) { - ws->allocFailed = 1; - return NULL; - } - ws->allocStart = alloc; - return alloc; -} - -/** - * Unaligned. - */ -BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) { - return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers); -} - -/** - * Aligned on sizeof(unsigned). - */ -void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) { - assert((bytes & (sizeof(U32)-1)) == 0); - return ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, sizeof(U32)), ZSTD_cwksp_alloc_aligned); -} - -/** - * Aligned on sizeof(unsigned). These buffers have the special property that - * their values remain constrained, allowing us to re-use them without - * memset()-ing them. - */ -void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { - const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned; - void* alloc = ws->tableEnd; - void* end = (BYTE *)alloc + bytes; - void* top = ws->allocStart; - DEBUGLOG(4, "cwksp: reserving table %zd bytes, %zd bytes remaining", - bytes, ZSTD_cwksp_available_space(ws) - bytes); - assert((bytes & (sizeof(U32)-1)) == 0); - ZSTD_cwksp_internal_advance_phase(ws, phase); - assert(end <= top); - if (end > top) { - DEBUGLOG(4, "cwksp: object alloc failed!"); - ws->allocFailed = 1; - return NULL; - } - ws->tableEnd = end; - return alloc; -} - -/** - * Aligned on sizeof(void*). - */ -void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { - size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); - void* start = ws->objectEnd; - void* end = (BYTE*)start + roundedBytes; - DEBUGLOG(4, - "cwksp: reserving object %zd bytes (rounded to %zd), %zd bytes remaining", - bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); - assert(((size_t)start & (sizeof(void*)-1)) == 0); - assert((bytes & (sizeof(void*)-1)) == 0); - /* we must be in the first phase, no advance is possible */ - if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { - DEBUGLOG(4, "cwksp: object alloc failed!"); - ws->allocFailed = 1; - return NULL; - } - ws->objectEnd = end; - ws->tableEnd = end; - return start; -} - -/** - * Invalidates table allocations. - * All other allocations remain valid. - */ -void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { - DEBUGLOG(4, "cwksp: clearing tables!"); - ws->tableEnd = ws->objectEnd; -} - -/** - * Invalidates all buffer, aligned, and table allocations. - * Object allocations remain valid. - */ -void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { - DEBUGLOG(4, "cwksp: clearing!"); - ws->tableEnd = ws->objectEnd; - ws->allocStart = ws->workspaceEnd; - ws->allocFailed = 0; - if (ws->phase > ZSTD_cwksp_alloc_buffers) { - ws->phase = ZSTD_cwksp_alloc_buffers; - } -} - -void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) { - DEBUGLOG(4, "cwksp: init'ing workspace with %zd bytes", size); - assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ - ws->workspace = start; - ws->workspaceEnd = (BYTE*)start + size; - ws->objectEnd = ws->workspace; - ws->phase = ZSTD_cwksp_alloc_objects; - ZSTD_cwksp_clear(ws); - ws->workspaceOversizedDuration = 0; -} - -size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { - void* workspace = ZSTD_malloc(size, customMem); - DEBUGLOG(4, "cwksp: creating new workspace with %zd bytes", size); - RETURN_ERROR_IF(workspace == NULL, memory_allocation); - ZSTD_cwksp_init(ws, workspace, size); - return 0; -} - -void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { - DEBUGLOG(4, "cwksp: freeing workspace"); - ZSTD_free(ws->workspace, customMem); - ws->workspace = NULL; - ws->workspaceEnd = NULL; - ZSTD_cwksp_clear(ws); -} - -void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) { - *dst = *src; - memset(src, 0, sizeof(ZSTD_cwksp)); -} - -size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { - return (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace; -} - -int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { - return ws->allocFailed; -} - -size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { - return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); -} - -void ZSTD_cwksp_bump_oversized_duration( - ZSTD_cwksp* ws, size_t additionalNeededSpace) { - if (ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)) { - ws->workspaceOversizedDuration++; - } else { - ws->workspaceOversizedDuration = 0; - } -} - -int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeededSpace) { - return ZSTD_cwksp_available_space(ws) >= additionalNeededSpace; -} - -int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace) { - return ZSTD_cwksp_check_available( - ws, additionalNeededSpace * ZSTD_WORKSPACETOOLARGE_FACTOR); -} - -int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace) { - return ZSTD_cwksp_check_too_large(ws, additionalNeededSpace) - && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; -} +/* all functions have been inlined... */ #if defined (__cplusplus) } diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 80d174f0..cd44d8ad 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -147,76 +147,218 @@ typedef struct { /** * Align must be a power of 2. */ -size_t ZSTD_cwksp_align(size_t size, size_t const align); +MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) { + size_t const mask = align - 1; + assert((align & mask) == 0); + return (size + mask) & ~mask; +} + +MEM_STATIC void ZSTD_cwksp_internal_advance_phase( + ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) { + assert(phase >= ws->phase); + if (phase > ws->phase) { + if (ws->phase < ZSTD_cwksp_alloc_buffers && + phase >= ZSTD_cwksp_alloc_buffers) { + } + if (ws->phase < ZSTD_cwksp_alloc_aligned && + phase >= ZSTD_cwksp_alloc_aligned) { + /* If unaligned allocations down from a too-large top have left us + * unaligned, we need to realign our alloc ptr. Technically, this + * can consume space that is unaccounted for in the neededSpace + * calculation. However, I believe this can only happen when the + * workspace is too large, and specifically when it is too large + * by a larger margin than the space that will be consumed. */ + /* TODO: cleaner, compiler warning friendly way to do this??? */ + ws->allocStart = (BYTE*)ws->allocStart - ((size_t)ws->allocStart & (sizeof(U32)-1)); + } + ws->phase = phase; + } +} /** - * Unaligned. + * Internal function. Do not use directly. */ -BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes); +MEM_STATIC void* ZSTD_cwksp_reserve_internal( + ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { + void* alloc; + void* bottom = ws->tableEnd; + ZSTD_cwksp_internal_advance_phase(ws, phase); + alloc = (BYTE *)ws->allocStart - bytes; + DEBUGLOG(4, "cwksp: reserving %zd bytes, %zd bytes remaining", + bytes, ZSTD_cwksp_available_space(ws) - bytes); + assert(alloc >= bottom); + if (alloc < bottom) { + ws->allocFailed = 1; + return NULL; + } + ws->allocStart = alloc; + return alloc; +} /** - * Aligned on sizeof(unsigned). + * Reserves and returns unaligned memory. */ -void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes); +MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) { + return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers); +} + +/** + * Reserves and returns memory sized on and aligned on sizeof(unsigned). + */ +MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) { + assert((bytes & (sizeof(U32)-1)) == 0); + return ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, sizeof(U32)), ZSTD_cwksp_alloc_aligned); +} /** * Aligned on sizeof(unsigned). These buffers have the special property that * their values remain constrained, allowing us to re-use them without * memset()-ing them. */ -void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes); +MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { + const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned; + void* alloc = ws->tableEnd; + void* end = (BYTE *)alloc + bytes; + void* top = ws->allocStart; + DEBUGLOG(4, "cwksp: reserving table %zd bytes, %zd bytes remaining", + bytes, ZSTD_cwksp_available_space(ws) - bytes); + assert((bytes & (sizeof(U32)-1)) == 0); + ZSTD_cwksp_internal_advance_phase(ws, phase); + assert(end <= top); + if (end > top) { + DEBUGLOG(4, "cwksp: object alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->tableEnd = end; + return alloc; +} /** * Aligned on sizeof(void*). */ -void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes); +MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { + size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); + void* start = ws->objectEnd; + void* end = (BYTE*)start + roundedBytes; + DEBUGLOG(4, + "cwksp: reserving object %zd bytes (rounded to %zd), %zd bytes remaining", + bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); + assert(((size_t)start & (sizeof(void*)-1)) == 0); + assert((bytes & (sizeof(void*)-1)) == 0); + /* we must be in the first phase, no advance is possible */ + if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { + DEBUGLOG(4, "cwksp: object alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->objectEnd = end; + ws->tableEnd = end; + return start; +} /** * Invalidates table allocations. * All other allocations remain valid. */ -void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws); +MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: clearing tables!"); + ws->tableEnd = ws->objectEnd; +} /** * Invalidates all buffer, aligned, and table allocations. * Object allocations remain valid. */ -void ZSTD_cwksp_clear(ZSTD_cwksp* ws); +MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: clearing!"); + ws->tableEnd = ws->objectEnd; + ws->allocStart = ws->workspaceEnd; + ws->allocFailed = 0; + if (ws->phase > ZSTD_cwksp_alloc_buffers) { + ws->phase = ZSTD_cwksp_alloc_buffers; + } +} /** * The provided workspace takes ownership of the buffer [start, start+size). * Any existing values in the workspace are ignored (the previously managed * buffer, if present, must be separately freed). */ -void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size); +MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) { + DEBUGLOG(4, "cwksp: init'ing workspace with %zd bytes", size); + assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ + ws->workspace = start; + ws->workspaceEnd = (BYTE*)start + size; + ws->objectEnd = ws->workspace; + ws->phase = ZSTD_cwksp_alloc_objects; + ZSTD_cwksp_clear(ws); + ws->workspaceOversizedDuration = 0; +} -size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem); +MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { + void* workspace = ZSTD_malloc(size, customMem); + DEBUGLOG(4, "cwksp: creating new workspace with %zd bytes", size); + RETURN_ERROR_IF(workspace == NULL, memory_allocation); + ZSTD_cwksp_init(ws, workspace, size); + return 0; +} -void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem); +MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { + DEBUGLOG(4, "cwksp: freeing workspace"); + ZSTD_free(ws->workspace, customMem); + ws->workspace = NULL; + ws->workspaceEnd = NULL; + ZSTD_cwksp_clear(ws); +} /** * Moves the management of a workspace from one cwksp to another. The src cwksp * is left in an invalid state (src must be re-init()'ed before its used again). */ -void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src); +MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) { + *dst = *src; + memset(src, 0, sizeof(ZSTD_cwksp)); +} -size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws); +MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { + return (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace; +} -int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws); +MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { + return ws->allocFailed; +} /*-************************************* * Functions Checking Free Space ***************************************/ -size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws); +MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); +} -int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeededSpace); +MEM_STATIC int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_available_space(ws) >= additionalNeededSpace; +} -int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace); +MEM_STATIC int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_available( + ws, additionalNeededSpace * ZSTD_WORKSPACETOOLARGE_FACTOR); +} -int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace); +MEM_STATIC int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_too_large(ws, additionalNeededSpace) + && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} -void ZSTD_cwksp_bump_oversized_duration(ZSTD_cwksp* ws, size_t additionalNeededSpace); +MEM_STATIC void ZSTD_cwksp_bump_oversized_duration( + ZSTD_cwksp* ws, size_t additionalNeededSpace) { + if (ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)) { + ws->workspaceOversizedDuration++; + } else { + ws->workspaceOversizedDuration = 0; + } +} #if defined (__cplusplus) } From 81208fd7c2bdf7ef266ce95a7ba20e5475e69aa5 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 9 Sep 2019 19:10:09 -0400 Subject: [PATCH 055/144] Forward Declare `ZSTD_cwksp_available_space` to Fix Build --- lib/compress/zstd_cwksp.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index cd44d8ad..76ace25e 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -144,6 +144,8 @@ typedef struct { * Functions ***************************************/ +MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws); + /** * Align must be a power of 2. */ From ea1ad123da7f760cac8d57bbaf81d367af17aebb Mon Sep 17 00:00:00 2001 From: Dario Pavlovic Date: Mon, 9 Sep 2019 16:13:24 -0700 Subject: [PATCH 056/144] Addressing nits --- tests/fuzz/fuzz_data_producer.h | 4 ++++ tests/fuzz/simple_decompress.c | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/fuzz/fuzz_data_producer.h b/tests/fuzz/fuzz_data_producer.h index 4a12e130..4fcf6fd4 100644 --- a/tests/fuzz/fuzz_data_producer.h +++ b/tests/fuzz/fuzz_data_producer.h @@ -9,6 +9,10 @@ /** * Helper APIs for generating random data from input data stream. + The producer reads bytes from the end of the input and appends them together + to generate a random number in the requested range. If it runs out of input + data, it will keep returning the same value (min) over and over again. + */ #ifndef FUZZ_DATA_PRODUCER_H diff --git a/tests/fuzz/simple_decompress.c b/tests/fuzz/simple_decompress.c index 803f7f86..a68813ee 100644 --- a/tests/fuzz/simple_decompress.c +++ b/tests/fuzz/simple_decompress.c @@ -31,8 +31,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) FUZZ_ASSERT(dctx); } - size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 2 * size); - void* rBuf = malloc(bufSize); + size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 10 * size); + void *rBuf = malloc(bufSize); FUZZ_ASSERT(rBuf); /* Restrict to remaining data. If we run out of data while generating params, From 9e7bb55e14b112e514a0e8d5ac7f19fd64f353c1 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Mon, 9 Sep 2019 20:04:46 -0700 Subject: [PATCH 057/144] Addressing comments --- lib/compress/zstd_compress.c | 156 +++++++++++++------------- lib/compress/zstd_compress_internal.h | 2 +- lib/zstd.h | 4 +- tests/fuzzer.c | 4 +- 4 files changed, 80 insertions(+), 86 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index f8588b34..56da1664 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -13,7 +13,6 @@ ***************************************/ #include /* INT_MAX */ #include /* memset */ -#include #include "cpu.h" #include "mem.h" #include "hist.h" /* HIST_countFast_wksp */ @@ -2265,6 +2264,77 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) return ZSTDbss_compress; } +static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) +{ + const seqStore_t* seqStore = ZSTD_getSeqStore(zc); + const seqDef* seqs = seqStore->sequencesStart; + size_t seqsSize = seqStore->sequences - seqs; + + ZSTD_Sequence* outSeqs = &zc->seqCollector.seqStart[zc->seqCollector.seqIndex]; + size_t i; size_t position; int repIdx; + + assert(zc->seqCollector.seqIndex + 1 < zc->seqCollector.maxSequences); + for (i = 0, position = 0; i < seqsSize; ++i) { + outSeqs[i].offset = seqs[i].offset; + outSeqs[i].litLength = seqs[i].litLength; + outSeqs[i].matchLength = seqs[i].matchLength + MINMATCH; + + if (i == seqStore->longLengthPos) { + if (seqStore->longLengthID == 1) { + outSeqs[i].litLength += 0x10000; + } else if (seqStore->longLengthID == 2) { + outSeqs[i].matchLength += 0x10000; + } + } + + if (outSeqs[i].offset <= ZSTD_REP_NUM) { + outSeqs[i].rep = 1; + repIdx = i - outSeqs[i].offset; + + if (repIdx >= 0) { + outSeqs[i].offset = outSeqs[repIdx].offset; + } + + if (repIdx == -1) { + outSeqs[i].offset = 1; + } else if (repIdx == -2) { + outSeqs[i].offset = 4; + } else if (repIdx == -3) { + outSeqs[i].offset = 8; + } + } else { + outSeqs[i].offset -= ZSTD_REP_NUM; + } + + position += outSeqs[i].litLength; + outSeqs[i].matchPos = position; + position += outSeqs[i].matchLength; + } + zc->seqCollector.seqIndex += seqsSize; +} + +/* We call compress2() and collect sequences after each block + * compression. The function stores the ZSTD_Sequences in outSeqs + * and returns the number of collected sequences from all blocks. + */ +size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, + size_t outSeqsSize, const void* src, size_t srcSize) +{ + const size_t dstCapacity = ZSTD_compressBound(srcSize * sizeof(void*)); + void* dst = ZSTD_malloc(dstCapacity, ZSTD_defaultCMem); + + SeqCollector seqCollector; + seqCollector.collectSequences = 1; + seqCollector.seqStart = outSeqs; + seqCollector.seqIndex = 0; + seqCollector.maxSequences = outSeqsSize; + zc->seqCollector = seqCollector; + + ZSTD_compress2(zc, dst, dstCapacity, src, srcSize); + ZSTD_free(dst, ZSTD_defaultCMem); + return zc->seqCollector.seqIndex; +} + static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize) @@ -2288,6 +2358,10 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, zc->entropyWorkspace, HUF_WORKSPACE_SIZE /* statically allocated in resetCCtx */, zc->bmi2); + if (zc->seqCollector.collectSequences) { + ZSTD_copyBlockSequences(zc); + } + out: if (!ZSTD_isError(cSize) && cSize != 0) { /* confirm repcodes and entropy tables when emitting a compressed block */ @@ -2324,82 +2398,6 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, ZSTD_CCtx_params } } -static void ZSTD_copyBlockSequences(const seqStore_t* seqStore, seqDef* seqs, - ZSTD_Sequence* outSeqs, size_t seqsSize) -{ - size_t i; size_t position; int repIdx; - for (i = 0, position = 0; i < seqsSize; ++i) { - outSeqs[i].offset = seqs[i].offset; - outSeqs[i].litLength = seqs[i].litLength; - outSeqs[i].matchLength = seqs[i].matchLength + 3 /* min match */; - - if (i == seqStore->longLengthPos) { - if (seqStore->longLengthID == 1) { - outSeqs[i].litLength += 0x10000; - } else if (seqStore->longLengthID == 2) { - outSeqs[i].matchLength += 0x10000; - } - } - - if (outSeqs[i].offset <= 3 /* num reps */) { - outSeqs[i].rep = 1; - repIdx = i - outSeqs[i].offset; - - if (repIdx >= 0) { - outSeqs[i].offset = outSeqs[repIdx].offset; - } - - if (repIdx == -1) { - outSeqs[i].offset = 1; - } else if (repIdx == -2) { - outSeqs[i].offset = 4; - } else if (repIdx == -3) { - outSeqs[i].offset = 8; - } - } else { - outSeqs[i].offset -= 3 /* num reps */; - } - - position += outSeqs[i].litLength; - outSeqs[i].matchPos = position; - position += outSeqs[i].matchLength; - } -} - -static void ZSTD_getBlockSequences(ZSTD_CCtx* cctx, const seqStore_t* seqStore) -{ - size_t seqsSize = seqStore->sequences - seqStore->sequencesStart; - - assert(cctx->seqCollector.maxSequences > - (cctx->seqCollector.seqCurrent - cctx->seqCollector.seqStart) + seqsSize); - - ZSTD_copyBlockSequences(seqStore, seqStore->sequencesStart, - cctx->seqCollector.seqCurrent, seqsSize); - cctx->seqCollector.seqCurrent += seqsSize; -} - -size_t ZSTD_getSequences(ZSTD_CCtx* zc, const void* src, - size_t srcSize, ZSTD_Sequence* outSeqs, size_t outSeqsSize, - int level) -{ - size_t dstCapacity = ZSTD_compressBound(srcSize * sizeof(void*)); - void* dst = malloc(dstCapacity); - size_t seqsSize; - - SeqCollector seqCollector; - seqCollector.collectSequences = 1; - seqCollector.seqStart = outSeqs; - seqCollector.seqCurrent = outSeqs; - seqCollector.maxSequences = outSeqsSize; - zc->seqCollector = seqCollector; - - ZSTD_compressCCtx(zc, dst, dstCapacity, src, srcSize, level); - seqsSize = zc->seqCollector.seqCurrent - zc->seqCollector.seqStart; - - free(dst); - return seqsSize; -} - /*! ZSTD_compress_frameChunk() : * Compress a chunk of data into one or multiple blocks. * All blocks will be terminated, all input will be consumed. @@ -2443,10 +2441,6 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); FORWARD_IF_ERROR(cSize); - if (cctx->seqCollector.collectSequences) { - ZSTD_getBlockSequences(cctx, ZSTD_getSeqStore(cctx)); - } - if (cSize == 0) { /* block is not compressible */ cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); FORWARD_IF_ERROR(cSize); diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index d40d5340..e3ed93eb 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -195,7 +195,7 @@ typedef struct { typedef struct { int collectSequences; ZSTD_Sequence* seqStart; - ZSTD_Sequence* seqCurrent; + size_t seqIndex; size_t maxSequences; } SeqCollector; diff --git a/lib/zstd.h b/lib/zstd.h index 782940ef..b2c66e75 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1218,8 +1218,8 @@ ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcS * or an error code (if srcSize is too small) */ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_getSequences(ZSTD_CCtx* zc, const void* src, - size_t srcSize, ZSTD_Sequence* outSeqs, size_t outSeqsSize, int level); +ZSTDLIB_API size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, + size_t outSeqsSize, const void* src, size_t srcSize); /*************************************** diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 09fe4695..fdf6960b 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1962,8 +1962,8 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences zeros : ", testNb++); memset(CNBuffer, 0, 1000000); - assert(ZSTD_getSequences(ZSTD_createCCtx(), CNBuffer, 1000000, - compressedBuffer, 1000000, 3) == 1000000 / 131071 + 1); + assert(ZSTD_getSequences(ZSTD_createCCtx(), compressedBuffer, 1000000, + CNBuffer, 1000000) == 1000000 / 131071 + 1); /* All zeroes test (test bug #137) */ #define ZEROESLENGTH 100 From caaf43b258f7ff40c3ddfa7dcfa7f4c0980bb6a6 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Tue, 10 Sep 2019 09:30:37 -0700 Subject: [PATCH 058/144] Using a number instead of M prefix in head call --- tests/playTests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index 5d9373a0..3b74ed22 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -243,7 +243,7 @@ test -f tmp.zst # destination file should still be present rm tmp* println "\n===> decompression only tests " -head -c 1M /dev/zero > tmp +head -c 1048576 /dev/zero > tmp $ZSTD -d -o tmp1 "$TESTDIR/golden-decompression/rle-first-block.zst" $DIFF -s tmp1 tmp rm tmp* From b9b9a1c8e9b2a0e27c485c691a46d695e8c90c95 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 10 Sep 2019 09:36:02 -0700 Subject: [PATCH 059/144] fix education decoder test when `zstd` in not installed on local system by allowing `ZSTD` variable to hold a custom location for the binary --- Makefile | 2 +- doc/educational_decoder/Makefile | 33 ++++++++++++++++++++++++-------- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index da3112e3..efb555c3 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,7 @@ test: MOREFLAGS += -g -DDEBUGLEVEL=$(DEBUGLEVEL) -Werror test: MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants $(MAKE) -C $(TESTDIR) $@ - $(MAKE) -C doc/educational_decoder test + ZSTD=../../programs/zstd $(MAKE) -C doc/educational_decoder test ## shortest: same as `make check` .PHONY: shortest diff --git a/doc/educational_decoder/Makefile b/doc/educational_decoder/Makefile index fc080375..b2ed9f33 100644 --- a/doc/educational_decoder/Makefile +++ b/doc/educational_decoder/Makefile @@ -1,10 +1,21 @@ +# ################################################################ +# Copyright (c) 2016-present, Yann Collet, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ################################################################ + +ZSTD ?= zstd # requires zstd installation on local system +DIFF ?= diff HARNESS_FILES=*.c MULTITHREAD_LDFLAGS = -pthread DEBUGFLAGS= -g -DZSTD_DEBUG=1 CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) -CFLAGS ?= -O3 +CFLAGS ?= -O2 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum \ -Wredundant-decls -Wstrict-prototypes -Wundef \ @@ -22,16 +33,22 @@ clean: @$(RM) -rf harness.dSYM test: harness - @zstd README.md -o tmp.zst + # + # Testing single-file decompression with educational decoder + # + @$(ZSTD) README.md -o tmp.zst @./harness tmp.zst tmp - @diff -s tmp README.md + @$(DIFF) -s tmp README.md @$(RM) -f tmp* - # present files for training multiple times, to reach minimum threshold - @zstd --train harness.c zstd_decompress.c zstd_decompress.h README.md \ + # + # Testing dictionary decompression with education decoder + # + # note : files are presented multiple for training, to reach minimum threshold + @$(ZSTD) --train harness.c zstd_decompress.c zstd_decompress.h README.md \ harness.c zstd_decompress.c zstd_decompress.h README.md \ harness.c zstd_decompress.c zstd_decompress.h README.md - @zstd -D dictionary README.md -o tmp.zst + @$(ZSTD) -D dictionary README.md -o tmp.zst @./harness tmp.zst tmp dictionary - @diff -s tmp README.md + @$(DIFF) -s tmp README.md @$(RM) -f tmp* dictionary - @make clean + @$(MAKE) clean From e3c582591855a2e70a8c88976a9f05c6b1a0ea8e Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Tue, 10 Sep 2019 10:06:02 -0700 Subject: [PATCH 060/144] Fizing litLength == 0 case --- lib/compress/zstd_compress.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 56da1664..83bcb52d 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2291,11 +2291,22 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) outSeqs[i].rep = 1; repIdx = i - outSeqs[i].offset; + /* Not first block */ if (repIdx >= 0) { - outSeqs[i].offset = outSeqs[repIdx].offset; - } - - if (repIdx == -1) { + /* Special case where litLength == 0 */ + if (outSeqs[i].litLength == 0) { + /* When the offset is 3 */ + if (outSeqs[i].offset > 2) { + outSeqs[i].offset = outSeqs[repIdx - 1].offset - 1; + /* When the offset is either 1 or 2 */ + } else { + outSeqs[i].offset = outSeqs[repIdx - 1].offset; + } + } else { + outSeqs[i].offset = outSeqs[repIdx].offset; + } + /* First block */ + } else if (repIdx == -1) { outSeqs[i].offset = 1; } else if (repIdx == -2) { outSeqs[i].offset = 4; From a9d373f0938a53f814a812032357f7b9756ed8f0 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 10 Sep 2019 16:03:13 -0400 Subject: [PATCH 061/144] Remove Empty lib/compress/zstd_cwksp.c --- build/VS2008/fullbench/fullbench.vcproj | 4 ---- build/VS2008/fuzzer/fuzzer.vcproj | 4 ---- build/VS2008/zstd/zstd.vcproj | 4 ---- build/VS2008/zstdlib/zstdlib.vcproj | 4 ---- build/VS2010/fullbench/fullbench.vcxproj | 1 - build/VS2010/fuzzer/fuzzer.vcxproj | 1 - build/VS2010/libzstd-dll/libzstd-dll.vcxproj | 1 - build/VS2010/libzstd/libzstd.vcxproj | 1 - build/VS2010/zstd/zstd.vcxproj | 1 - build/meson/lib/meson.build | 1 - lib/compress/zstd_cwksp.c | 21 -------------------- 11 files changed, 43 deletions(-) delete mode 100644 lib/compress/zstd_cwksp.c diff --git a/build/VS2008/fullbench/fullbench.vcproj b/build/VS2008/fullbench/fullbench.vcproj index ffb0a015..66ac8223 100644 --- a/build/VS2008/fullbench/fullbench.vcproj +++ b/build/VS2008/fullbench/fullbench.vcproj @@ -372,10 +372,6 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c" > - - diff --git a/build/VS2008/fuzzer/fuzzer.vcproj b/build/VS2008/fuzzer/fuzzer.vcproj index c3875ec7..a9008a67 100644 --- a/build/VS2008/fuzzer/fuzzer.vcproj +++ b/build/VS2008/fuzzer/fuzzer.vcproj @@ -420,10 +420,6 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c" > - - diff --git a/build/VS2008/zstd/zstd.vcproj b/build/VS2008/zstd/zstd.vcproj index 5b57dde5..4d75a9d3 100644 --- a/build/VS2008/zstd/zstd.vcproj +++ b/build/VS2008/zstd/zstd.vcproj @@ -432,10 +432,6 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c" > - - diff --git a/build/VS2008/zstdlib/zstdlib.vcproj b/build/VS2008/zstdlib/zstdlib.vcproj index e864b5fb..100a501a 100644 --- a/build/VS2008/zstdlib/zstdlib.vcproj +++ b/build/VS2008/zstdlib/zstdlib.vcproj @@ -404,10 +404,6 @@ RelativePath="..\..\..\lib\compress\zstd_compress_sequences.c" > - - diff --git a/build/VS2010/fullbench/fullbench.vcxproj b/build/VS2010/fullbench/fullbench.vcxproj index 65e7ff78..4597239b 100644 --- a/build/VS2010/fullbench/fullbench.vcxproj +++ b/build/VS2010/fullbench/fullbench.vcxproj @@ -169,7 +169,6 @@ - diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj index e9b88a8a..a6f136ee 100644 --- a/build/VS2010/fuzzer/fuzzer.vcxproj +++ b/build/VS2010/fuzzer/fuzzer.vcxproj @@ -169,7 +169,6 @@ - diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj index 4140879d..50c4e817 100644 --- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj +++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj @@ -33,7 +33,6 @@ - diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj index 02690cef..c4e82877 100644 --- a/build/VS2010/libzstd/libzstd.vcxproj +++ b/build/VS2010/libzstd/libzstd.vcxproj @@ -33,7 +33,6 @@ - diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj index 2a21ba4c..1058d229 100644 --- a/build/VS2010/zstd/zstd.vcxproj +++ b/build/VS2010/zstd/zstd.vcxproj @@ -34,7 +34,6 @@ - diff --git a/build/meson/lib/meson.build b/build/meson/lib/meson.build index 48f1e7cc..ef669327 100644 --- a/build/meson/lib/meson.build +++ b/build/meson/lib/meson.build @@ -30,7 +30,6 @@ libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_compress.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_compress_literals.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_compress_sequences.c'), - join_paths(zstd_rootdir, 'lib/compress/zstd_cwksp.c'), join_paths(zstd_rootdir, 'lib/compress/zstdmt_compress.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_fast.c'), join_paths(zstd_rootdir, 'lib/compress/zstd_double_fast.c'), diff --git a/lib/compress/zstd_cwksp.c b/lib/compress/zstd_cwksp.c deleted file mode 100644 index a757910d..00000000 --- a/lib/compress/zstd_cwksp.c +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#include "zstd_cwksp.h" - -#if defined (__cplusplus) -extern "C" { -#endif - -/* all functions have been inlined... */ - -#if defined (__cplusplus) -} -#endif From 47199480da07531a1e2b982483e08e23eff54970 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Tue, 10 Sep 2019 13:18:59 -0700 Subject: [PATCH 062/144] Cleaning up parsing per suggestion --- lib/compress/zstd_compress.c | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 83bcb52d..b4fdf387 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2291,27 +2291,17 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) outSeqs[i].rep = 1; repIdx = i - outSeqs[i].offset; - /* Not first block */ - if (repIdx >= 0) { - /* Special case where litLength == 0 */ - if (outSeqs[i].litLength == 0) { - /* When the offset is 3 */ - if (outSeqs[i].offset > 2) { - outSeqs[i].offset = outSeqs[repIdx - 1].offset - 1; - /* When the offset is either 1 or 2 */ - } else { - outSeqs[i].offset = outSeqs[repIdx - 1].offset; - } + if (outSeqs[i].litLength == 0) { + if (outSeqs[i].offset < 3) { + --repIdx; } else { - outSeqs[i].offset = outSeqs[repIdx].offset; + repIdx = i - 1; } - /* First block */ - } else if (repIdx == -1) { - outSeqs[i].offset = 1; - } else if (repIdx == -2) { - outSeqs[i].offset = 4; - } else if (repIdx == -3) { - outSeqs[i].offset = 8; + } + assert(repIdx >= -3); + outSeqs[i].offset = repIdx >= 0 ? outSeqs[repIdx].offset : repStartValue[-repIdx - 1]; + if (outSeqs[i].offset == 3) { + --outSeqs[i].offset; } } else { outSeqs[i].offset -= ZSTD_REP_NUM; From 1407919d132f1b29998939705fc3a7f0240f3a7c Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Tue, 10 Sep 2019 15:10:50 -0700 Subject: [PATCH 063/144] Addressing comments on parsing --- lib/compress/zstd_compress.c | 5 +++-- lib/zstd.h | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index b4fdf387..849a9f42 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2288,7 +2288,7 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) } if (outSeqs[i].offset <= ZSTD_REP_NUM) { - outSeqs[i].rep = 1; + outSeqs[i].rep = outSeqs[i].offset; repIdx = i - outSeqs[i].offset; if (outSeqs[i].litLength == 0) { @@ -2297,10 +2297,11 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) } else { repIdx = i - 1; } + ++outSeqs[i].rep; } assert(repIdx >= -3); outSeqs[i].offset = repIdx >= 0 ? outSeqs[repIdx].offset : repStartValue[-repIdx - 1]; - if (outSeqs[i].offset == 3) { + if (outSeqs[i].offset == 4) { --outSeqs[i].offset; } } else { diff --git a/lib/zstd.h b/lib/zstd.h index b2c66e75..97feb77d 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1077,7 +1077,7 @@ typedef struct { unsigned int offset; unsigned int litLength; unsigned int matchLength; - int rep; + unsigned int rep; } ZSTD_Sequence; typedef struct { From 17b6da2e0f57f4df3d6034811657d85d24ad831e Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 3 Sep 2019 13:08:24 -0400 Subject: [PATCH 064/144] Track Usable Table Space in Compression Workspace --- lib/compress/zstd_cwksp.h | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 76ace25e..e312e406 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -133,6 +133,7 @@ typedef struct { void* objectEnd; void* tableEnd; + void* tableValidEnd; void* allocStart; int allocFailed; @@ -161,6 +162,7 @@ MEM_STATIC void ZSTD_cwksp_internal_advance_phase( if (phase > ws->phase) { if (ws->phase < ZSTD_cwksp_alloc_buffers && phase >= ZSTD_cwksp_alloc_buffers) { + ws->tableValidEnd = ws->objectEnd; } if (ws->phase < ZSTD_cwksp_alloc_aligned && phase >= ZSTD_cwksp_alloc_aligned) { @@ -193,6 +195,9 @@ MEM_STATIC void* ZSTD_cwksp_reserve_internal( ws->allocFailed = 1; return NULL; } + if (alloc < ws->tableValidEnd) { + ws->tableValidEnd = alloc; + } ws->allocStart = alloc; return alloc; } @@ -256,9 +261,39 @@ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { } ws->objectEnd = end; ws->tableEnd = end; + ws->tableValidEnd = end; return start; } +MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_dirty"); + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + ws->tableValidEnd = ws->objectEnd; +} + +MEM_STATIC void ZSTD_cwksp_mark_tables_clean(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_clean"); + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + if (ws->tableValidEnd < ws->tableEnd) { + ws->tableValidEnd = ws->tableEnd; + } +} + +/** + * Zero the part of the allocated tables not already marked clean. + */ +MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: ZSTD_cwksp_clean_tables"); + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + if (ws->tableValidEnd < ws->tableEnd) { + memset(ws->tableValidEnd, 0, (BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd); + } + ZSTD_cwksp_mark_tables_clean(ws); +} + /** * Invalidates table allocations. * All other allocations remain valid. @@ -293,6 +328,7 @@ MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) { ws->workspace = start; ws->workspaceEnd = (BYTE*)start + size; ws->objectEnd = ws->workspace; + ws->tableValidEnd = ws->objectEnd; ws->phase = ZSTD_cwksp_alloc_objects; ZSTD_cwksp_clear(ws); ws->workspaceOversizedDuration = 0; From 14c5471d5e2307961ffe73b2d4337da071dbed9e Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 10 Sep 2019 17:24:02 -0400 Subject: [PATCH 065/144] Introduce `ZSTD_indexResetPolicy_e` Enum --- lib/compress/zstd_compress.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index df47ac78..9aaa3eec 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1359,15 +1359,32 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params, return 0; } -typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset } ZSTD_compResetPolicy_e; +typedef enum { + ZSTDcrp_continue, + ZSTDcrp_noMemset +} ZSTD_compResetPolicy_e; -typedef enum { ZSTD_resetTarget_CDict, ZSTD_resetTarget_CCtx } ZSTD_resetTarget_e; +/** + * Controls, for this matchState reset, whether indexing can continue where it + * left off (ZSTDirp_continue), or whether it needs to be restarted from zero + * (ZSTDirp_reset). + */ +typedef enum { + ZSTDirp_continue, + ZSTDirp_reset +} ZSTD_indexResetPolicy_e; + +typedef enum { + ZSTD_resetTarget_CDict, + ZSTD_resetTarget_CCtx +} ZSTD_resetTarget_e; static size_t ZSTD_reset_matchState(ZSTD_matchState_t* ms, ZSTD_cwksp* ws, const ZSTD_compressionParameters* cParams, - ZSTD_compResetPolicy_e const crp, ZSTD_resetTarget_e const forWho) + const ZSTD_compResetPolicy_e crp, + const ZSTD_resetTarget_e forWho) { size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); size_t const hSize = ((size_t)1) << cParams->hashLog; From 0492b9a9ecee89d8a53a6e009e1a4ce8136a888f Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 10 Sep 2019 17:28:52 -0400 Subject: [PATCH 066/144] Accept `ZSTD_indexResetPolicy_e` Param in `ZSTD_reset_matchState()` --- lib/compress/zstd_compress.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 9aaa3eec..2474c9f2 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1384,6 +1384,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, ZSTD_cwksp* ws, const ZSTD_compressionParameters* cParams, const ZSTD_compResetPolicy_e crp, + const ZSTD_indexResetPolicy_e forceResetIndex, const ZSTD_resetTarget_e forWho) { size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); @@ -1391,6 +1392,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; size_t const h3Size = ((size_t)1) << hashLog3; + (void)forceResetIndex; ms->hashLog3 = hashLog3; memset(&ms->window, 0, sizeof(ms->window)); @@ -1482,6 +1484,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ws, ¶ms.cParams, crp, + ZSTDirp_reset, ZSTD_resetTarget_CCtx)); } return ZSTD_continueCCtx(zc, ¶ms, pledgedSrcSize); @@ -1607,7 +1610,9 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, &zc->blockState.matchState, ws, ¶ms.cParams, - crp, ZSTD_resetTarget_CCtx)); + crp, + ZSTDirp_reset, + ZSTD_resetTarget_CCtx)); /* ldm hash table */ /* initialize bucketOffsets table separately for pointer alignment */ @@ -3156,7 +3161,9 @@ static size_t ZSTD_initCDict_internal( &cdict->matchState, &cdict->workspace, &cParams, - ZSTDcrp_continue, ZSTD_resetTarget_CDict)); + ZSTDcrp_continue, + ZSTDirp_reset, + ZSTD_resetTarget_CDict)); /* (Maybe) load the dictionary * Skips loading the dictionary if it is <= 8 bytes. */ From 5b10bb5ec392edb35b5fd25da75cc8113191df83 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 10 Sep 2019 17:38:32 -0400 Subject: [PATCH 067/144] Rename `ZSTD_compResetPolicy_e` Values and Add Comment --- lib/compress/zstd_compress.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 2474c9f2..7d6e322e 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1359,9 +1359,16 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params, return 0; } +/** + * Controls, for this matchState reset, whether the tables need to be cleared / + * prepared for the coming compression (ZSTDcrp_makeClean), or whether the + * tables can be left unclean (ZSTDcrp_leaveDirty), because we know that a + * subsequent operation will overwrite the table space anyways (e.g., copying + * the matchState contents in from a CDict). + */ typedef enum { - ZSTDcrp_continue, - ZSTDcrp_noMemset + ZSTDcrp_makeClean, + ZSTDcrp_leaveDirty } ZSTD_compResetPolicy_e; /** @@ -1413,8 +1420,8 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, "failed a workspace allocation in ZSTD_reset_matchState"); - DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_noMemset); - if (crp!=ZSTDcrp_noMemset) { + DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_leaveDirty); + if (crp!=ZSTDcrp_leaveDirty) { /* reset tables only */ memset(ms->hashTable, 0, hSize * sizeof(U32)); memset(ms->chainTable, 0, chainSize * sizeof(U32)); @@ -1467,7 +1474,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); zc->isFirstBlock = 1; - if (crp == ZSTDcrp_continue) { + if (crp == ZSTDcrp_makeClean) { if (ZSTD_equivalentParams(&zc->appliedParams, ¶ms, zc->inBuffSize, zc->seqStore.maxNbSeq, zc->seqStore.maxNbLit, @@ -1688,7 +1695,7 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, params.cParams = ZSTD_adjustCParams_internal(*cdict_cParams, pledgedSrcSize, 0); params.cParams.windowLog = windowLog; FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, - ZSTDcrp_continue, zbuff)); + ZSTDcrp_makeClean, zbuff)); assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); } @@ -1737,7 +1744,7 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, params.cParams = *cdict_cParams; params.cParams.windowLog = windowLog; FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, - ZSTDcrp_noMemset, zbuff)); + ZSTDcrp_leaveDirty, zbuff)); assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog); assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog); @@ -1820,7 +1827,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, params.cParams = srcCCtx->appliedParams.cParams; params.fParams = fParams; ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize, - ZSTDcrp_noMemset, zbuff); + ZSTDcrp_leaveDirty, zbuff); assert(dstCCtx->appliedParams.cParams.windowLog == srcCCtx->appliedParams.cParams.windowLog); assert(dstCCtx->appliedParams.cParams.strategy == srcCCtx->appliedParams.cParams.strategy); assert(dstCCtx->appliedParams.cParams.hashLog == srcCCtx->appliedParams.cParams.hashLog); @@ -2895,7 +2902,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, } FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, *params, pledgedSrcSize, - ZSTDcrp_continue, zbuff) ); + ZSTDcrp_makeClean, zbuff) ); { size_t const dictID = ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, params, dict, dictSize, dictContentType, dtlm, @@ -3161,7 +3168,7 @@ static size_t ZSTD_initCDict_internal( &cdict->matchState, &cdict->workspace, &cParams, - ZSTDcrp_continue, + ZSTDcrp_makeClean, ZSTDirp_reset, ZSTD_resetTarget_CDict)); /* (Maybe) load the dictionary From ad16eda5e402de068bb6b6036e94088694c2e82b Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 10 Sep 2019 17:43:35 -0400 Subject: [PATCH 068/144] `ZSTD_reset_matchState` Optionally Doesn't Restart Indexing --- lib/compress/zstd_compress.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 7d6e322e..ed1eafbe 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1399,13 +1399,16 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; size_t const h3Size = ((size_t)1) << hashLog3; - (void)forceResetIndex; + if (forceResetIndex == ZSTDirp_reset) { + memset(&ms->window, 0, sizeof(ms->window)); + ms->window.dictLimit = 1; /* start from 1, so that 1st position is valid */ + ms->window.lowLimit = 1; /* it ensures first and later CCtx usages compress the same */ + ms->window.nextSrc = ms->window.base + 1; /* see issue #1241 */ + ZSTD_cwksp_mark_tables_dirty(ws); + } ms->hashLog3 = hashLog3; - memset(&ms->window, 0, sizeof(ms->window)); - ms->window.dictLimit = 1; /* start from 1, so that 1st position is valid */ - ms->window.lowLimit = 1; /* it ensures first and later CCtx usages compress the same */ - ms->window.nextSrc = ms->window.base + 1; /* see issue #1241 */ + ZSTD_invalidateMatchState(ms); assert(!ZSTD_cwksp_reserve_failed(ws)); /* check that allocation hasn't already failed */ @@ -1423,9 +1426,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_leaveDirty); if (crp!=ZSTDcrp_leaveDirty) { /* reset tables only */ - memset(ms->hashTable, 0, hSize * sizeof(U32)); - memset(ms->chainTable, 0, chainSize * sizeof(U32)); - memset(ms->hashTable3, 0, h3Size * sizeof(U32)); + ZSTD_cwksp_clean_tables(ws); } /* opt parser space */ From 1b28e804163231956ec22d78e861682f29c741be Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 10 Sep 2019 17:54:11 -0400 Subject: [PATCH 069/144] Remove Fast Continue Path in `ZSTD_resetCCtx_internal()` --- lib/compress/zstd_compress.c | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index ed1eafbe..af9ecad2 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1475,29 +1475,6 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); zc->isFirstBlock = 1; - if (crp == ZSTDcrp_makeClean) { - if (ZSTD_equivalentParams(&zc->appliedParams, ¶ms, - zc->inBuffSize, - zc->seqStore.maxNbSeq, zc->seqStore.maxNbLit, - zbuff, pledgedSrcSize) ) { - DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> consider continue mode"); - ZSTD_cwksp_bump_oversized_duration(ws, 0); - if (!ZSTD_cwksp_check_wasteful(ws, 0)) { - DEBUGLOG(4, "continue mode confirmed (wLog1=%u, blockSize1=%zu)", - zc->appliedParams.cParams.windowLog, zc->blockSize); - if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) { - /* prefer a reset, faster than a rescale */ - FORWARD_IF_ERROR(ZSTD_reset_matchState( - &zc->blockState.matchState, - ws, - ¶ms.cParams, - crp, - ZSTDirp_reset, - ZSTD_resetTarget_CCtx)); - } - return ZSTD_continueCCtx(zc, ¶ms, pledgedSrcSize); - } } } - DEBUGLOG(4, "ZSTD_equivalentParams()==0 -> reset CCtx"); if (params.ldmParams.enableLdm) { /* Adjust long distance matching parameters */ From 9968a53e912aae109aae61c08cb8fb07f9980471 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 10 Sep 2019 17:59:45 -0400 Subject: [PATCH 070/144] Remove No-Longer-Used Continuation Functions --- lib/compress/zstd_compress.c | 114 +++-------------------------------- 1 file changed, 9 insertions(+), 105 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index af9ecad2..21b13fa8 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1214,17 +1214,6 @@ size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx) return 0; /* over-simplification; could also check if context is currently running in streaming mode, and in which case, report how many bytes are left to be flushed within output buffer */ } - - -static U32 ZSTD_equivalentCParams(ZSTD_compressionParameters cParams1, - ZSTD_compressionParameters cParams2) -{ - return (cParams1.hashLog == cParams2.hashLog) - & (cParams1.chainLog == cParams2.chainLog) - & (cParams1.strategy == cParams2.strategy) /* opt parser space */ - & ((cParams1.minMatch==3) == (cParams2.minMatch==3)); /* hashlog3 space */ -} - static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1, ZSTD_compressionParameters cParams2) { @@ -1239,71 +1228,6 @@ static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1, assert(cParams1.strategy == cParams2.strategy); } -/** The parameters are equivalent if ldm is not enabled in both sets or - * all the parameters are equivalent. */ -static U32 ZSTD_equivalentLdmParams(ldmParams_t ldmParams1, - ldmParams_t ldmParams2) -{ - return (!ldmParams1.enableLdm && !ldmParams2.enableLdm) || - (ldmParams1.enableLdm == ldmParams2.enableLdm && - ldmParams1.hashLog == ldmParams2.hashLog && - ldmParams1.bucketSizeLog == ldmParams2.bucketSizeLog && - ldmParams1.minMatchLength == ldmParams2.minMatchLength && - ldmParams1.hashRateLog == ldmParams2.hashRateLog); -} - -typedef enum { ZSTDb_not_buffered, ZSTDb_buffered } ZSTD_buffered_policy_e; - -/* ZSTD_sufficientBuff() : - * check internal buffers exist for streaming if buffPol == ZSTDb_buffered . - * Note : they are assumed to be correctly sized if ZSTD_equivalentCParams()==1 */ -static U32 ZSTD_sufficientBuff(size_t bufferSize1, size_t maxNbSeq1, - size_t maxNbLit1, - ZSTD_buffered_policy_e buffPol2, - ZSTD_compressionParameters cParams2, - U64 pledgedSrcSize) -{ - size_t const windowSize2 = MAX(1, (size_t)MIN(((U64)1 << cParams2.windowLog), pledgedSrcSize)); - size_t const blockSize2 = MIN(ZSTD_BLOCKSIZE_MAX, windowSize2); - size_t const maxNbSeq2 = blockSize2 / ((cParams2.minMatch == 3) ? 3 : 4); - size_t const maxNbLit2 = blockSize2; - size_t const neededBufferSize2 = (buffPol2==ZSTDb_buffered) ? windowSize2 + blockSize2 : 0; - DEBUGLOG(4, "ZSTD_sufficientBuff: is neededBufferSize2=%u <= bufferSize1=%u", - (U32)neededBufferSize2, (U32)bufferSize1); - DEBUGLOG(4, "ZSTD_sufficientBuff: is maxNbSeq2=%u <= maxNbSeq1=%u", - (U32)maxNbSeq2, (U32)maxNbSeq1); - DEBUGLOG(4, "ZSTD_sufficientBuff: is maxNbLit2=%u <= maxNbLit1=%u", - (U32)maxNbLit2, (U32)maxNbLit1); - return (maxNbLit2 <= maxNbLit1) - & (maxNbSeq2 <= maxNbSeq1) - & (neededBufferSize2 <= bufferSize1); -} - -/** Equivalence for resetCCtx purposes */ -static U32 ZSTD_equivalentParams(const ZSTD_CCtx_params* params1, - const ZSTD_CCtx_params* params2, - size_t buffSize1, - size_t maxNbSeq1, size_t maxNbLit1, - ZSTD_buffered_policy_e buffPol2, - U64 pledgedSrcSize) -{ - DEBUGLOG(4, "ZSTD_equivalentParams: pledgedSrcSize=%u", (U32)pledgedSrcSize); - if (!ZSTD_equivalentCParams(params1->cParams, params2->cParams)) { - DEBUGLOG(4, "ZSTD_equivalentCParams() == 0"); - return 0; - } - if (!ZSTD_equivalentLdmParams(params1->ldmParams, params2->ldmParams)) { - DEBUGLOG(4, "ZSTD_equivalentLdmParams() == 0"); - return 0; - } - if (!ZSTD_sufficientBuff(buffSize1, maxNbSeq1, maxNbLit1, buffPol2, - params2->cParams, pledgedSrcSize)) { - DEBUGLOG(4, "ZSTD_sufficientBuff() == 0"); - return 0; - } - return 1; -} - static void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs) { int i; @@ -1329,35 +1253,15 @@ static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms) ms->dictMatchState = NULL; } -/*! ZSTD_continueCCtx() : - * reuse CCtx without reset (note : requires no dictionary) */ -static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params, U64 pledgedSrcSize) -{ - size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize)); - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); - DEBUGLOG(4, "ZSTD_continueCCtx: re-use context in place"); - - cctx->blockSize = blockSize; /* previous block size could be different even for same windowLog, due to pledgedSrcSize */ - cctx->appliedParams = *params; - cctx->blockState.matchState.cParams = params->cParams; - cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; - cctx->consumedSrcSize = 0; - cctx->isFirstBlock = 1; - cctx->producedCSize = 0; - if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN) - cctx->appliedParams.fParams.contentSizeFlag = 0; - DEBUGLOG(4, "pledged content size : %u ; flag : %u", - (U32)pledgedSrcSize, cctx->appliedParams.fParams.contentSizeFlag); - cctx->stage = ZSTDcs_init; - cctx->dictID = 0; - if (params->ldmParams.enableLdm) - ZSTD_window_clear(&cctx->ldmState.window); - ZSTD_referenceExternalSequences(cctx, NULL, 0); - ZSTD_invalidateMatchState(&cctx->blockState.matchState); - ZSTD_reset_compressedBlockState(cctx->blockState.prevCBlock); - XXH64_reset(&cctx->xxhState, 0); - return 0; -} +/** + * Indicates whether this compression proceeds directly from user-provided + * source buffer to user-provided destination buffer (ZSTDb_not_buffered), or + * whether the context needs to buffer the input/output (ZSTDb_buffered). + */ +typedef enum { + ZSTDb_not_buffered, + ZSTDb_buffered +} ZSTD_buffered_policy_e; /** * Controls, for this matchState reset, whether the tables need to be cleared / From f31ef28ff894c9ccdbac1dbc611cb9e702d71738 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 10 Sep 2019 17:55:41 -0400 Subject: [PATCH 071/144] Only Reset Indexing in `ZSTD_resetCCtx_internal()` When Necessary --- lib/compress/zstd_compress.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 21b13fa8..ec7e7836 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1398,6 +1398,14 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const matchStateSize = ZSTD_sizeof_matchState(¶ms.cParams, /* forCCtx */ 1); size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize); + ZSTD_indexResetPolicy_e needsIndexReset = ZSTDirp_continue; + + if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) { + needsIndexReset = ZSTDirp_reset; + } + + ZSTD_cwksp_bump_oversized_duration(ws, 0); + /* Check if workspace is large enough, alloc a new one if needed */ { size_t const cctxSpace = zc->staticSize ? sizeof(ZSTD_CCtx) : 0; size_t const entropySpace = HUF_WORKSPACE_SIZE; @@ -1430,6 +1438,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize"); + needsIndexReset = ZSTDirp_reset; + ZSTD_cwksp_free(ws, zc->customMem); FORWARD_IF_ERROR(ZSTD_cwksp_create(ws, neededSpace, zc->customMem)); @@ -1500,7 +1510,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ws, ¶ms.cParams, crp, - ZSTDirp_reset, + needsIndexReset, ZSTD_resetTarget_CCtx)); /* ldm hash table */ From edb3ad053eb0784b6bc7e03778758b513dc23852 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 10 Sep 2019 18:02:22 -0400 Subject: [PATCH 072/144] Comments --- lib/compress/zstd_compress.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index ec7e7836..3fb77d59 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1490,6 +1490,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, /* ldm bucketOffsets table */ if (params.ldmParams.enableLdm) { + /* TODO: avoid memset? */ size_t const ldmBucketSize = ((size_t)1) << (params.ldmParams.hashLog - params.ldmParams.bucketSizeLog); @@ -1514,8 +1515,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_resetTarget_CCtx)); /* ldm hash table */ - /* initialize bucketOffsets table separately for pointer alignment */ if (params.ldmParams.enableLdm) { + /* TODO: avoid memset? */ size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog; zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t)); memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); From 0630d084cb0e0ea829630cc4fb1ebdb40af91652 Mon Sep 17 00:00:00 2001 From: Dario Pavlovic Date: Tue, 10 Sep 2019 16:14:43 -0700 Subject: [PATCH 073/144] [Fuzz] Improve data generation #1723 Converting the rest of the tests to use the new data producer. --- tests/fuzz/block_decompress.c | 2 - tests/fuzz/block_round_trip.c | 17 ++++--- tests/fuzz/dictionary_decompress.c | 14 +++--- tests/fuzz/dictionary_round_trip.c | 27 ++++++----- tests/fuzz/fuzz_data_producer.c | 12 ++++- tests/fuzz/fuzz_data_producer.h | 6 +++ tests/fuzz/simple_compress.c | 25 ++++++----- tests/fuzz/simple_decompress.c | 3 +- tests/fuzz/simple_round_trip.c | 18 ++++---- tests/fuzz/stream_decompress.c | 25 +++++++---- tests/fuzz/stream_round_trip.c | 44 ++++++++++-------- tests/fuzz/zstd_helpers.c | 72 +++++++++++++++--------------- tests/fuzz/zstd_helpers.h | 11 ++--- 13 files changed, 159 insertions(+), 117 deletions(-) diff --git a/tests/fuzz/block_decompress.c b/tests/fuzz/block_decompress.c index 3cccc32f..a904b446 100644 --- a/tests/fuzz/block_decompress.c +++ b/tests/fuzz/block_decompress.c @@ -28,8 +28,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { size_t const neededBufSize = ZSTD_BLOCKSIZE_MAX; - FUZZ_seed(&src, &size); - /* Allocate all buffers and contexts if not already allocated */ if (neededBufSize > bufSize) { free(rBuf); diff --git a/tests/fuzz/block_round_trip.c b/tests/fuzz/block_round_trip.c index 64ca5fc4..326d5b24 100644 --- a/tests/fuzz/block_round_trip.c +++ b/tests/fuzz/block_round_trip.c @@ -20,6 +20,7 @@ #include #include "fuzz_helpers.h" #include "zstd.h" +#include "fuzz_data_producer.h" static const int kMaxClevel = 19; @@ -28,13 +29,12 @@ static ZSTD_DCtx *dctx = NULL; static void* cBuf = NULL; static void* rBuf = NULL; static size_t bufSize = 0; -static uint32_t seed; static size_t roundTripTest(void *result, size_t resultCapacity, void *compressed, size_t compressedCapacity, - const void *src, size_t srcSize) + const void *src, size_t srcSize, + int cLevel) { - int const cLevel = FUZZ_rand(&seed) % kMaxClevel; ZSTD_parameters const params = ZSTD_getParams(cLevel, srcSize, 0); size_t ret = ZSTD_compressBegin_advanced(cctx, NULL, 0, params, srcSize); FUZZ_ZASSERT(ret); @@ -52,10 +52,11 @@ static size_t roundTripTest(void *result, size_t resultCapacity, int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - size_t neededBufSize; + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + int cLevel = FUZZ_dataProducer_uint32(producer) % kMaxClevel; + size = FUZZ_dataProducer_remainingBytes(producer); - seed = FUZZ_seed(&src, &size); - neededBufSize = size; + size_t neededBufSize = size; if (size > ZSTD_BLOCKSIZE_MAX) return 0; @@ -79,11 +80,13 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { size_t const result = - roundTripTest(rBuf, neededBufSize, cBuf, neededBufSize, src, size); + roundTripTest(rBuf, neededBufSize, cBuf, neededBufSize, src, size, + cLevel); FUZZ_ZASSERT(result); FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size"); FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!"); } + FUZZ_dataProducer_free(producer); #ifndef STATEFUL_FUZZING ZSTD_freeCCtx(cctx); cctx = NULL; ZSTD_freeDCtx(dctx); dctx = NULL; diff --git a/tests/fuzz/dictionary_decompress.c b/tests/fuzz/dictionary_decompress.c index e900054f..5f660ac5 100644 --- a/tests/fuzz/dictionary_decompress.c +++ b/tests/fuzz/dictionary_decompress.c @@ -18,12 +18,13 @@ #include #include "fuzz_helpers.h" #include "zstd_helpers.h" +#include "fuzz_data_producer.h" static ZSTD_DCtx *dctx = NULL; int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - uint32_t seed = FUZZ_seed(&src, &size); + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); FUZZ_dict_t dict; ZSTD_DDict* ddict = NULL; int i; @@ -32,19 +33,19 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) dctx = ZSTD_createDCtx(); FUZZ_ASSERT(dctx); } - dict = FUZZ_train(src, size, &seed); - if (FUZZ_rand32(&seed, 0, 1) == 0) { + dict = FUZZ_train(src, size, producer); + if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) { ddict = ZSTD_createDDict(dict.buff, dict.size); FUZZ_ASSERT(ddict); } else { FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced( dctx, dict.buff, dict.size, - (ZSTD_dictLoadMethod_e)FUZZ_rand32(&seed, 0, 1), - (ZSTD_dictContentType_e)FUZZ_rand32(&seed, 0, 2))); + (ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1), + (ZSTD_dictContentType_e)FUZZ_dataProducer_uint32Range(producer, 0, 2))); } /* Run it 10 times over 10 output sizes. Reuse the context and dict. */ for (i = 0; i < 10; ++i) { - size_t const bufSize = FUZZ_rand32(&seed, 0, 2 * size); + size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 2 * size); void* rBuf = malloc(bufSize); FUZZ_ASSERT(rBuf); if (ddict) { @@ -55,6 +56,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) free(rBuf); } free(dict.buff); + FUZZ_dataProducer_free(producer); ZSTD_freeDDict(ddict); #ifndef STATEFUL_FUZZING ZSTD_freeDCtx(dctx); dctx = NULL; diff --git a/tests/fuzz/dictionary_round_trip.c b/tests/fuzz/dictionary_round_trip.c index e28c65c9..fe0a217b 100644 --- a/tests/fuzz/dictionary_round_trip.c +++ b/tests/fuzz/dictionary_round_trip.c @@ -19,22 +19,23 @@ #include #include "fuzz_helpers.h" #include "zstd_helpers.h" +#include "fuzz_data_producer.h" static const int kMaxClevel = 19; static ZSTD_CCtx *cctx = NULL; static ZSTD_DCtx *dctx = NULL; -static uint32_t seed; static size_t roundTripTest(void *result, size_t resultCapacity, void *compressed, size_t compressedCapacity, - const void *src, size_t srcSize) + const void *src, size_t srcSize, + FUZZ_dataProducer_t *producer) { ZSTD_dictContentType_e dictContentType = ZSTD_dct_auto; - FUZZ_dict_t dict = FUZZ_train(src, srcSize, &seed); + FUZZ_dict_t dict = FUZZ_train(src, srcSize, producer); size_t cSize; - if ((FUZZ_rand(&seed) & 15) == 0) { - int const cLevel = FUZZ_rand(&seed) % kMaxClevel; + if ((FUZZ_dataProducer_uint32(producer) & 15) == 0) { + int const cLevel = FUZZ_dataProducer_uint32(producer) % kMaxClevel; cSize = ZSTD_compress_usingDict(cctx, compressed, compressedCapacity, @@ -42,20 +43,20 @@ static size_t roundTripTest(void *result, size_t resultCapacity, dict.buff, dict.size, cLevel); } else { - dictContentType = FUZZ_rand32(&seed, 0, 2); - FUZZ_setRandomParameters(cctx, srcSize, &seed); + dictContentType = FUZZ_dataProducer_uint32Range(producer, 0, 2); + FUZZ_setRandomParameters(cctx, srcSize, producer); /* Disable checksum so we can use sizes smaller than compress bound. */ FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 0)); FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary_advanced( cctx, dict.buff, dict.size, - (ZSTD_dictLoadMethod_e)FUZZ_rand32(&seed, 0, 1), + (ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1), dictContentType)); cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize); } FUZZ_ZASSERT(cSize); FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced( dctx, dict.buff, dict.size, - (ZSTD_dictLoadMethod_e)FUZZ_rand32(&seed, 0, 1), + (ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1), dictContentType)); { size_t const ret = ZSTD_decompressDCtx( @@ -72,12 +73,13 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) size_t cBufSize = ZSTD_compressBound(size); void* cBuf; - seed = FUZZ_seed(&src, &size); + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + /* Half of the time fuzz with a 1 byte smaller output size. * This will still succeed because we force the checksum to be disabled, * giving us 4 bytes of overhead. */ - cBufSize -= FUZZ_rand32(&seed, 0, 1); + cBufSize -= FUZZ_dataProducer_uint32Range(producer, 0, 1); cBuf = malloc(cBufSize); if (!cctx) { @@ -91,13 +93,14 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { size_t const result = - roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size); + roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size, producer); FUZZ_ZASSERT(result); FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size"); FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!"); } free(rBuf); free(cBuf); + FUZZ_dataProducer_free(producer); #ifndef STATEFUL_FUZZING ZSTD_freeCCtx(cctx); cctx = NULL; ZSTD_freeDCtx(dctx); dctx = NULL; diff --git a/tests/fuzz/fuzz_data_producer.c b/tests/fuzz/fuzz_data_producer.c index a083f636..6dcc1413 100644 --- a/tests/fuzz/fuzz_data_producer.c +++ b/tests/fuzz/fuzz_data_producer.c @@ -49,9 +49,19 @@ uint32_t FUZZ_dataProducer_uint32Range(FUZZ_dataProducer_t *producer, uint32_t m } uint32_t FUZZ_dataProducer_uint32(FUZZ_dataProducer_t *producer) { - return FUZZ_dataProducer_uint32Range(producer, 0, 0xffffffff); + return FUZZ_dataProducer_uint32Range(producer, 0, 0xffffffff); } size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer){ return producer->size; } + +size_t FUZZ_dataProducer_contract(FUZZ_dataProducer_t *producer, size_t newSize) +{ + newSize = newSize > producer->size ? producer->size : newSize; + + size_t remaining = producer->size - newSize; + producer->data = producer->data + remaining; + producer->size = newSize; + return remaining; +} diff --git a/tests/fuzz/fuzz_data_producer.h b/tests/fuzz/fuzz_data_producer.h index 4fcf6fd4..668c87f2 100644 --- a/tests/fuzz/fuzz_data_producer.h +++ b/tests/fuzz/fuzz_data_producer.h @@ -44,4 +44,10 @@ uint32_t FUZZ_dataProducer_uint32(FUZZ_dataProducer_t *producer); /* Returns the size of the remaining bytes of data in the producer */ size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer); +/* Tells the producer to contract to newSize bytes of data it currently uses, +counted from the end, and forget about the rest. If newSize > current data size, +nothing happens. Returns the number of bytes the producer won't use anymore, +after contracting. */ +size_t FUZZ_dataProducer_contract(FUZZ_dataProducer_t *producer, size_t newSize); + #endif // FUZZ_DATA_PRODUCER_H diff --git a/tests/fuzz/simple_compress.c b/tests/fuzz/simple_compress.c index aaed4035..29de4701 100644 --- a/tests/fuzz/simple_compress.c +++ b/tests/fuzz/simple_compress.c @@ -18,28 +18,31 @@ #include #include "fuzz_helpers.h" #include "zstd.h" +#include "fuzz_data_producer.h" static ZSTD_CCtx *cctx = NULL; int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - uint32_t seed = FUZZ_seed(&src, &size); + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + + int const level = (int)FUZZ_dataProducer_uint32Range( + producer, 0, 19 + 3) - 3; /* [-3, 19] */ size_t const maxSize = ZSTD_compressBound(size); - int i; + size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, maxSize); + + size = FUZZ_dataProducer_remainingBytes(producer); + if (!cctx) { cctx = ZSTD_createCCtx(); FUZZ_ASSERT(cctx); } - /* Run it 10 times over 10 output sizes. Reuse the context. */ - for (i = 0; i < 10; ++i) { - int const level = (int)FUZZ_rand32(&seed, 0, 19 + 3) - 3; /* [-3, 19] */ - size_t const bufSize = FUZZ_rand32(&seed, 0, maxSize); - void* rBuf = malloc(bufSize); - FUZZ_ASSERT(rBuf); - ZSTD_compressCCtx(cctx, rBuf, bufSize, src, size, level); - free(rBuf); - } + void *rBuf = malloc(bufSize); + FUZZ_ASSERT(rBuf); + ZSTD_compressCCtx(cctx, rBuf, bufSize, src, size, level); + free(rBuf); + FUZZ_dataProducer_free(producer); #ifndef STATEFUL_FUZZING ZSTD_freeCCtx(cctx); cctx = NULL; #endif diff --git a/tests/fuzz/simple_decompress.c b/tests/fuzz/simple_decompress.c index a68813ee..0ab634fa 100644 --- a/tests/fuzz/simple_decompress.c +++ b/tests/fuzz/simple_decompress.c @@ -25,7 +25,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); - int i; if (!dctx) { dctx = ZSTD_createDCtx(); FUZZ_ASSERT(dctx); @@ -37,7 +36,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) /* Restrict to remaining data. If we run out of data while generating params, we should still continue and let decompression happen on empty data. */ - size = FUZZ_dataProducer_remainingBytes(producer); + size = FUZZ_dataProducer_remainingBytes(producer); ZSTD_decompressDCtx(dctx, rBuf, bufSize, src, size); free(rBuf); diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c index 7e3b6609..b88d404f 100644 --- a/tests/fuzz/simple_round_trip.c +++ b/tests/fuzz/simple_round_trip.c @@ -20,23 +20,24 @@ #include #include "fuzz_helpers.h" #include "zstd_helpers.h" +#include "fuzz_data_producer.h" static const int kMaxClevel = 19; static ZSTD_CCtx *cctx = NULL; static ZSTD_DCtx *dctx = NULL; -static uint32_t seed; static size_t roundTripTest(void *result, size_t resultCapacity, void *compressed, size_t compressedCapacity, - const void *src, size_t srcSize) + const void *src, size_t srcSize, + FUZZ_dataProducer_t *producer) { size_t cSize; - if (FUZZ_rand(&seed) & 1) { - FUZZ_setRandomParameters(cctx, srcSize, &seed); + if (FUZZ_dataProducer_uint32(producer) & 1) { + FUZZ_setRandomParameters(cctx, srcSize, producer); cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize); } else { - int const cLevel = FUZZ_rand(&seed) % kMaxClevel; + int const cLevel = FUZZ_dataProducer_uint32(producer) % kMaxClevel; cSize = ZSTD_compressCCtx( cctx, compressed, compressedCapacity, src, srcSize, cLevel); } @@ -51,12 +52,12 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) size_t cBufSize = ZSTD_compressBound(size); void* cBuf; - seed = FUZZ_seed(&src, &size); + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); /* Half of the time fuzz with a 1 byte smaller output size. * This will still succeed because we don't use a dictionary, so the dictID * field is empty, giving us 4 bytes of overhead. */ - cBufSize -= FUZZ_rand32(&seed, 0, 1); + cBufSize -= FUZZ_dataProducer_uint32Range(producer, 0, 1); cBuf = malloc(cBufSize); FUZZ_ASSERT(cBuf && rBuf); @@ -72,13 +73,14 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { size_t const result = - roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size); + roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size, producer); FUZZ_ZASSERT(result); FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size"); FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!"); } free(rBuf); free(cBuf); + FUZZ_dataProducer_free(producer); #ifndef STATEFUL_FUZZING ZSTD_freeCCtx(cctx); cctx = NULL; ZSTD_freeDCtx(dctx); dctx = NULL; diff --git a/tests/fuzz/stream_decompress.c b/tests/fuzz/stream_decompress.c index 68e120d7..4d5c4996 100644 --- a/tests/fuzz/stream_decompress.c +++ b/tests/fuzz/stream_decompress.c @@ -19,6 +19,7 @@ #include #include "fuzz_helpers.h" #include "zstd.h" +#include "fuzz_data_producer.h" static size_t const kBufSize = ZSTD_BLOCKSIZE_MAX; @@ -26,22 +27,23 @@ static ZSTD_DStream *dstream = NULL; static void* buf = NULL; uint32_t seed; -static ZSTD_outBuffer makeOutBuffer(void) +static ZSTD_outBuffer makeOutBuffer(FUZZ_dataProducer_t *producer) { ZSTD_outBuffer buffer = { buf, 0, 0 }; - buffer.size = (FUZZ_rand(&seed) % kBufSize) + 1; + buffer.size = (FUZZ_dataProducer_uint32(producer) % kBufSize) + 1; FUZZ_ASSERT(buffer.size <= kBufSize); return buffer; } -static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size) +static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size, + FUZZ_dataProducer_t *producer) { ZSTD_inBuffer buffer = { *src, 0, 0 }; FUZZ_ASSERT(*size > 0); - buffer.size = (FUZZ_rand(&seed) % *size) + 1; + buffer.size = (FUZZ_dataProducer_uint32(producer) % *size) + 1; FUZZ_ASSERT(buffer.size <= *size); *src += buffer.size; *size -= buffer.size; @@ -51,13 +53,17 @@ static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size) int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - seed = FUZZ_seed(&src, &size); + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); + size = FUZZ_dataProducer_contract(producer, producerSliceSize); /* Allocate all buffers and contexts if not already allocated */ if (!buf) { buf = malloc(kBufSize); - FUZZ_ASSERT(buf); - } + FUZZ_ASSERT(buf); + } if (!dstream) { dstream = ZSTD_createDStream(); @@ -67,9 +73,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) } while (size > 0) { - ZSTD_inBuffer in = makeInBuffer(&src, &size); + ZSTD_inBuffer in = makeInBuffer(&src, &size, producer); while (in.pos != in.size) { - ZSTD_outBuffer out = makeOutBuffer(); + ZSTD_outBuffer out = makeOutBuffer(producer); size_t const rc = ZSTD_decompressStream(dstream, &out, &in); if (ZSTD_isError(rc)) goto error; } @@ -79,5 +85,6 @@ error: #ifndef STATEFUL_FUZZING ZSTD_freeDStream(dstream); dstream = NULL; #endif + FUZZ_dataProducer_free(producer); return 0; } diff --git a/tests/fuzz/stream_round_trip.c b/tests/fuzz/stream_round_trip.c index d13c2dbe..4569222f 100644 --- a/tests/fuzz/stream_round_trip.c +++ b/tests/fuzz/stream_round_trip.c @@ -20,31 +20,33 @@ #include #include "fuzz_helpers.h" #include "zstd_helpers.h" +#include "fuzz_data_producer.h" ZSTD_CCtx *cctx = NULL; static ZSTD_DCtx *dctx = NULL; static uint8_t* cBuf = NULL; static uint8_t* rBuf = NULL; static size_t bufSize = 0; -static uint32_t seed; -static ZSTD_outBuffer makeOutBuffer(uint8_t *dst, size_t capacity) +static ZSTD_outBuffer makeOutBuffer(uint8_t *dst, size_t capacity, + FUZZ_dataProducer_t *producer) { ZSTD_outBuffer buffer = { dst, 0, 0 }; FUZZ_ASSERT(capacity > 0); - buffer.size = (FUZZ_rand(&seed) % capacity) + 1; + buffer.size = (FUZZ_dataProducer_uint32(producer) % capacity) + 1; FUZZ_ASSERT(buffer.size <= capacity); return buffer; } -static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size) +static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size, + FUZZ_dataProducer_t *producer) { ZSTD_inBuffer buffer = { *src, 0, 0 }; FUZZ_ASSERT(*size > 0); - buffer.size = (FUZZ_rand(&seed) % *size) + 1; + buffer.size = (FUZZ_dataProducer_uint32(producer) % *size) + 1; FUZZ_ASSERT(buffer.size <= *size); *src += buffer.size; *size -= buffer.size; @@ -53,23 +55,24 @@ static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size) } static size_t compress(uint8_t *dst, size_t capacity, - const uint8_t *src, size_t srcSize) + const uint8_t *src, size_t srcSize, + FUZZ_dataProducer_t *producer) { size_t dstSize = 0; ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); - FUZZ_setRandomParameters(cctx, srcSize, &seed); + FUZZ_setRandomParameters(cctx, srcSize, producer); while (srcSize > 0) { - ZSTD_inBuffer in = makeInBuffer(&src, &srcSize); + ZSTD_inBuffer in = makeInBuffer(&src, &srcSize, producer); /* Mode controls the action. If mode == -1 we pick a new mode */ int mode = -1; while (in.pos < in.size || mode != -1) { - ZSTD_outBuffer out = makeOutBuffer(dst, capacity); + ZSTD_outBuffer out = makeOutBuffer(dst, capacity, producer); /* Previous action finished, pick a new mode. */ - if (mode == -1) mode = FUZZ_rand(&seed) % 10; + if (mode == -1) mode = FUZZ_dataProducer_uint32(producer) % 10; switch (mode) { - case 0: /* fall-though */ - case 1: /* fall-though */ + case 0: /* fall-through */ + case 1: /* fall-through */ case 2: { size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush); @@ -85,9 +88,9 @@ static size_t compress(uint8_t *dst, size_t capacity, /* Reset the compressor when the frame is finished */ if (ret == 0) { ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); - if ((FUZZ_rand(&seed) & 7) == 0) { + if ((FUZZ_dataProducer_uint32(producer) & 7) == 0) { size_t const remaining = in.size - in.pos; - FUZZ_setRandomParameters(cctx, remaining, &seed); + FUZZ_setRandomParameters(cctx, remaining, producer); } mode = -1; } @@ -107,7 +110,7 @@ static size_t compress(uint8_t *dst, size_t capacity, } for (;;) { ZSTD_inBuffer in = {NULL, 0, 0}; - ZSTD_outBuffer out = makeOutBuffer(dst, capacity); + ZSTD_outBuffer out = makeOutBuffer(dst, capacity, producer); size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); FUZZ_ZASSERT(ret); @@ -123,9 +126,13 @@ static size_t compress(uint8_t *dst, size_t capacity, int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { size_t neededBufSize; + neededBufSize = ZSTD_compressBound(size) * 5; - seed = FUZZ_seed(&src, &size); - neededBufSize = ZSTD_compressBound(size) * 2; + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); + size = FUZZ_dataProducer_contract(producer, producerSliceSize); /* Allocate all buffers and contexts if not already allocated */ if (neededBufSize > bufSize) { @@ -146,7 +153,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) } { - size_t const cSize = compress(cBuf, neededBufSize, src, size); + size_t const cSize = compress(cBuf, neededBufSize, src, size, producer); size_t const rSize = ZSTD_decompressDCtx(dctx, rBuf, neededBufSize, cBuf, cSize); FUZZ_ZASSERT(rSize); @@ -154,6 +161,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!"); } + FUZZ_dataProducer_free(producer); #ifndef STATEFUL_FUZZING ZSTD_freeCCtx(cctx); cctx = NULL; ZSTD_freeDCtx(dctx); dctx = NULL; diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c index 5ff057b8..2635de4c 100644 --- a/tests/fuzz/zstd_helpers.c +++ b/tests/fuzz/zstd_helpers.c @@ -23,47 +23,47 @@ static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, int value) } static void setRand(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned min, - unsigned max, uint32_t *state) { - unsigned const value = FUZZ_rand32(state, min, max); + unsigned max, FUZZ_dataProducer_t *producer) { + unsigned const value = FUZZ_dataProducer_uint32Range(producer, min, max); set(cctx, param, value); } -ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state) +ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, FUZZ_dataProducer_t *producer) { /* Select compression parameters */ ZSTD_compressionParameters cParams; - cParams.windowLog = FUZZ_rand32(state, ZSTD_WINDOWLOG_MIN, 15); - cParams.hashLog = FUZZ_rand32(state, ZSTD_HASHLOG_MIN, 15); - cParams.chainLog = FUZZ_rand32(state, ZSTD_CHAINLOG_MIN, 16); - cParams.searchLog = FUZZ_rand32(state, ZSTD_SEARCHLOG_MIN, 9); - cParams.minMatch = FUZZ_rand32(state, ZSTD_MINMATCH_MIN, + cParams.windowLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_WINDOWLOG_MIN, 15); + cParams.hashLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_HASHLOG_MIN, 15); + cParams.chainLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_CHAINLOG_MIN, 16); + cParams.searchLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_SEARCHLOG_MIN, 9); + cParams.minMatch = FUZZ_dataProducer_uint32Range(producer, ZSTD_MINMATCH_MIN, ZSTD_MINMATCH_MAX); - cParams.targetLength = FUZZ_rand32(state, 0, 512); - cParams.strategy = FUZZ_rand32(state, ZSTD_STRATEGY_MIN, ZSTD_STRATEGY_MAX); + cParams.targetLength = FUZZ_dataProducer_uint32Range(producer, 0, 512); + cParams.strategy = FUZZ_dataProducer_uint32Range(producer, ZSTD_STRATEGY_MIN, ZSTD_STRATEGY_MAX); return ZSTD_adjustCParams(cParams, srcSize, 0); } -ZSTD_frameParameters FUZZ_randomFParams(uint32_t *state) +ZSTD_frameParameters FUZZ_randomFParams(FUZZ_dataProducer_t *producer) { /* Select frame parameters */ ZSTD_frameParameters fParams; - fParams.contentSizeFlag = FUZZ_rand32(state, 0, 1); - fParams.checksumFlag = FUZZ_rand32(state, 0, 1); - fParams.noDictIDFlag = FUZZ_rand32(state, 0, 1); + fParams.contentSizeFlag = FUZZ_dataProducer_uint32Range(producer, 0, 1); + fParams.checksumFlag = FUZZ_dataProducer_uint32Range(producer, 0, 1); + fParams.noDictIDFlag = FUZZ_dataProducer_uint32Range(producer, 0, 1); return fParams; } -ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state) +ZSTD_parameters FUZZ_randomParams(size_t srcSize, FUZZ_dataProducer_t *producer) { ZSTD_parameters params; - params.cParams = FUZZ_randomCParams(srcSize, state); - params.fParams = FUZZ_randomFParams(state); + params.cParams = FUZZ_randomCParams(srcSize, producer); + params.fParams = FUZZ_randomFParams(producer); return params; } -void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state) +void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer_t *producer) { - ZSTD_compressionParameters cParams = FUZZ_randomCParams(srcSize, state); + ZSTD_compressionParameters cParams = FUZZ_randomCParams(srcSize, producer); set(cctx, ZSTD_c_windowLog, cParams.windowLog); set(cctx, ZSTD_c_hashLog, cParams.hashLog); set(cctx, ZSTD_c_chainLog, cParams.chainLog); @@ -72,30 +72,30 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state) set(cctx, ZSTD_c_targetLength, cParams.targetLength); set(cctx, ZSTD_c_strategy, cParams.strategy); /* Select frame parameters */ - setRand(cctx, ZSTD_c_contentSizeFlag, 0, 1, state); - setRand(cctx, ZSTD_c_checksumFlag, 0, 1, state); - setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, state); + setRand(cctx, ZSTD_c_contentSizeFlag, 0, 1, producer); + setRand(cctx, ZSTD_c_checksumFlag, 0, 1, producer); + setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, producer); /* Select long distance matching parameters */ - setRand(cctx, ZSTD_c_enableLongDistanceMatching, 0, 1, state); - setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, state); + setRand(cctx, ZSTD_c_enableLongDistanceMatching, 0, 1, producer); + setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, producer); setRand(cctx, ZSTD_c_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN, - ZSTD_LDM_MINMATCH_MAX, state); + ZSTD_LDM_MINMATCH_MAX, producer); setRand(cctx, ZSTD_c_ldmBucketSizeLog, 0, ZSTD_LDM_BUCKETSIZELOG_MAX, - state); + producer); setRand(cctx, ZSTD_c_ldmHashRateLog, ZSTD_LDM_HASHRATELOG_MIN, - ZSTD_LDM_HASHRATELOG_MAX, state); + ZSTD_LDM_HASHRATELOG_MAX, producer); /* Set misc parameters */ - setRand(cctx, ZSTD_c_nbWorkers, 0, 2, state); - setRand(cctx, ZSTD_c_rsyncable, 0, 1, state); - setRand(cctx, ZSTD_c_forceMaxWindow, 0, 1, state); - setRand(cctx, ZSTD_c_literalCompressionMode, 0, 2, state); - setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, state); - if (FUZZ_rand32(state, 0, 1) == 0) { - setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, state); + setRand(cctx, ZSTD_c_nbWorkers, 0, 2, producer); + setRand(cctx, ZSTD_c_rsyncable, 0, 1, producer); + setRand(cctx, ZSTD_c_forceMaxWindow, 0, 1, producer); + setRand(cctx, ZSTD_c_literalCompressionMode, 0, 2, producer); + setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, producer); + if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) { + setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer); } } -FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, uint32_t *state) +FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, FUZZ_dataProducer_t *producer) { size_t const dictSize = MAX(srcSize / 8, 1024); size_t const totalSampleSize = dictSize * 11; @@ -110,7 +110,7 @@ FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, uint32_t *state) for (sample = 0; sample < nbSamples; ++sample) { size_t const remaining = totalSampleSize - pos; - size_t const offset = FUZZ_rand32(state, 0, MAX(srcSize, 1) - 1); + size_t const offset = FUZZ_dataProducer_uint32Range(producer, 0, MAX(srcSize, 1) - 1); size_t const limit = MIN(srcSize - offset, remaining); size_t const toCopy = MIN(limit, remaining / (nbSamples - sample)); memcpy(samples + pos, src + offset, toCopy); diff --git a/tests/fuzz/zstd_helpers.h b/tests/fuzz/zstd_helpers.h index 457e6e99..f2001f8b 100644 --- a/tests/fuzz/zstd_helpers.h +++ b/tests/fuzz/zstd_helpers.h @@ -17,17 +17,18 @@ #define ZSTD_STATIC_LINKING_ONLY #include "zstd.h" +#include "fuzz_data_producer.h" #include #ifdef __cplusplus extern "C" { #endif -void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state); +void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer_t *producer); -ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state); -ZSTD_frameParameters FUZZ_randomFParams(uint32_t *state); -ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state); +ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, FUZZ_dataProducer_t *producer); +ZSTD_frameParameters FUZZ_randomFParams(FUZZ_dataProducer_t *producer); +ZSTD_parameters FUZZ_randomParams(size_t srcSize, FUZZ_dataProducer_t *producer); typedef struct { void* buff; @@ -38,7 +39,7 @@ typedef struct { * NOTE: Don't use this to train production dictionaries, it is only optimized * for speed, and doesn't care about dictionary quality. */ -FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, uint32_t *state); +FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, FUZZ_dataProducer_t *producer); #ifdef __cplusplus From 23cc2d851083fdc1c1af15fa4190ce74e29b73dd Mon Sep 17 00:00:00 2001 From: Dario Pavlovic Date: Tue, 10 Sep 2019 16:52:38 -0700 Subject: [PATCH 074/144] All tests should give some portion of data to the producer and use the rest. --- tests/fuzz/block_round_trip.c | 6 +++++- tests/fuzz/dictionary_decompress.c | 9 +++++++-- tests/fuzz/dictionary_round_trip.c | 11 +++++++---- tests/fuzz/simple_compress.c | 9 ++++++--- tests/fuzz/simple_decompress.c | 30 +++++++++++++++--------------- tests/fuzz/simple_round_trip.c | 7 +++++++ tests/fuzz/stream_round_trip.c | 6 +++--- tests/fuzz/zstd_frame_info.c | 4 ---- 8 files changed, 50 insertions(+), 32 deletions(-) diff --git a/tests/fuzz/block_round_trip.c b/tests/fuzz/block_round_trip.c index 326d5b24..9ead57a5 100644 --- a/tests/fuzz/block_round_trip.c +++ b/tests/fuzz/block_round_trip.c @@ -52,9 +52,13 @@ static size_t roundTripTest(void *result, size_t resultCapacity, int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); + size = FUZZ_dataProducer_contract(producer, producerSliceSize); + int cLevel = FUZZ_dataProducer_uint32(producer) % kMaxClevel; - size = FUZZ_dataProducer_remainingBytes(producer); size_t neededBufSize = size; if (size > ZSTD_BLOCKSIZE_MAX) diff --git a/tests/fuzz/dictionary_decompress.c b/tests/fuzz/dictionary_decompress.c index 5f660ac5..54903f1f 100644 --- a/tests/fuzz/dictionary_decompress.c +++ b/tests/fuzz/dictionary_decompress.c @@ -24,7 +24,12 @@ static ZSTD_DCtx *dctx = NULL; int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); + size = FUZZ_dataProducer_contract(producer, producerSliceSize); + FUZZ_dict_t dict; ZSTD_DDict* ddict = NULL; int i; @@ -43,8 +48,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) (ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1), (ZSTD_dictContentType_e)FUZZ_dataProducer_uint32Range(producer, 0, 2))); } - /* Run it 10 times over 10 output sizes. Reuse the context and dict. */ - for (i = 0; i < 10; ++i) { + + { size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 2 * size); void* rBuf = malloc(bufSize); FUZZ_ASSERT(rBuf); diff --git a/tests/fuzz/dictionary_round_trip.c b/tests/fuzz/dictionary_round_trip.c index fe0a217b..cd1ca38f 100644 --- a/tests/fuzz/dictionary_round_trip.c +++ b/tests/fuzz/dictionary_round_trip.c @@ -68,13 +68,16 @@ static size_t roundTripTest(void *result, size_t resultCapacity, int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); + size = FUZZ_dataProducer_contract(producer, producerSliceSize); + size_t const rBufSize = size; void* rBuf = malloc(rBufSize); size_t cBufSize = ZSTD_compressBound(size); - void* cBuf; - - FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); - + void *cBuf; /* Half of the time fuzz with a 1 byte smaller output size. * This will still succeed because we force the checksum to be disabled, * giving us 4 bytes of overhead. diff --git a/tests/fuzz/simple_compress.c b/tests/fuzz/simple_compress.c index 29de4701..b8c6394c 100644 --- a/tests/fuzz/simple_compress.c +++ b/tests/fuzz/simple_compress.c @@ -24,14 +24,17 @@ static ZSTD_CCtx *cctx = NULL; int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); + size = FUZZ_dataProducer_contract(producer, producerSliceSize); - int const level = (int)FUZZ_dataProducer_uint32Range( - producer, 0, 19 + 3) - 3; /* [-3, 19] */ size_t const maxSize = ZSTD_compressBound(size); size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, maxSize); - size = FUZZ_dataProducer_remainingBytes(producer); + int const level = (int)FUZZ_dataProducer_uint32Range( + producer, 0, 19 + 3) - 3; /* [-3, 19] */ if (!cctx) { cctx = ZSTD_createCCtx(); diff --git a/tests/fuzz/simple_decompress.c b/tests/fuzz/simple_decompress.c index 0ab634fa..5c7680bf 100644 --- a/tests/fuzz/simple_decompress.c +++ b/tests/fuzz/simple_decompress.c @@ -23,25 +23,25 @@ static ZSTD_DCtx *dctx = NULL; int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); + size = FUZZ_dataProducer_contract(producer, producerSliceSize); - if (!dctx) { - dctx = ZSTD_createDCtx(); - FUZZ_ASSERT(dctx); - } + if (!dctx) { + dctx = ZSTD_createDCtx(); + FUZZ_ASSERT(dctx); + } - size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 10 * size); - void *rBuf = malloc(bufSize); - FUZZ_ASSERT(rBuf); + size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 10 * size); + void *rBuf = malloc(bufSize); + FUZZ_ASSERT(rBuf); - /* Restrict to remaining data. If we run out of data while generating params, - we should still continue and let decompression happen on empty data. */ - size = FUZZ_dataProducer_remainingBytes(producer); + ZSTD_decompressDCtx(dctx, rBuf, bufSize, src, size); + free(rBuf); - ZSTD_decompressDCtx(dctx, rBuf, bufSize, src, size); - free(rBuf); - - FUZZ_dataProducer_free(producer); + FUZZ_dataProducer_free(producer); #ifndef STATEFUL_FUZZING ZSTD_freeDCtx(dctx); dctx = NULL; diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c index b88d404f..f19e37a3 100644 --- a/tests/fuzz/simple_round_trip.c +++ b/tests/fuzz/simple_round_trip.c @@ -52,12 +52,19 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) size_t cBufSize = ZSTD_compressBound(size); void* cBuf; + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); + size = FUZZ_dataProducer_contract(producer, producerSliceSize); + /* Half of the time fuzz with a 1 byte smaller output size. * This will still succeed because we don't use a dictionary, so the dictID * field is empty, giving us 4 bytes of overhead. */ cBufSize -= FUZZ_dataProducer_uint32Range(producer, 0, 1); + size = FUZZ_dataProducer_remainingBytes(producer); + cBuf = malloc(cBufSize); FUZZ_ASSERT(cBuf && rBuf); diff --git a/tests/fuzz/stream_round_trip.c b/tests/fuzz/stream_round_trip.c index 4569222f..02bb26da 100644 --- a/tests/fuzz/stream_round_trip.c +++ b/tests/fuzz/stream_round_trip.c @@ -125,15 +125,15 @@ static size_t compress(uint8_t *dst, size_t capacity, int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { - size_t neededBufSize; - neededBufSize = ZSTD_compressBound(size) * 5; - /* Give a random portion of src data to the producer, to use for parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); size = FUZZ_dataProducer_contract(producer, producerSliceSize); + size_t neededBufSize; + neededBufSize = ZSTD_compressBound(size) * 5; + /* Allocate all buffers and contexts if not already allocated */ if (neededBufSize > bufSize) { free(cBuf); diff --git a/tests/fuzz/zstd_frame_info.c b/tests/fuzz/zstd_frame_info.c index 7512d5f4..359cf128 100644 --- a/tests/fuzz/zstd_frame_info.c +++ b/tests/fuzz/zstd_frame_info.c @@ -21,10 +21,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) { ZSTD_frameHeader zfh; - /* Consume the seed to be compatible with the corpora of other decompression - * fuzzers. - */ - FUZZ_seed(&src, &size); /* You can fuzz any helper functions here that are fast, and take zstd * compressed data as input. E.g. don't expect the input to be a dictionary, * so don't fuzz ZSTD_getDictID_fromDict(). From 13e29a56de32e649e37c87ffc64f11e6be55edc1 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 11 Sep 2019 11:18:45 -0400 Subject: [PATCH 075/144] Shrink Clean Table Area When Copying Table Contents into Context The source matchState is potentially at a lower current index, which means that any extra table space not overwritten by the copy may now contain invalid indices. The simple solution is to unconditionally shrink the valid table area to just the area overwritten. --- lib/compress/zstd_compress.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 3fb77d59..64fe952a 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1643,6 +1643,8 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog); } + ZSTD_cwksp_mark_tables_dirty(&cctx->workspace); + /* copy tables */ { size_t const chainSize = (cdict_cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cdict_cParams->chainLog); size_t const hSize = (size_t)1 << cdict_cParams->hashLog; @@ -1660,6 +1662,8 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32)); } + ZSTD_cwksp_mark_tables_clean(&cctx->workspace); + /* copy dictionary offsets */ { ZSTD_matchState_t const* srcMatchState = &cdict->matchState; ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState; @@ -1728,6 +1732,8 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, assert(dstCCtx->blockState.matchState.hashLog3 == srcCCtx->blockState.matchState.hashLog3); } + ZSTD_cwksp_mark_tables_dirty(&dstCCtx->workspace); + /* copy tables */ { size_t const chainSize = (srcCCtx->appliedParams.cParams.strategy == ZSTD_fast) ? 0 : ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog); size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog; @@ -1738,6 +1744,8 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, memcpy(dstCCtx->blockState.matchState.hashTable, srcCCtx->blockState.matchState.hashTable, tableSpace); /* presumes all tables follow each other */ } + ZSTD_cwksp_mark_tables_clean(&dstCCtx->workspace); + /* copy dictionary offsets */ { const ZSTD_matchState_t* srcMatchState = &srcCCtx->blockState.matchState; From 1999b2ed9bdae1c0e6b5a6f976c8a67dc33d133d Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 11 Sep 2019 11:21:00 -0400 Subject: [PATCH 076/144] Update DEBUGLOG Statements --- lib/compress/zstd_compress.c | 1 + lib/compress/zstd_cwksp.h | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 64fe952a..28341c0d 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1303,6 +1303,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; size_t const h3Size = ((size_t)1) << hashLog3; + DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset); if (forceResetIndex == ZSTDirp_reset) { memset(&ms->window, 0, sizeof(ms->window)); ms->window.dictLimit = 1; /* start from 1, so that 1st position is valid */ diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index e312e406..4c146145 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -188,7 +188,7 @@ MEM_STATIC void* ZSTD_cwksp_reserve_internal( void* bottom = ws->tableEnd; ZSTD_cwksp_internal_advance_phase(ws, phase); alloc = (BYTE *)ws->allocStart - bytes; - DEBUGLOG(4, "cwksp: reserving %zd bytes, %zd bytes remaining", + DEBUGLOG(5, "cwksp: reserving %zd bytes, %zd bytes remaining", bytes, ZSTD_cwksp_available_space(ws) - bytes); assert(alloc >= bottom); if (alloc < bottom) { @@ -227,7 +227,7 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { void* alloc = ws->tableEnd; void* end = (BYTE *)alloc + bytes; void* top = ws->allocStart; - DEBUGLOG(4, "cwksp: reserving table %zd bytes, %zd bytes remaining", + DEBUGLOG(5, "cwksp: reserving table %zd bytes, %zd bytes remaining", bytes, ZSTD_cwksp_available_space(ws) - bytes); assert((bytes & (sizeof(U32)-1)) == 0); ZSTD_cwksp_internal_advance_phase(ws, phase); @@ -248,7 +248,7 @@ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); void* start = ws->objectEnd; void* end = (BYTE*)start + roundedBytes; - DEBUGLOG(4, + DEBUGLOG(5, "cwksp: reserving object %zd bytes (rounded to %zd), %zd bytes remaining", bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); assert(((size_t)start & (sizeof(void*)-1)) == 0); From bc020eec928b09c2cf99712d5b4a00afa34470de Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 11 Sep 2019 11:40:57 -0400 Subject: [PATCH 077/144] Also Shrink Clean Table Area When Reducing Indices --- lib/compress/zstd_compress.c | 37 +++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 28341c0d..e0ffbc7c 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2320,7 +2320,11 @@ out: } -static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, ZSTD_CCtx_params const* params, void const* ip, void const* iend) +static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, + ZSTD_CCtx_params const* params, + void const* ip, + void const* iend) { if (ZSTD_window_needOverflowCorrection(ms->window, iend)) { U32 const maxDist = (U32)1 << params->cParams.windowLog; @@ -2329,7 +2333,9 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, ZSTD_CCtx_params ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + ZSTD_cwksp_mark_tables_dirty(ws); ZSTD_reduceIndex(ms, params, correction); + ZSTD_cwksp_mark_tables_clean(ws); if (ms->nextToUpdate < correction) ms->nextToUpdate = 0; else ms->nextToUpdate -= correction; /* invalidate dictionaries on overflow correction */ @@ -2372,7 +2378,8 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, "not enough space to store compressed block"); if (remaining < blockSize) blockSize = remaining; - ZSTD_overflowCorrectIfNeeded(ms, &cctx->appliedParams, ip, ip + blockSize); + ZSTD_overflowCorrectIfNeeded( + ms, &cctx->workspace, &cctx->appliedParams, ip, ip + blockSize); ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); /* Ensure hash/chain table insertion resumes no sooner than lowlimit */ @@ -2515,7 +2522,9 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, if (!frame) { /* overflow check and correction for block mode */ - ZSTD_overflowCorrectIfNeeded(ms, &cctx->appliedParams, src, (BYTE const*)src + srcSize); + ZSTD_overflowCorrectIfNeeded( + ms, &cctx->workspace, &cctx->appliedParams, + src, (BYTE const*)src + srcSize); } DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSize); @@ -2568,6 +2577,7 @@ size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const * @return : 0, or an error code */ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, ZSTD_CCtx_params const* params, const void* src, size_t srcSize, ZSTD_dictTableLoadMethod_e dtlm) @@ -2588,7 +2598,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, size_t const chunk = MIN(remaining, ZSTD_CHUNKSIZE_MAX); const BYTE* const ichunk = ip + chunk; - ZSTD_overflowCorrectIfNeeded(ms, params, ip, ichunk); + ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, ichunk); switch(params->cParams.strategy) { @@ -2651,6 +2661,7 @@ static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSym */ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, ZSTD_CCtx_params const* params, const void* dict, size_t dictSize, ZSTD_dictTableLoadMethod_e dtlm, @@ -2746,7 +2757,8 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, bs->entropy.fse.offcode_repeatMode = FSE_repeat_valid; bs->entropy.fse.matchlength_repeatMode = FSE_repeat_valid; bs->entropy.fse.litlength_repeatMode = FSE_repeat_valid; - FORWARD_IF_ERROR(ZSTD_loadDictionaryContent(ms, params, dictPtr, dictContentSize, dtlm)); + FORWARD_IF_ERROR(ZSTD_loadDictionaryContent( + ms, ws, params, dictPtr, dictContentSize, dtlm)); return dictID; } } @@ -2756,6 +2768,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, static size_t ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, const ZSTD_CCtx_params* params, const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, @@ -2769,19 +2782,21 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, /* dict restricted modes */ if (dictContentType == ZSTD_dct_rawContent) - return ZSTD_loadDictionaryContent(ms, params, dict, dictSize, dtlm); + return ZSTD_loadDictionaryContent(ms, ws, params, dict, dictSize, dtlm); if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) { if (dictContentType == ZSTD_dct_auto) { DEBUGLOG(4, "raw content dictionary detected"); - return ZSTD_loadDictionaryContent(ms, params, dict, dictSize, dtlm); + return ZSTD_loadDictionaryContent( + ms, ws, params, dict, dictSize, dtlm); } RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong); assert(0); /* impossible */ } /* dict as full zstd dictionary */ - return ZSTD_loadZstdDictionary(bs, ms, params, dict, dictSize, dtlm, workspace); + return ZSTD_loadZstdDictionary( + bs, ms, ws, params, dict, dictSize, dtlm, workspace); } /*! ZSTD_compressBegin_internal() : @@ -2807,7 +2822,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, ZSTDcrp_makeClean, zbuff) ); { size_t const dictID = ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, - params, dict, dictSize, dictContentType, dtlm, + &cctx->workspace, params, dict, dictSize, dictContentType, dtlm, cctx->entropyWorkspace); FORWARD_IF_ERROR(dictID); assert(dictID <= UINT_MAX); @@ -3082,8 +3097,8 @@ static size_t ZSTD_initCDict_internal( params.fParams.contentSizeFlag = 1; params.cParams = cParams; { size_t const dictID = ZSTD_compress_insertDictionary( - &cdict->cBlockState, &cdict->matchState, ¶ms, - cdict->dictContent, cdict->dictContentSize, + &cdict->cBlockState, &cdict->matchState, &cdict->workspace, + ¶ms, cdict->dictContent, cdict->dictContentSize, dictContentType, ZSTD_dtlm_full, cdict->entropyWorkspace); FORWARD_IF_ERROR(dictID); assert(dictID <= (size_t)(U32)-1); From b5b24c2a0d68aee159e01e8d0ad2f657f7c10b5c Mon Sep 17 00:00:00 2001 From: Dario Pavlovic Date: Wed, 11 Sep 2019 10:09:29 -0700 Subject: [PATCH 078/144] Combining fuzz_data_producer restrict calls into a single function --- tests/fuzz/block_round_trip.c | 3 +-- tests/fuzz/dictionary_decompress.c | 3 +-- tests/fuzz/dictionary_round_trip.c | 3 +-- tests/fuzz/fuzz_data_producer.c | 7 +++++++ tests/fuzz/fuzz_data_producer.h | 11 +++++++---- tests/fuzz/simple_compress.c | 3 +-- tests/fuzz/simple_decompress.c | 3 +-- tests/fuzz/simple_round_trip.c | 3 +-- tests/fuzz/stream_decompress.c | 3 +-- tests/fuzz/stream_round_trip.c | 3 +-- 10 files changed, 22 insertions(+), 20 deletions(-) diff --git a/tests/fuzz/block_round_trip.c b/tests/fuzz/block_round_trip.c index 9ead57a5..a0079d35 100644 --- a/tests/fuzz/block_round_trip.c +++ b/tests/fuzz/block_round_trip.c @@ -55,8 +55,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) /* Give a random portion of src data to the producer, to use for parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); - size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); - size = FUZZ_dataProducer_contract(producer, producerSliceSize); + size = FUZZ_dataProducer_reserveDataPrefix(producer); int cLevel = FUZZ_dataProducer_uint32(producer) % kMaxClevel; diff --git a/tests/fuzz/dictionary_decompress.c b/tests/fuzz/dictionary_decompress.c index 54903f1f..3bbd9bf5 100644 --- a/tests/fuzz/dictionary_decompress.c +++ b/tests/fuzz/dictionary_decompress.c @@ -27,8 +27,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) /* Give a random portion of src data to the producer, to use for parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); - size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); - size = FUZZ_dataProducer_contract(producer, producerSliceSize); + size = FUZZ_dataProducer_reserveDataPrefix(producer); FUZZ_dict_t dict; ZSTD_DDict* ddict = NULL; diff --git a/tests/fuzz/dictionary_round_trip.c b/tests/fuzz/dictionary_round_trip.c index cd1ca38f..5a4b9503 100644 --- a/tests/fuzz/dictionary_round_trip.c +++ b/tests/fuzz/dictionary_round_trip.c @@ -71,8 +71,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) /* Give a random portion of src data to the producer, to use for parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); - size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); - size = FUZZ_dataProducer_contract(producer, producerSliceSize); + size = FUZZ_dataProducer_reserveDataPrefix(producer); size_t const rBufSize = size; void* rBuf = malloc(rBufSize); diff --git a/tests/fuzz/fuzz_data_producer.c b/tests/fuzz/fuzz_data_producer.c index 6dcc1413..d6893e4f 100644 --- a/tests/fuzz/fuzz_data_producer.c +++ b/tests/fuzz/fuzz_data_producer.c @@ -65,3 +65,10 @@ size_t FUZZ_dataProducer_contract(FUZZ_dataProducer_t *producer, size_t newSize) producer->size = newSize; return remaining; } + +size_t FUZZ_dataProducer_reserveDataPrefix(FUZZ_dataProducer_t *producer) +{ + size_t producerSliceSize = FUZZ_dataProducer_uint32Range( + producer, 0, producer->size); + return FUZZ_dataProducer_contract(producer, producerSliceSize); +} diff --git a/tests/fuzz/fuzz_data_producer.h b/tests/fuzz/fuzz_data_producer.h index 668c87f2..8eea1e25 100644 --- a/tests/fuzz/fuzz_data_producer.h +++ b/tests/fuzz/fuzz_data_producer.h @@ -44,10 +44,13 @@ uint32_t FUZZ_dataProducer_uint32(FUZZ_dataProducer_t *producer); /* Returns the size of the remaining bytes of data in the producer */ size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer); -/* Tells the producer to contract to newSize bytes of data it currently uses, -counted from the end, and forget about the rest. If newSize > current data size, -nothing happens. Returns the number of bytes the producer won't use anymore, -after contracting. */ +/* Restricts the producer to only the last newSize bytes of data. +If newSize > current data size, nothing happens. Returns the number of bytes +the producer won't use anymore, after contracting. */ size_t FUZZ_dataProducer_contract(FUZZ_dataProducer_t *producer, size_t newSize); +/* Restricts the producer to use only the last X bytes of data, where X is + a random number in the interval [0, data_size]. Returns the size of the + remaining data the producer won't use anymore (the prefix). */ +size_t FUZZ_dataProducer_reserveDataPrefix(FUZZ_dataProducer_t *producer); #endif // FUZZ_DATA_PRODUCER_H diff --git a/tests/fuzz/simple_compress.c b/tests/fuzz/simple_compress.c index b8c6394c..74ab0d63 100644 --- a/tests/fuzz/simple_compress.c +++ b/tests/fuzz/simple_compress.c @@ -27,8 +27,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) /* Give a random portion of src data to the producer, to use for parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); - size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); - size = FUZZ_dataProducer_contract(producer, producerSliceSize); + size = FUZZ_dataProducer_reserveDataPrefix(producer); size_t const maxSize = ZSTD_compressBound(size); size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, maxSize); diff --git a/tests/fuzz/simple_decompress.c b/tests/fuzz/simple_decompress.c index 5c7680bf..6182746a 100644 --- a/tests/fuzz/simple_decompress.c +++ b/tests/fuzz/simple_decompress.c @@ -26,8 +26,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) /* Give a random portion of src data to the producer, to use for parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); - size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); - size = FUZZ_dataProducer_contract(producer, producerSliceSize); + size = FUZZ_dataProducer_reserveDataPrefix(producer); if (!dctx) { dctx = ZSTD_createDCtx(); diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c index f19e37a3..4a07d16a 100644 --- a/tests/fuzz/simple_round_trip.c +++ b/tests/fuzz/simple_round_trip.c @@ -55,8 +55,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) /* Give a random portion of src data to the producer, to use for parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); - size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); - size = FUZZ_dataProducer_contract(producer, producerSliceSize); + size = FUZZ_dataProducer_reserveDataPrefix(producer); /* Half of the time fuzz with a 1 byte smaller output size. * This will still succeed because we don't use a dictionary, so the dictID diff --git a/tests/fuzz/stream_decompress.c b/tests/fuzz/stream_decompress.c index 4d5c4996..70582e11 100644 --- a/tests/fuzz/stream_decompress.c +++ b/tests/fuzz/stream_decompress.c @@ -56,8 +56,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) /* Give a random portion of src data to the producer, to use for parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); - size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); - size = FUZZ_dataProducer_contract(producer, producerSliceSize); + size = FUZZ_dataProducer_reserveDataPrefix(producer); /* Allocate all buffers and contexts if not already allocated */ if (!buf) { diff --git a/tests/fuzz/stream_round_trip.c b/tests/fuzz/stream_round_trip.c index 02bb26da..08a4927a 100644 --- a/tests/fuzz/stream_round_trip.c +++ b/tests/fuzz/stream_round_trip.c @@ -128,8 +128,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) /* Give a random portion of src data to the producer, to use for parameter generation. The rest will be used for (de)compression */ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); - size_t producerSliceSize = FUZZ_dataProducer_uint32Range(producer, 0, size); - size = FUZZ_dataProducer_contract(producer, producerSliceSize); + size = FUZZ_dataProducer_reserveDataPrefix(producer); size_t neededBufSize; neededBufSize = ZSTD_compressBound(size) * 5; From 7c57e2b9cac597845265f877c8c866bf38b7ce01 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 11 Sep 2019 13:14:26 -0400 Subject: [PATCH 079/144] Zero `h3size` When `h3log` is 0 This led to a nasty edgecase, where index reduction for modes that don't use the h3 table would have a degenerate table (size 4) allocated and marked clean, but which would not be re-indexed. --- lib/compress/zstd_compress.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index e0ffbc7c..0d2e9440 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1301,7 +1301,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); size_t const hSize = ((size_t)1) << cParams->hashLog; U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; - size_t const h3Size = ((size_t)1) << hashLog3; + size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset); if (forceResetIndex == ZSTDirp_reset) { @@ -1658,7 +1658,8 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, } /* Zero the hashTable3, since the cdict never fills it */ - { size_t const h3Size = (size_t)1 << cctx->blockState.matchState.hashLog3; + { int const h3log = cctx->blockState.matchState.hashLog3; + size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; assert(cdict->matchState.hashLog3 == 0); memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32)); } @@ -1738,7 +1739,8 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, /* copy tables */ { size_t const chainSize = (srcCCtx->appliedParams.cParams.strategy == ZSTD_fast) ? 0 : ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog); size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog; - size_t const h3Size = (size_t)1 << srcCCtx->blockState.matchState.hashLog3; + int const h3log = srcCCtx->blockState.matchState.hashLog3; + size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); assert((U32*)dstCCtx->blockState.matchState.chainTable == (U32*)dstCCtx->blockState.matchState.hashTable + hSize); /* chainTable must follow hashTable */ assert((U32*)dstCCtx->blockState.matchState.hashTable3 == (U32*)dstCCtx->blockState.matchState.chainTable + chainSize); From ed4c2c60c342db07c16746188b71aad217f70f93 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 11 Sep 2019 13:17:19 -0400 Subject: [PATCH 080/144] Add Fuzzer Test Case for Index Reduction --- tests/fuzzer.c | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index fe656e6c..53bcb063 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -2150,6 +2150,78 @@ static int basicUnitTests(U32 const seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : table cleanliness through index reduction : ", testNb++); + { + int cLevel; + uint32_t approxIndex = 0; + uint32_t maxIndex = ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)); /* ZSTD_CURRENT_MAX from zstd_compress_internal.h */ + + /* vastly overprovision space in a static context so that we can do all + * this without ever reallocating, which would reset the indices */ + size_t const staticCCtxSize = 2 * ZSTD_estimateCCtxSize(22); + void* const staticCCtxBuffer = malloc(staticCCtxSize); + ZSTD_CCtx* cctx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize); + + /* bump the indices so the following compressions happen at high + * indices. */ + { + ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 }; + ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500)); + while (approxIndex <= (maxIndex / 4) * 3) { + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); + approxIndex += in.pos; + CHECK(in.pos == in.size); + in.pos = 0; + out.pos = 0; + } + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); + } + + /* spew a bunch of stuff into the table area */ + for (cLevel = 1; cLevel <= 22; cLevel++) { + ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 }; + ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel)); + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); + approxIndex += in.pos; + } + + /* now crank the indices so we overflow */ + { + ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 }; + ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500)); + while (approxIndex <= maxIndex) { + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); + approxIndex += in.pos; + CHECK(in.pos == in.size); + in.pos = 0; + out.pos = 0; + } + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); + } + + /* do a bunch of compressions again in low indices and ensure we don't + * hit untracked invalid indices */ + for (cLevel = 1; cLevel <= 22; cLevel++) { + ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 }; + ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel)); + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); + approxIndex += in.pos; + } + + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + _end: free(CNBuffer); free(compressedBuffer); From 5707c8a9d5f9b8ec89467862d42561e811c194a2 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 11 Sep 2019 13:23:59 -0400 Subject: [PATCH 081/144] Speed Up Test a Little --- tests/fuzzer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 53bcb063..cfe477d3 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -2181,7 +2181,7 @@ static int basicUnitTests(U32 const seed, double compressibility) /* spew a bunch of stuff into the table area */ for (cLevel = 1; cLevel <= 22; cLevel++) { - ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 }; + ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / cLevel, 0 }; ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel)); @@ -2209,7 +2209,7 @@ static int basicUnitTests(U32 const seed, double compressibility) /* do a bunch of compressions again in low indices and ensure we don't * hit untracked invalid indices */ for (cLevel = 1; cLevel <= 22; cLevel++) { - ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 }; + ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / cLevel, 0 }; ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel)); From ff67c62458869928680cb6c856299214f75d6d94 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 11 Sep 2019 13:59:09 -0400 Subject: [PATCH 082/144] Fix Compilation Error (`uint32_t` -> `size_t`) --- tests/fuzzer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index cfe477d3..0956b9c5 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -2153,8 +2153,8 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "test%3i : table cleanliness through index reduction : ", testNb++); { int cLevel; - uint32_t approxIndex = 0; - uint32_t maxIndex = ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)); /* ZSTD_CURRENT_MAX from zstd_compress_internal.h */ + size_t approxIndex = 0; + size_t maxIndex = ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)); /* ZSTD_CURRENT_MAX from zstd_compress_internal.h */ /* vastly overprovision space in a static context so that we can do all * this without ever reallocating, which would reset the indices */ From 194c542598a07ce13978d9306777e35bf4dcf3f7 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 11 Sep 2019 14:25:30 -0400 Subject: [PATCH 083/144] Fix Memory Leak in Test --- tests/fuzzer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 0956b9c5..0ae0b394 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -2219,6 +2219,7 @@ static int basicUnitTests(U32 const seed, double compressibility) } ZSTD_freeCCtx(cctx); + free(staticCCtxBuffer); } DISPLAYLEVEL(3, "OK \n"); From a10c1916130625b9a9cb2534a0a09109bcdbd2a9 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 11 Sep 2019 16:40:29 -0400 Subject: [PATCH 084/144] `__msan_poison()` Workspace When Preparing for Re-Use --- lib/common/mem.h | 17 +++++++++++++++++ lib/compress/zstd_compress.c | 14 ++++++++++++++ lib/compress/zstd_cwksp.h | 24 ++++++++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/lib/common/mem.h b/lib/common/mem.h index c10d7f61..3dc4f428 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -47,6 +47,23 @@ extern "C" { #define MEM_STATIC_ASSERT(c) { enum { MEM_static_assert = 1/(int)(!!(c)) }; } MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } +/* detects whether we are being compiled under msan */ +#if defined (__has_feature) +# if __has_feature(memory_sanitizer) +# define MEMORY_SANITIZER 1 +# endif +#endif + +#if defined (MEMORY_SANITIZER) +# include +#endif + +#if defined (MEMORY_SANITIZER) +# define MEM_SKIP_MSAN __attribute__((no_sanitize("memory"))) +#else +# define MEM_SKIP_MSAN +#endif + /*-************************************************************** * Basic Types diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 0d2e9440..2f0736b2 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1799,6 +1799,20 @@ ZSTD_reduceTable_internal (U32* const table, U32 const size, U32 const reducerVa int rowNb; assert((size & (ZSTD_ROWSIZE-1)) == 0); /* multiple of ZSTD_ROWSIZE */ assert(size < (1U<<31)); /* can be casted to int */ + +#if defined (MEMORY_SANITIZER) && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) + /* To validate that the table re-use logic is sound, and that we don't + * access table space that we haven't cleaned, we re-"poison" the table + * space every time we mark it dirty. + * + * This function however is intended to operate on those dirty tables and + * re-clean them. So when this function is used correctly, we can unpoison + * the memory it operated on. This introduces a blind spot though, since + * if we now try to operate on __actually__ poisoned memory, we will not + * detect that. */ + __msan_unpoison(table, size * sizeof(U32)); +#endif + for (rowNb=0 ; rowNb < nbRows ; rowNb++) { int column; for (column=0; columntableValidEnd - (BYTE*)ws->objectEnd; + assert(__msan_test_shadow(ws->objectEnd, size) == -1); + __msan_poison(ws->objectEnd, size); + } +#endif + assert(ws->tableValidEnd >= ws->objectEnd); assert(ws->tableValidEnd <= ws->allocStart); ws->tableValidEnd = ws->objectEnd; @@ -309,6 +321,18 @@ MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { */ MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { DEBUGLOG(4, "cwksp: clearing!"); + +#if defined (MEMORY_SANITIZER) && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) + /* To validate that the context re-use logic is sound, and that we don't + * access stuff that this compression hasn't initialized, we re-"poison" + * the workspace (or at least the non-static, non-table parts of it) + * every time we start a new compression. */ + { + size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->tableValidEnd; + __msan_poison(ws->tableValidEnd, size); + } +#endif + ws->tableEnd = ws->objectEnd; ws->allocStart = ws->workspaceEnd; ws->allocFailed = 0; From 51d90668ba4f3f23bc09f2b603290691be30f77e Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 11 Sep 2019 16:41:41 -0400 Subject: [PATCH 085/144] Add Assertions to Confirm that Workspace Pointers are Correctly Ordered --- lib/compress/zstd_cwksp.h | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 796b8542..98499b86 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -147,6 +147,16 @@ typedef struct { MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws); +MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) { + (void)ws; + assert(ws->workspace <= ws->objectEnd); + assert(ws->objectEnd <= ws->tableEnd); + assert(ws->objectEnd <= ws->tableValidEnd); + assert(ws->tableEnd <= ws->allocStart); + assert(ws->tableValidEnd <= ws->allocStart); + assert(ws->allocStart <= ws->workspaceEnd); +} + /** * Align must be a power of 2. */ @@ -190,8 +200,10 @@ MEM_STATIC void* ZSTD_cwksp_reserve_internal( alloc = (BYTE *)ws->allocStart - bytes; DEBUGLOG(5, "cwksp: reserving %zd bytes, %zd bytes remaining", bytes, ZSTD_cwksp_available_space(ws) - bytes); + ZSTD_cwksp_assert_internal_consistency(ws); assert(alloc >= bottom); if (alloc < bottom) { + DEBUGLOG(4, "cwksp: alloc failed!"); ws->allocFailed = 1; return NULL; } @@ -231,9 +243,10 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { bytes, ZSTD_cwksp_available_space(ws) - bytes); assert((bytes & (sizeof(U32)-1)) == 0); ZSTD_cwksp_internal_advance_phase(ws, phase); + ZSTD_cwksp_assert_internal_consistency(ws); assert(end <= top); if (end > top) { - DEBUGLOG(4, "cwksp: object alloc failed!"); + DEBUGLOG(4, "cwksp: table alloc failed!"); ws->allocFailed = 1; return NULL; } @@ -253,6 +266,7 @@ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); assert(((size_t)start & (sizeof(void*)-1)) == 0); assert((bytes & (sizeof(void*)-1)) == 0); + ZSTD_cwksp_assert_internal_consistency(ws); /* we must be in the first phase, no advance is possible */ if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { DEBUGLOG(4, "cwksp: object alloc failed!"); @@ -282,6 +296,7 @@ MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) { assert(ws->tableValidEnd >= ws->objectEnd); assert(ws->tableValidEnd <= ws->allocStart); ws->tableValidEnd = ws->objectEnd; + ZSTD_cwksp_assert_internal_consistency(ws); } MEM_STATIC void ZSTD_cwksp_mark_tables_clean(ZSTD_cwksp* ws) { @@ -291,6 +306,7 @@ MEM_STATIC void ZSTD_cwksp_mark_tables_clean(ZSTD_cwksp* ws) { if (ws->tableValidEnd < ws->tableEnd) { ws->tableValidEnd = ws->tableEnd; } + ZSTD_cwksp_assert_internal_consistency(ws); } /** @@ -313,6 +329,7 @@ MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) { MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { DEBUGLOG(4, "cwksp: clearing tables!"); ws->tableEnd = ws->objectEnd; + ZSTD_cwksp_assert_internal_consistency(ws); } /** @@ -339,6 +356,7 @@ MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { if (ws->phase > ZSTD_cwksp_alloc_buffers) { ws->phase = ZSTD_cwksp_alloc_buffers; } + ZSTD_cwksp_assert_internal_consistency(ws); } /** @@ -356,6 +374,7 @@ MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) { ws->phase = ZSTD_cwksp_alloc_objects; ZSTD_cwksp_clear(ws); ws->workspaceOversizedDuration = 0; + ZSTD_cwksp_assert_internal_consistency(ws); } MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { @@ -369,9 +388,7 @@ MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { DEBUGLOG(4, "cwksp: freeing workspace"); ZSTD_free(ws->workspace, customMem); - ws->workspace = NULL; - ws->workspaceEnd = NULL; - ZSTD_cwksp_clear(ws); + memset(ws, 0, sizeof(ZSTD_cwksp)); } /** From 20c69077d1ce197cb6cf7f1bf8f581fb98607b71 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 11 Sep 2019 17:03:09 -0400 Subject: [PATCH 086/144] Shrink Table Valid End During Alloc Alignment / Phase Change --- lib/compress/zstd_cwksp.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 98499b86..39d064c4 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -184,6 +184,9 @@ MEM_STATIC void ZSTD_cwksp_internal_advance_phase( * by a larger margin than the space that will be consumed. */ /* TODO: cleaner, compiler warning friendly way to do this??? */ ws->allocStart = (BYTE*)ws->allocStart - ((size_t)ws->allocStart & (sizeof(U32)-1)); + if (ws->allocStart < ws->tableValidEnd) { + ws->tableValidEnd = ws->allocStart; + } } ws->phase = phase; } From 5a9baae9cfa9b5684f553fc9a03a7becee16287e Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 12 Sep 2019 11:48:46 -0400 Subject: [PATCH 087/144] Set M-Time Nanoseconds --- programs/util.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/programs/util.c b/programs/util.c index b05ac3e9..3e4c3335 100644 --- a/programs/util.c +++ b/programs/util.c @@ -57,7 +57,8 @@ int UTIL_setFileStat(const char *filename, stat_t *statbuf) #if defined(_WIN32) || (PLATFORM_POSIX_VERSION < 200809L) struct utimbuf timebuf; #else - struct timespec timebuf[2] = {}; + /* (atime, mtime) */ + struct timespec timebuf[2] = { {0, UTIME_NOW}, statbuf->st_mtim }; #endif if (!UTIL_isRegularFile(filename)) @@ -68,8 +69,6 @@ int UTIL_setFileStat(const char *filename, stat_t *statbuf) timebuf.modtime = statbuf->st_mtime; res += utime(filename, &timebuf); /* set access and modification times */ #else - timebuf[0].tv_nsec = UTIME_NOW; - timebuf[1].tv_sec = statbuf->st_mtime; res += utimensat(AT_FDCWD, filename, timebuf, 0); /* set access and modification times */ #endif From e0fb7e1eafc7fa2cd96220df0d68237dadd2ca3a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 12 Sep 2019 09:39:15 -0700 Subject: [PATCH 088/144] ignore dictionary artifacts --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4c297053..acd9552b 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ zstdmt # Test artefacts tmp* dictionary. +dictionary NUL # Build artefacts From 92c58c4d5d36f6ecbd2f3d2be8d7a4dc7d4d3900 Mon Sep 17 00:00:00 2001 From: Dario Pavlovic Date: Thu, 12 Sep 2019 12:40:12 -0700 Subject: [PATCH 089/144] Use range instead of the generic uint32 method to use less bytes when generating necessary numbers. --- tests/fuzz/block_round_trip.c | 5 ++--- tests/fuzz/dictionary_decompress.c | 3 +-- tests/fuzz/dictionary_round_trip.c | 6 ++---- tests/fuzz/fuzz_data_producer.c | 11 +++++++++++ tests/fuzz/fuzz_data_producer.h | 4 ++++ tests/fuzz/simple_compress.c | 6 +++--- tests/fuzz/simple_round_trip.c | 8 +++----- tests/fuzz/stream_decompress.c | 4 ++-- tests/fuzz/stream_round_trip.c | 8 ++++---- tests/fuzz/zstd_helpers.c | 3 +++ tests/fuzz/zstd_helpers.h | 4 +++- 11 files changed, 38 insertions(+), 24 deletions(-) diff --git a/tests/fuzz/block_round_trip.c b/tests/fuzz/block_round_trip.c index a0079d35..0f948d4f 100644 --- a/tests/fuzz/block_round_trip.c +++ b/tests/fuzz/block_round_trip.c @@ -20,10 +20,9 @@ #include #include "fuzz_helpers.h" #include "zstd.h" +#include "zstd_helpers.h" #include "fuzz_data_producer.h" -static const int kMaxClevel = 19; - static ZSTD_CCtx *cctx = NULL; static ZSTD_DCtx *dctx = NULL; static void* cBuf = NULL; @@ -57,7 +56,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); size = FUZZ_dataProducer_reserveDataPrefix(producer); - int cLevel = FUZZ_dataProducer_uint32(producer) % kMaxClevel; + int const cLevel = FUZZ_dataProducer_int32Range(producer, kMinClevel, kMaxClevel); size_t neededBufSize = size; if (size > ZSTD_BLOCKSIZE_MAX) diff --git a/tests/fuzz/dictionary_decompress.c b/tests/fuzz/dictionary_decompress.c index 3bbd9bf5..9cc69fa3 100644 --- a/tests/fuzz/dictionary_decompress.c +++ b/tests/fuzz/dictionary_decompress.c @@ -31,7 +31,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) FUZZ_dict_t dict; ZSTD_DDict* ddict = NULL; - int i; if (!dctx) { dctx = ZSTD_createDCtx(); @@ -49,7 +48,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) } { - size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 2 * size); + size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 10 * size); void* rBuf = malloc(bufSize); FUZZ_ASSERT(rBuf); if (ddict) { diff --git a/tests/fuzz/dictionary_round_trip.c b/tests/fuzz/dictionary_round_trip.c index 5a4b9503..9411b50a 100644 --- a/tests/fuzz/dictionary_round_trip.c +++ b/tests/fuzz/dictionary_round_trip.c @@ -21,8 +21,6 @@ #include "zstd_helpers.h" #include "fuzz_data_producer.h" -static const int kMaxClevel = 19; - static ZSTD_CCtx *cctx = NULL; static ZSTD_DCtx *dctx = NULL; @@ -34,8 +32,8 @@ static size_t roundTripTest(void *result, size_t resultCapacity, ZSTD_dictContentType_e dictContentType = ZSTD_dct_auto; FUZZ_dict_t dict = FUZZ_train(src, srcSize, producer); size_t cSize; - if ((FUZZ_dataProducer_uint32(producer) & 15) == 0) { - int const cLevel = FUZZ_dataProducer_uint32(producer) % kMaxClevel; + if (FUZZ_dataProducer_uint32Range(producer, 0, 15) == 0) { + int const cLevel = FUZZ_dataProducer_int32Range(producer, kMinClevel, kMaxClevel); cSize = ZSTD_compress_usingDict(cctx, compressed, compressedCapacity, diff --git a/tests/fuzz/fuzz_data_producer.c b/tests/fuzz/fuzz_data_producer.c index d6893e4f..b465337e 100644 --- a/tests/fuzz/fuzz_data_producer.c +++ b/tests/fuzz/fuzz_data_producer.c @@ -52,6 +52,17 @@ uint32_t FUZZ_dataProducer_uint32(FUZZ_dataProducer_t *producer) { return FUZZ_dataProducer_uint32Range(producer, 0, 0xffffffff); } +int32_t FUZZ_dataProducer_int32Range(FUZZ_dataProducer_t *producer, + int32_t min, int32_t max) +{ + FUZZ_ASSERT(min <= max); + + if (min < 0) + return (int)FUZZ_dataProducer_uint32Range(producer, 0, max - min) + min; + + return FUZZ_dataProducer_uint32Range(producer, min, max); +} + size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer){ return producer->size; } diff --git a/tests/fuzz/fuzz_data_producer.h b/tests/fuzz/fuzz_data_producer.h index 8eea1e25..8153c066 100644 --- a/tests/fuzz/fuzz_data_producer.h +++ b/tests/fuzz/fuzz_data_producer.h @@ -41,6 +41,10 @@ uint32_t FUZZ_dataProducer_uint32Range(FUZZ_dataProducer_t *producer, uint32_t m /* Returns a uint32 value */ uint32_t FUZZ_dataProducer_uint32(FUZZ_dataProducer_t *producer); +/* Returns a signed value between [min, ,max] */ +int32_t FUZZ_dataProducer_int32Range(FUZZ_dataProducer_t *producer, + int32_t min, int32_t max); + /* Returns the size of the remaining bytes of data in the producer */ size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer); diff --git a/tests/fuzz/simple_compress.c b/tests/fuzz/simple_compress.c index 74ab0d63..487be3a3 100644 --- a/tests/fuzz/simple_compress.c +++ b/tests/fuzz/simple_compress.c @@ -18,6 +18,7 @@ #include #include "fuzz_helpers.h" #include "zstd.h" +#include "zstd_helpers.h" #include "fuzz_data_producer.h" static ZSTD_CCtx *cctx = NULL; @@ -32,8 +33,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) size_t const maxSize = ZSTD_compressBound(size); size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, maxSize); - int const level = (int)FUZZ_dataProducer_uint32Range( - producer, 0, 19 + 3) - 3; /* [-3, 19] */ + int const cLevel = FUZZ_dataProducer_int32Range(producer, kMinClevel, kMaxClevel); if (!cctx) { cctx = ZSTD_createCCtx(); @@ -42,7 +42,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) void *rBuf = malloc(bufSize); FUZZ_ASSERT(rBuf); - ZSTD_compressCCtx(cctx, rBuf, bufSize, src, size, level); + ZSTD_compressCCtx(cctx, rBuf, bufSize, src, size, cLevel); free(rBuf); FUZZ_dataProducer_free(producer); #ifndef STATEFUL_FUZZING diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c index 4a07d16a..2d1d0598 100644 --- a/tests/fuzz/simple_round_trip.c +++ b/tests/fuzz/simple_round_trip.c @@ -22,8 +22,6 @@ #include "zstd_helpers.h" #include "fuzz_data_producer.h" -static const int kMaxClevel = 19; - static ZSTD_CCtx *cctx = NULL; static ZSTD_DCtx *dctx = NULL; @@ -33,11 +31,12 @@ static size_t roundTripTest(void *result, size_t resultCapacity, FUZZ_dataProducer_t *producer) { size_t cSize; - if (FUZZ_dataProducer_uint32(producer) & 1) { + if (FUZZ_dataProducer_uint32Range(producer, 0, 1)) { FUZZ_setRandomParameters(cctx, srcSize, producer); cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize); } else { - int const cLevel = FUZZ_dataProducer_uint32(producer) % kMaxClevel; + int const cLevel = FUZZ_dataProducer_int32Range(producer, kMinClevel, kMaxClevel); + cSize = ZSTD_compressCCtx( cctx, compressed, compressedCapacity, src, srcSize, cLevel); } @@ -62,7 +61,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) * field is empty, giving us 4 bytes of overhead. */ cBufSize -= FUZZ_dataProducer_uint32Range(producer, 0, 1); - size = FUZZ_dataProducer_remainingBytes(producer); cBuf = malloc(cBufSize); diff --git a/tests/fuzz/stream_decompress.c b/tests/fuzz/stream_decompress.c index 70582e11..c71cc9d3 100644 --- a/tests/fuzz/stream_decompress.c +++ b/tests/fuzz/stream_decompress.c @@ -31,7 +31,7 @@ static ZSTD_outBuffer makeOutBuffer(FUZZ_dataProducer_t *producer) { ZSTD_outBuffer buffer = { buf, 0, 0 }; - buffer.size = (FUZZ_dataProducer_uint32(producer) % kBufSize) + 1; + buffer.size = (FUZZ_dataProducer_uint32Range(producer, 1, kBufSize)); FUZZ_ASSERT(buffer.size <= kBufSize); return buffer; @@ -43,7 +43,7 @@ static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size, ZSTD_inBuffer buffer = { *src, 0, 0 }; FUZZ_ASSERT(*size > 0); - buffer.size = (FUZZ_dataProducer_uint32(producer) % *size) + 1; + buffer.size = (FUZZ_dataProducer_uint32Range(producer, 1, *size)); FUZZ_ASSERT(buffer.size <= *size); *src += buffer.size; *size -= buffer.size; diff --git a/tests/fuzz/stream_round_trip.c b/tests/fuzz/stream_round_trip.c index 08a4927a..c534a904 100644 --- a/tests/fuzz/stream_round_trip.c +++ b/tests/fuzz/stream_round_trip.c @@ -34,7 +34,7 @@ static ZSTD_outBuffer makeOutBuffer(uint8_t *dst, size_t capacity, ZSTD_outBuffer buffer = { dst, 0, 0 }; FUZZ_ASSERT(capacity > 0); - buffer.size = (FUZZ_dataProducer_uint32(producer) % capacity) + 1; + buffer.size = (FUZZ_dataProducer_uint32Range(producer, 1, capacity)); FUZZ_ASSERT(buffer.size <= capacity); return buffer; @@ -46,7 +46,7 @@ static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size, ZSTD_inBuffer buffer = { *src, 0, 0 }; FUZZ_ASSERT(*size > 0); - buffer.size = (FUZZ_dataProducer_uint32(producer) % *size) + 1; + buffer.size = (FUZZ_dataProducer_uint32Range(producer, 1, *size)); FUZZ_ASSERT(buffer.size <= *size); *src += buffer.size; *size -= buffer.size; @@ -69,7 +69,7 @@ static size_t compress(uint8_t *dst, size_t capacity, while (in.pos < in.size || mode != -1) { ZSTD_outBuffer out = makeOutBuffer(dst, capacity, producer); /* Previous action finished, pick a new mode. */ - if (mode == -1) mode = FUZZ_dataProducer_uint32(producer) % 10; + if (mode == -1) mode = FUZZ_dataProducer_uint32Range(producer, 0, 9); switch (mode) { case 0: /* fall-through */ case 1: /* fall-through */ @@ -88,7 +88,7 @@ static size_t compress(uint8_t *dst, size_t capacity, /* Reset the compressor when the frame is finished */ if (ret == 0) { ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); - if ((FUZZ_dataProducer_uint32(producer) & 7) == 0) { + if (FUZZ_dataProducer_uint32Range(producer, 0, 7) == 0) { size_t const remaining = in.size - in.pos; FUZZ_setRandomParameters(cctx, remaining, producer); } diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c index 2635de4c..90bf1a15 100644 --- a/tests/fuzz/zstd_helpers.c +++ b/tests/fuzz/zstd_helpers.c @@ -17,6 +17,9 @@ #include "zstd.h" #include "zdict.h" +const int kMinClevel = -3; +const int kMaxClevel = 19; + static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, int value) { FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value)); diff --git a/tests/fuzz/zstd_helpers.h b/tests/fuzz/zstd_helpers.h index f2001f8b..2210bcaf 100644 --- a/tests/fuzz/zstd_helpers.h +++ b/tests/fuzz/zstd_helpers.h @@ -24,6 +24,9 @@ extern "C" { #endif +extern const int kMinClevel; +extern const int kMaxClevel; + void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer_t *producer); ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, FUZZ_dataProducer_t *producer); @@ -41,7 +44,6 @@ typedef struct { */ FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, FUZZ_dataProducer_t *producer); - #ifdef __cplusplus } #endif From 47bb4c6a23439813b2d20f668f69313d519304d7 Mon Sep 17 00:00:00 2001 From: Dario Pavlovic Date: Thu, 12 Sep 2019 12:45:28 -0700 Subject: [PATCH 090/144] Update tests/fuzz/fuzz_data_producer.h --- tests/fuzz/fuzz_data_producer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fuzz/fuzz_data_producer.h b/tests/fuzz/fuzz_data_producer.h index 8153c066..f2b60967 100644 --- a/tests/fuzz/fuzz_data_producer.h +++ b/tests/fuzz/fuzz_data_producer.h @@ -41,7 +41,7 @@ uint32_t FUZZ_dataProducer_uint32Range(FUZZ_dataProducer_t *producer, uint32_t m /* Returns a uint32 value */ uint32_t FUZZ_dataProducer_uint32(FUZZ_dataProducer_t *producer); -/* Returns a signed value between [min, ,max] */ +/* Returns a signed value between [min, max] */ int32_t FUZZ_dataProducer_int32Range(FUZZ_dataProducer_t *producer, int32_t min, int32_t max); From cd8588077e4119ad9a328463b760edd59b529430 Mon Sep 17 00:00:00 2001 From: Dario Pavlovic Date: Thu, 12 Sep 2019 13:10:34 -0700 Subject: [PATCH 091/144] It's time for all of rng seed code to go. Goodbye --- tests/fuzz/fuzz.h | 10 ---------- tests/fuzz/fuzz.py | 2 -- tests/fuzz/fuzz_helpers.h | 31 ------------------------------- 3 files changed, 43 deletions(-) diff --git a/tests/fuzz/fuzz.h b/tests/fuzz/fuzz.h index 8850025b..6d53aa6d 100644 --- a/tests/fuzz/fuzz.h +++ b/tests/fuzz/fuzz.h @@ -17,12 +17,6 @@ * test code paths which are only executed when contexts are reused. * WARNING: Makes reproducing crashes much harder. * Default: Not defined. - * @param FUZZ_RNG_SEED_SIZE: - * The number of bytes of the source to look at when constructing a seed - * for the deterministic RNG. These bytes are discarded before passing - * the data to zstd functions. Every fuzzer initializes the RNG exactly - * once before doing anything else, even if it is unused. - * Default: 4. * @param DEBUGLEVEL: * This is a parameter for the zstd library. Defining `DEBUGLEVEL=1` * enables assert() statements in the zstd library. Higher levels enable @@ -42,10 +36,6 @@ #ifndef FUZZ_H #define FUZZ_H -#ifndef FUZZ_RNG_SEED_SIZE -# define FUZZ_RNG_SEED_SIZE 4 -#endif - #include #include diff --git a/tests/fuzz/fuzz.py b/tests/fuzz/fuzz.py index faf8ce8a..c790c837 100755 --- a/tests/fuzz/fuzz.py +++ b/tests/fuzz/fuzz.py @@ -740,10 +740,8 @@ def gen(args): for name in os.listdir(samples): samplename = abs_join(samples, name) outname = abs_join(seed, name) - rng_seed = os.urandom(args.fuzz_rng_seed_size) with open(samplename, 'rb') as sample: with open(outname, 'wb') as out: - out.write(rng_seed) CHUNK_SIZE = 131072 chunk = sample.read(CHUNK_SIZE) while len(chunk) > 0: diff --git a/tests/fuzz/fuzz_helpers.h b/tests/fuzz/fuzz_helpers.h index 0ee85fc7..3de917fd 100644 --- a/tests/fuzz/fuzz_helpers.h +++ b/tests/fuzz/fuzz_helpers.h @@ -55,37 +55,6 @@ extern "C" { #define FUZZ_STATIC static #endif -/** - * Deterministically constructs a seed based on the fuzz input. - * Consumes up to the first FUZZ_RNG_SEED_SIZE bytes of the input. - */ -FUZZ_STATIC uint32_t FUZZ_seed(uint8_t const **src, size_t* size) { - uint8_t const *data = *src; - size_t const toHash = MIN(FUZZ_RNG_SEED_SIZE, *size); - *size -= toHash; - *src += toHash; - return XXH32(data, toHash, 0); -} - -#define FUZZ_rotl32(x, r) (((x) << (r)) | ((x) >> (32 - (r)))) - -FUZZ_STATIC uint32_t FUZZ_rand(uint32_t *state) { - static const uint32_t prime1 = 2654435761U; - static const uint32_t prime2 = 2246822519U; - uint32_t rand32 = *state; - rand32 *= prime1; - rand32 += prime2; - rand32 = FUZZ_rotl32(rand32, 13); - *state = rand32; - return rand32 >> 5; -} - -/* Returns a random numer in the range [min, max]. */ -FUZZ_STATIC uint32_t FUZZ_rand32(uint32_t *state, uint32_t min, uint32_t max) { - uint32_t random = FUZZ_rand(state); - return min + (random % (max - min + 1)); -} - #ifdef __cplusplus } #endif From e1ec8004ccfb9666aa720cf6931c265481930074 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 12 Sep 2019 16:27:05 -0400 Subject: [PATCH 092/144] Formatting and Clean Up --- programs/util.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/programs/util.c b/programs/util.c index 3e4c3335..3988295d 100644 --- a/programs/util.c +++ b/programs/util.c @@ -54,22 +54,24 @@ int UTIL_getFileStat(const char* infilename, stat_t *statbuf) int UTIL_setFileStat(const char *filename, stat_t *statbuf) { int res = 0; -#if defined(_WIN32) || (PLATFORM_POSIX_VERSION < 200809L) - struct utimbuf timebuf; -#else - /* (atime, mtime) */ - struct timespec timebuf[2] = { {0, UTIME_NOW}, statbuf->st_mtim }; -#endif if (!UTIL_isRegularFile(filename)) return -1; + /* set access and modification times */ #if defined(_WIN32) || (PLATFORM_POSIX_VERSION < 200809L) - timebuf.actime = time(NULL); - timebuf.modtime = statbuf->st_mtime; - res += utime(filename, &timebuf); /* set access and modification times */ + { + struct utimbuf timebuf; + timebuf.actime = time(NULL); + timebuf.modtime = statbuf->st_mtime; + res += utime(filename, &timebuf); + } #else - res += utimensat(AT_FDCWD, filename, timebuf, 0); /* set access and modification times */ + { + /* (atime, mtime) */ + struct timespec timebuf[2] = { {0, UTIME_NOW}, statbuf->st_mtim }; + res += utimensat(AT_FDCWD, filename, timebuf, 0); + } #endif #if !defined(_WIN32) From 43da5bf27e707286ba8679c5f10a109b8a3f1447 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 12 Sep 2019 14:43:50 -0700 Subject: [PATCH 093/144] Rearranging assert and allowing 4 extra for FSE_BLOCKBOUND() --- contrib/linux-kernel/lib/zstd/fse.h | 2 +- lib/common/bitstream.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/linux-kernel/lib/zstd/fse.h b/contrib/linux-kernel/lib/zstd/fse.h index 7460ab04..a694199f 100644 --- a/contrib/linux-kernel/lib/zstd/fse.h +++ b/contrib/linux-kernel/lib/zstd/fse.h @@ -232,7 +232,7 @@ If there is an error, the function will return an error code, which can be teste *******************************************/ /* FSE buffer bounds */ #define FSE_NCOUNTBOUND 512 -#define FSE_BLOCKBOUND(size) (size + (size >> 7)) +#define FSE_BLOCKBOUND(size) (size + (size >> 7) + 4 /* constant for initial fse states */ ) #define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ /* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ diff --git a/lib/common/bitstream.h b/lib/common/bitstream.h index 7bdb0604..fb10cca3 100644 --- a/lib/common/bitstream.h +++ b/lib/common/bitstream.h @@ -244,9 +244,9 @@ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) { size_t const nbBytes = bitC->bitPos >> 3; assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + assert(bitC->ptr <= bitC->endPtr); MEM_writeLEST(bitC->ptr, bitC->bitContainer); bitC->ptr += nbBytes; - assert(bitC->ptr <= bitC->endPtr); bitC->bitPos &= 7; bitC->bitContainer >>= nbBytes*8; } From 7c4578160e4c7cc6ddf308a534175956908af387 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 12 Sep 2019 15:02:01 -0700 Subject: [PATCH 094/144] [fuzz] Generate seed data up to 256KB --- tests/fuzz/fuzz.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fuzz/fuzz.py b/tests/fuzz/fuzz.py index c790c837..9df68df0 100755 --- a/tests/fuzz/fuzz.py +++ b/tests/fuzz/fuzz.py @@ -660,7 +660,7 @@ def gen_parser(args): parser.add_argument( '--max-size-log', type=int, - default=13, + default=18, help='Maximum sample size to generate') parser.add_argument( '--seed', @@ -720,7 +720,7 @@ def gen(args): if info.frame_type == FrameType.BLOCK: cmd += [ '--gen-blocks', - '--max-block-size-log={}'.format(args.max_size_log) + '--max-block-size-log={}'.format(min(args.max_size_log, 17)) ] else: cmd += ['--max-content-size-log={}'.format(args.max_size_log)] From fe9af338ed9cb1e1b3c2a6a5a219156ad1993e6a Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 12 Sep 2019 15:35:27 -0700 Subject: [PATCH 095/144] Added assert to BIT_flushBits() --- lib/common/bitstream.h | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/common/bitstream.h b/lib/common/bitstream.h index fb10cca3..306e019c 100644 --- a/lib/common/bitstream.h +++ b/lib/common/bitstream.h @@ -260,6 +260,7 @@ MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) { size_t const nbBytes = bitC->bitPos >> 3; assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + assert(bitC->ptr <= bitC->endPtr); MEM_writeLEST(bitC->ptr, bitC->bitContainer); bitC->ptr += nbBytes; if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; From d721fcf3eecbda37b03209bbfd8b07d684956fb3 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 13 Sep 2019 10:32:38 -0700 Subject: [PATCH 096/144] [fuzz] Fix leak in block_round_trip --- tests/fuzz/block_round_trip.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fuzz/block_round_trip.c b/tests/fuzz/block_round_trip.c index 0f948d4f..89f060a6 100644 --- a/tests/fuzz/block_round_trip.c +++ b/tests/fuzz/block_round_trip.c @@ -60,7 +60,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) size_t neededBufSize = size; if (size > ZSTD_BLOCKSIZE_MAX) - return 0; + size = ZSTD_BLOCKSIZE_MAX; /* Allocate all buffers and contexts if not already allocated */ if (neededBufSize > bufSize || !cBuf || !rBuf) { From fbeaf6989e12491cca83f3379e41ea0529ac5f0c Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Sun, 15 Sep 2019 12:41:24 -0700 Subject: [PATCH 097/144] [libzstd] Improve advanced API docs --- lib/zstd.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/zstd.h b/lib/zstd.h index 38c99e01..52266ab5 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1641,7 +1641,10 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dic /**! ZSTD_initCStream_advanced() : * This function is deprecated, and is approximately equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_setZstdParams(zcs, params); // Set the zstd params and leave the rest as-is + * // Pseudocode: Set each zstd parameter and leave the rest as-is. + * for ((param, value) : params) { + * ZSTD_CCtx_setParameter(zcs, param, value); + * } * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); * @@ -1661,7 +1664,10 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDi /**! ZSTD_initCStream_usingCDict_advanced() : * This function is deprecated, and is approximately equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_setZstdFrameParams(zcs, fParams); // Set the zstd frame params and leave the rest as-is + * // Pseudocode: Set each zstd frame parameter and leave the rest as-is. + * for ((fParam, value) : fParams) { + * ZSTD_CCtx_setParameter(zcs, fParam, value); + * } * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); * ZSTD_CCtx_refCDict(zcs, cdict); * From 4607f80f5adba93b429cc801d62604a55678bf54 Mon Sep 17 00:00:00 2001 From: Dmitri Shubin Date: Mon, 16 Sep 2019 11:36:37 +0300 Subject: [PATCH 098/144] Don't hardcode installation directories in CMakeLists.txt Use paths provided by GNUInstallDirs module instead. --- build/cmake/lib/CMakeLists.txt | 8 ++++---- build/cmake/programs/CMakeLists.txt | 26 ++++++++++++++------------ 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index 5c0802bc..77b389ca 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -133,8 +133,8 @@ endif () if (UNIX) # pkg-config set(PREFIX "${CMAKE_INSTALL_PREFIX}") - set(LIBDIR "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}") - set(INCLUDEDIR "${CMAKE_INSTALL_PREFIX}/include") + set(LIBDIR "${CMAKE_INSTALL_FULL_LIBDIR}") + set(INCLUDEDIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}") set(VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}") add_custom_target(libzstd.pc ALL ${CMAKE_COMMAND} -DIN="${LIBRARY_DIR}/libzstd.pc.in" -DOUT="libzstd.pc" @@ -152,10 +152,10 @@ install(FILES ${LIBRARY_DIR}/dictBuilder/zdict.h ${LIBRARY_DIR}/dictBuilder/cover.h ${LIBRARY_DIR}/common/zstd_errors.h - DESTINATION "include") + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") if (ZSTD_BUILD_SHARED) - install(TARGETS libzstd_shared RUNTIME DESTINATION "bin" + install(TARGETS libzstd_shared RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}") endif() diff --git a/build/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt index 50408bd9..8afd8350 100644 --- a/build/cmake/programs/CMakeLists.txt +++ b/build/cmake/programs/CMakeLists.txt @@ -31,15 +31,15 @@ target_link_libraries(zstd libzstd_static) if (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)") target_link_libraries(zstd rt) endif () -install(TARGETS zstd RUNTIME DESTINATION "bin") +install(TARGETS zstd RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") if (UNIX) add_custom_target(zstdcat ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdcat DEPENDS zstd COMMENT "Creating zstdcat symlink") add_custom_target(unzstd ALL ${CMAKE_COMMAND} -E create_symlink zstd unzstd DEPENDS zstd COMMENT "Creating unzstd symlink") - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat DESTINATION "bin") - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd DESTINATION "bin") - install(PROGRAMS ${PROGRAMS_DIR}/zstdgrep DESTINATION "bin") - install(PROGRAMS ${PROGRAMS_DIR}/zstdless DESTINATION "bin") + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat DESTINATION "${CMAKE_INSTALL_BINDIR}") + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd DESTINATION "${CMAKE_INSTALL_BINDIR}") + install(PROGRAMS ${PROGRAMS_DIR}/zstdgrep DESTINATION "${CMAKE_INSTALL_BINDIR}") + install(PROGRAMS ${PROGRAMS_DIR}/zstdless DESTINATION "${CMAKE_INSTALL_BINDIR}") add_custom_target(zstd.1 ALL ${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstd.1 . @@ -56,14 +56,16 @@ if (UNIX) # Define MAN_INSTALL_DIR if necessary if (MAN_INSTALL_DIR) else () - set(MAN_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/share/man/man1) + set(MAN_INSTALL_DIR ${CMAKE_INSTALL_MANDIR}/man1) endif () - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstd.1 DESTINATION "${MAN_INSTALL_DIR}") - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat.1 DESTINATION "${MAN_INSTALL_DIR}") - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd.1 DESTINATION "${MAN_INSTALL_DIR}") - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdgrep.1 DESTINATION "${MAN_INSTALL_DIR}") - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdless.1 DESTINATION "${MAN_INSTALL_DIR}") + install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/zstd.1 + ${CMAKE_CURRENT_BINARY_DIR}/zstdcat.1 + ${CMAKE_CURRENT_BINARY_DIR}/unzstd.1 + ${CMAKE_CURRENT_BINARY_DIR}/zstdgrep.1 + ${CMAKE_CURRENT_BINARY_DIR}/zstdless.1 + DESTINATION "${MAN_INSTALL_DIR}") add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c) target_link_libraries(zstd-frugal libzstd_static) @@ -79,7 +81,7 @@ if (ZSTD_MULTITHREAD_SUPPORT) target_link_libraries(zstd ${THREADS_LIBS}) add_custom_target(zstdmt ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdmt DEPENDS zstd COMMENT "Creating zstdmt symlink") - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdmt DESTINATION "bin") + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdmt DESTINATION "${CMAKE_INSTALL_BINDIR}") endif () endif () From bff6072e3a691a47638153d9daa1cd9de7c119b4 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Mon, 16 Sep 2019 08:26:21 -0700 Subject: [PATCH 099/144] Bailing early when collecting sequences and documentation --- lib/compress/zstd_compress.c | 13 +++++-------- lib/zstd.h | 16 +++++++++++----- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 849a9f42..833ae838 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2315,10 +2315,6 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) zc->seqCollector.seqIndex += seqsSize; } -/* We call compress2() and collect sequences after each block - * compression. The function stores the ZSTD_Sequences in outSeqs - * and returns the number of collected sequences from all blocks. - */ size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, size_t outSeqsSize, const void* src, size_t srcSize) { @@ -2351,6 +2347,11 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, if (bss == ZSTDbss_noCompress) { cSize = 0; goto out; } } + if (zc->seqCollector.collectSequences) { + ZSTD_copyBlockSequences(zc); + return 0; + } + /* encode sequences and literals */ cSize = ZSTD_compressSequences(&zc->seqStore, &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, @@ -2360,10 +2361,6 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, zc->entropyWorkspace, HUF_WORKSPACE_SIZE /* statically allocated in resetCCtx */, zc->bmi2); - if (zc->seqCollector.collectSequences) { - ZSTD_copyBlockSequences(zc); - } - out: if (!ZSTD_isError(cSize) && cSize != 0) { /* confirm repcodes and entropy tables when emitting a compressed block */ diff --git a/lib/zstd.h b/lib/zstd.h index 97feb77d..3e737d8f 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1073,11 +1073,11 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params; typedef struct { - unsigned int matchPos; - unsigned int offset; - unsigned int litLength; - unsigned int matchLength; - unsigned int rep; + unsigned int matchPos; /* match pos in dst */ + unsigned int offset; /* offset taking into account rep (different from seqdef) */ + unsigned int litLength; /* literal length */ + unsigned int matchLength; /* match length */ + unsigned int rep; /* 0 when seq not rep and seqDef.offset otherwise */ } ZSTD_Sequence; typedef struct { @@ -1218,6 +1218,12 @@ ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcS * or an error code (if srcSize is too small) */ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); +/*! ZSTD_getSequences() : + * Extract sequences from the sequence store + * zc can be used to insert custom compression params. + * This function invokes ZSTD_compress2 + * @return : number of sequences extracted + */ ZSTDLIB_API size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, size_t outSeqsSize, const void* src, size_t srcSize); From 72ea79cacd161ba8c2f0cf49217d0df5ff844d5f Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 16 Sep 2019 12:08:03 -0400 Subject: [PATCH 100/144] Don't Include `sanitizer/msan_interface.h`, Since Not All Platforms Provide It Instead, explicitly declare the functions we use. --- lib/common/mem.h | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/common/mem.h b/lib/common/mem.h index 3dc4f428..2b115ddb 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -55,7 +55,23 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size #endif #if defined (MEMORY_SANITIZER) -# include +/* Not all platforms that support msan provide sanitizers/msan_interface.h. + * We therefore declare the functions we need ourselves, rather than trying to + * include the header file... */ + +#include /* intptr_t */ + +/* Make memory region fully initialized (without changing its contents). */ +void __msan_unpoison(const volatile void *a, size_t size); + +/* Make memory region fully uninitialized (without changing its contents). + This is a legacy interface that does not update origin information. Use + __msan_allocated_memory() instead. */ +void __msan_poison(const volatile void *a, size_t size); + +/* Returns the offset of the first (at least partially) poisoned byte in the + memory range, or -1 if the whole range is good. */ +intptr_t __msan_test_shadow(const volatile void *x, size_t size); #endif #if defined (MEMORY_SANITIZER) From 1f93be0f6dbb9bad2103b71a249aedee8b5dc2ad Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Mon, 16 Sep 2019 13:35:45 -0700 Subject: [PATCH 101/144] Handling memory leak and potential side effect --- tests/fuzzer.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index fdf6960b..b11a3cc0 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1960,10 +1960,15 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "OK \n"); } - DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences zeros : ", testNb++); - memset(CNBuffer, 0, 1000000); - assert(ZSTD_getSequences(ZSTD_createCCtx(), compressedBuffer, 1000000, - CNBuffer, 1000000) == 1000000 / 131071 + 1); + { + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + assert(cctx != NULL); + DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences zeros : ", testNb++); + memset(CNBuffer, 0, 1000000); + assert(ZSTD_getSequences(cctx, compressedBuffer, 1000000, + CNBuffer, 1000000) == 1000000 / 131071 + 1); + ZSTD_freeCCtx(cctx); + } /* All zeroes test (test bug #137) */ #define ZEROESLENGTH 100 From 76fea3fb9922c68a45159aa68b536ac196f31ab7 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Mon, 16 Sep 2019 14:02:23 -0700 Subject: [PATCH 102/144] Resolving appveyor test failure implicit conversion --- lib/compress/zstd_compress.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 3fe84b6b..acb3b15e 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2290,13 +2290,13 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) if (outSeqs[i].offset <= ZSTD_REP_NUM) { outSeqs[i].rep = outSeqs[i].offset; - repIdx = i - outSeqs[i].offset; + repIdx = (unsigned int)i - outSeqs[i].offset; if (outSeqs[i].litLength == 0) { if (outSeqs[i].offset < 3) { --repIdx; } else { - repIdx = i - 1; + repIdx = (unsigned int)i - 1; } ++outSeqs[i].rep; } @@ -2310,7 +2310,7 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) } position += outSeqs[i].litLength; - outSeqs[i].matchPos = position; + outSeqs[i].matchPos = (unsigned int)position; position += outSeqs[i].matchLength; } zc->seqCollector.seqIndex += seqsSize; From 243200e5bf74649b7f7d42816df9cafe3f916269 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 17 Sep 2019 14:02:57 -0700 Subject: [PATCH 103/144] minor refactor of ZSTD_fast - reduced variables lifetime - more accurate code comments --- lib/compress/zstd_compress_internal.h | 10 ++--- lib/compress/zstd_fast.c | 59 ++++++++++++--------------- 2 files changed, 30 insertions(+), 39 deletions(-) diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index ae106e02..41861bd7 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -341,18 +341,18 @@ MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat) } /*! ZSTD_storeSeq() : - * Store a sequence (literal length, literals, offset code and match length code) into seqStore_t. - * `offsetCode` : distance to match + 3 (values 1-3 are repCodes). + * Store a sequence (litlen, litPtr, offCode and mlBase) into seqStore_t. + * `offCode` : distance to match + ZSTD_REP_MOVE (values <= ZSTD_REP_MOVE are repCodes). * `mlBase` : matchLength - MINMATCH */ -MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t mlBase) +MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offCode, size_t mlBase) { #if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6) static const BYTE* g_start = NULL; if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */ { U32 const pos = (U32)((const BYTE*)literals - g_start); DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offCode%7u", - pos, (U32)litLength, (U32)mlBase+MINMATCH, (U32)offsetCode); + pos, (U32)litLength, (U32)mlBase+MINMATCH, (U32)offCode); } #endif assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq); @@ -371,7 +371,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v seqStorePtr->sequences[0].litLength = (U16)litLength; /* match offset */ - seqStorePtr->sequences[0].offset = offsetCode + 1; + seqStorePtr->sequences[0].offset = offCode + 1; /* match Length */ if (mlBase>0xFFFF) { diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c index 59267ffb..75aec8a3 100644 --- a/lib/compress/zstd_fast.c +++ b/lib/compress/zstd_fast.c @@ -8,7 +8,7 @@ * You may select, at your option, one of the above-listed licenses. */ -#include "zstd_compress_internal.h" +#include "zstd_compress_internal.h" /* ZSTD_hashPtr, ZSTD_count, ZSTD_storeSeq */ #include "zstd_fast.h" @@ -43,8 +43,8 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms, } -FORCE_INLINE_TEMPLATE -size_t ZSTD_compressBlock_fast_generic( +FORCE_INLINE_TEMPLATE size_t +ZSTD_compressBlock_fast_generic( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize, U32 const mls) @@ -74,8 +74,7 @@ size_t ZSTD_compressBlock_fast_generic( DEBUGLOG(5, "ZSTD_compressBlock_fast_generic"); ip0 += (ip0 == prefixStart); ip1 = ip0 + 1; - { - U32 const maxRep = (U32)(ip0 - prefixStart); + { U32 const maxRep = (U32)(ip0 - prefixStart); if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; } @@ -118,8 +117,7 @@ size_t ZSTD_compressBlock_fast_generic( match0 = match1; goto _offset; } - { - size_t const step = ((ip0-anchor) >> (kSearchStrength - 1)) + stepSize; + { size_t const step = ((size_t)(ip0-anchor) >> (kSearchStrength - 1)) + stepSize; assert(step >= 2); ip0 += step; ip1 += step; @@ -138,7 +136,7 @@ _offset: /* Requires: ip0, match0 */ _match: /* Requires: ip0, match0, offcode */ /* Count the forward length */ mLength += ZSTD_count(ip0+mLength+4, match0+mLength+4, iend) + 4; - ZSTD_storeSeq(seqStore, ip0-anchor, anchor, offcode, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip0-anchor), anchor, offcode, mLength-MINMATCH); /* match found */ ip0 += mLength; anchor = ip0; @@ -150,16 +148,15 @@ _match: /* Requires: ip0, match0, offcode */ hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */ hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); - while ( (ip0 <= ilimit) - && ( (offset_2>0) - & (MEM_read32(ip0) == MEM_read32(ip0 - offset_2)) )) { + while ( ((ip0 <= ilimit) & (offset_2>0)) /* offset_2==0 means offset_2 is invalidated */ + && (MEM_read32(ip0) == MEM_read32(ip0 - offset_2)) ) { /* store sequence */ size_t const rLength = ZSTD_count(ip0+4, ip0+4-offset_2, iend) + 4; - U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ + { U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); ip0 += rLength; ip1 = ip0 + 1; - ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength-MINMATCH); + ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, 0 /*offCode*/, rLength-MINMATCH); anchor = ip0; continue; /* faster when present (confirmed on gcc-8) ... (?) */ } @@ -179,8 +176,7 @@ size_t ZSTD_compressBlock_fast( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - ZSTD_compressionParameters const* cParams = &ms->cParams; - U32 const mls = cParams->minMatch; + U32 const mls = ms->cParams.minMatch; assert(ms->dictMatchState == NULL); switch(mls) { @@ -348,8 +344,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - ZSTD_compressionParameters const* cParams = &ms->cParams; - U32 const mls = cParams->minMatch; + U32 const mls = ms->cParams.minMatch; assert(ms->dictMatchState != NULL); switch(mls) { @@ -408,16 +403,17 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const U32 repIndex = current + 1 - offset_1; const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; - size_t mLength; hashTable[h] = current; /* update hash table */ assert(offset_1 <= current +1); /* check repIndex */ if ( (((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > dictStartIndex)) && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; - mLength = ZSTD_count_2segments(ip+1 +4, repMatch +4, iend, repMatchEnd, prefixStart) + 4; + size_t const rLength = ZSTD_count_2segments(ip+1 +4, repMatch +4, iend, repMatchEnd, prefixStart) + 4; ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, rLength-MINMATCH); + ip += rLength; + anchor = ip; } else { if ( (matchIndex < dictStartIndex) || (MEM_read32(match) != MEM_read32(ip)) ) { @@ -427,19 +423,15 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( } { const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; - U32 offset; - mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; + U32 const offset = current - matchIndex; + size_t mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ - offset = current - matchIndex; - offset_2 = offset_1; - offset_1 = offset; + offset_2 = offset_1; offset_1 = offset; /* update offset history */ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ip += mLength; + anchor = ip; } } - /* found a match : store it */ - ip += mLength; - anchor = ip; - if (ip <= ilimit) { /* Fill Table */ hashTable[ZSTD_hashPtr(base+current+2, hlog, mls)] = current+2; @@ -448,13 +440,13 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( while (ip <= ilimit) { U32 const current2 = (U32)(ip-base); U32 const repIndex2 = current2 - offset_2; - const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; + const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (repIndex2 > dictStartIndex)) /* intentional overflow */ && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; - U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH); + { U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; } /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, 0 /*offcode*/, repLength2-MINMATCH); hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; ip += repLength2; anchor = ip; @@ -476,8 +468,7 @@ size_t ZSTD_compressBlock_fast_extDict( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - ZSTD_compressionParameters const* cParams = &ms->cParams; - U32 const mls = cParams->minMatch; + U32 const mls = ms->cParams.minMatch; switch(mls) { default: /* includes case 3 */ From 3cacc0a30bb3f39af6f4cb7f2ebca9ace9f45aff Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Tue, 17 Sep 2019 17:44:08 -0700 Subject: [PATCH 104/144] Casting void pointer to ZSTD_Sequence pointer --- tests/fuzzer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 62263fd1..a513e1b4 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1965,7 +1965,7 @@ static int basicUnitTests(U32 const seed, double compressibility) assert(cctx != NULL); DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences zeros : ", testNb++); memset(CNBuffer, 0, 1000000); - assert(ZSTD_getSequences(cctx, compressedBuffer, 1000000, + assert(ZSTD_getSequences(cctx, (ZSTD_Sequence*)compressedBuffer, 1000000, CNBuffer, 1000000) == 1000000 / 131071 + 1); ZSTD_freeCCtx(cctx); } From 51990246c3d5d8caa4d182e2298e12fc5761bf3d Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Wed, 18 Sep 2019 11:05:08 -0700 Subject: [PATCH 105/144] [test] Test the bounds of ZSTD_c_srcSizeHint --- tests/zstreamtest.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 9af08ebe..2047d4bd 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -1151,6 +1151,16 @@ static int basicUnitTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : ZSTD_c_srcSizeHint bounds : ", testNb++); + ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters); + CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, INT_MAX)); + { int srcSizeHint; + CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_srcSizeHint, &srcSizeHint)); + CHECK(!(srcSizeHint == INT_MAX), "srcSizeHint doesn't match"); + } + CHECK(!ZSTD_isError(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, -1)), "Out of range doesn't error"); + DISPLAYLEVEL(3, "OK \n"); + /* Overlen overwriting window data bug */ DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++); { /* This test has a window size of 1024 bytes and consists of 3 blocks: From f9b6abb896fb05027f242c526bb54719e124b24f Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Wed, 18 Sep 2019 13:29:05 -0700 Subject: [PATCH 106/144] Adding 4 blocks to FSE_BLOCKBOUND() in lib/common (different from last week) --- lib/common/fse.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common/fse.h b/lib/common/fse.h index 811c670b..deac3f99 100644 --- a/lib/common/fse.h +++ b/lib/common/fse.h @@ -308,7 +308,7 @@ If there is an error, the function will return an error code, which can be teste *******************************************/ /* FSE buffer bounds */ #define FSE_NCOUNTBOUND 512 -#define FSE_BLOCKBOUND(size) (size + (size>>7)) +#define FSE_BLOCKBOUND(size) (size + (size>>7) + 4 /* constant for initial fse states */) #define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ /* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ From 6e9f6813bb6d96a95c18049dd5a37c517a6664f5 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Wed, 18 Sep 2019 13:49:45 -0700 Subject: [PATCH 107/144] adding bit container size --- lib/common/fse.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common/fse.h b/lib/common/fse.h index deac3f99..a7553e37 100644 --- a/lib/common/fse.h +++ b/lib/common/fse.h @@ -308,7 +308,7 @@ If there is an error, the function will return an error code, which can be teste *******************************************/ /* FSE buffer bounds */ #define FSE_NCOUNTBOUND 512 -#define FSE_BLOCKBOUND(size) (size + (size>>7) + 4 /* constant for initial fse states */) +#define FSE_BLOCKBOUND(size) (size + (size>>7) + 4 /* fse states */ + sizeof(size_t) /* bitContainer */) #define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ /* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ From ae6d0e64ae0fcd66a6c1d2061a017bc7521ac7c0 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 19 Sep 2019 15:25:20 -0700 Subject: [PATCH 108/144] Addressing comments --- lib/compress/zstd_compress.c | 9 +++------ lib/zstd.h | 20 +++++++++++++++----- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index acb3b15e..ce352acd 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2302,7 +2302,7 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) } assert(repIdx >= -3); outSeqs[i].offset = repIdx >= 0 ? outSeqs[repIdx].offset : repStartValue[-repIdx - 1]; - if (outSeqs[i].offset == 4) { + if (outSeqs[i].rep == 4) { --outSeqs[i].offset; } } else { @@ -2319,9 +2319,6 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, size_t outSeqsSize, const void* src, size_t srcSize) { - const size_t dstCapacity = ZSTD_compressBound(srcSize * sizeof(void*)); - void* dst = ZSTD_malloc(dstCapacity, ZSTD_defaultCMem); - SeqCollector seqCollector; seqCollector.collectSequences = 1; seqCollector.seqStart = outSeqs; @@ -2329,8 +2326,8 @@ size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, seqCollector.maxSequences = outSeqsSize; zc->seqCollector = seqCollector; - ZSTD_compress2(zc, dst, dstCapacity, src, srcSize); - ZSTD_free(dst, ZSTD_defaultCMem); + /* We never write to dst when collecing sequences so setting dst = src is harmless */ + ZSTD_compress2(zc, (void*)src, srcSize, src, srcSize); return zc->seqCollector.seqIndex; } diff --git a/lib/zstd.h b/lib/zstd.h index 217a6d35..836aa723 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1078,11 +1078,21 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params; typedef struct { - unsigned int matchPos; /* match pos in dst */ - unsigned int offset; /* offset taking into account rep (different from seqdef) */ - unsigned int litLength; /* literal length */ - unsigned int matchLength; /* match length */ - unsigned int rep; /* 0 when seq not rep and seqDef.offset otherwise */ + unsigned int matchPos; /* Match pos in dst */ + /* If seqDef.offset > 3, then this is seqDef.offset - 3 + * If seqDef.offset < 3, then this is the corresponding repeat offset + * But if seqDef.offset < 3 and litLength == 0, this is the + * repeat offset before the corresponding repeat offset + * And if seqDef.offset == 3 and litLength == 0, this is the + * most recent repeat offset - 1 + */ + unsigned int offset; + unsigned int litLength; /* Literal length */ + unsigned int matchLength; /* Match length */ + /* 0 when seq not rep and seqDef.offset otherwise + * when litLength == 0 this will be <= 4, otherwise <= 3 like normal + */ + unsigned int rep; } ZSTD_Sequence; typedef struct { From efd37a64eaff5a0a26ae2566fdb45dc4a0c91673 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 19 Sep 2019 13:25:03 -0700 Subject: [PATCH 109/144] Optimize decompression and fix wildcopy overread * Bump `WILDCOPY_OVERLENGTH` to 16 to fix the wildcopy overread. * Optimize `ZSTD_wildcopy()` by removing unnecessary branches and unrolling the loop. * Extract `ZSTD_overlapCopy8()` into its own function. * Add `ZSTD_safecopy()` for `ZSTD_execSequenceEnd()`. It is optimized for single long sequences, since that is the important case that can end up in `ZSTD_execSequenceEnd()`. Without this optimization, decompressing a block with 1 long match goes from 5.7 GB/s to 800 MB/s. * Refactor `ZSTD_execSequenceEnd()`. * Increase the literal copy shortcut to 16. * Add a shortcut for offset >= 16. * Simplify `ZSTD_execSequence()` by pushing more cases into `ZSTD_execSequenceEnd()`. * Delete `ZSTD_execSequenceLong()` since it is exactly the same as `ZSTD_execSequence()`. clang-8 seeds +17.5% on silesia and +21.8% on enwik8. gcc-9 sees +12% on silesia and +15.5% on enwik8. TODO: More detailed measurements, and on more datasets. Crdit to OSS-Fuzz for finding the wildcopy overread. --- lib/common/zstd_internal.h | 97 ++++----- lib/compress/zstd_compress_internal.h | 5 +- lib/decompress/zstd_decompress_block.c | 282 ++++++++++++------------- 3 files changed, 179 insertions(+), 205 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index fb6246a1..007b03df 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -197,8 +197,8 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } static void ZSTD_copy16(void* dst, const void* src) { memcpy(dst, src, 16); } #define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; } -#define WILDCOPY_OVERLENGTH 8 -#define VECLEN 16 +#define WILDCOPY_OVERLENGTH 16 +#define WILDCOPY_VECLEN 16 typedef enum { ZSTD_no_overlap, @@ -207,83 +207,58 @@ typedef enum { } ZSTD_overlap_e; /*! ZSTD_wildcopy() : - * custom version of memcpy(), can overwrite up to WILDCOPY_OVERLENGTH bytes (if length==0) */ + * Custom version of memcpy(), can over read/write up to WILDCOPY_OVERLENGTH bytes (if length==0) + * @param ovtype controls the overlap detection + * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart. + * - ZSTD_overlap_src_before_dst: The src and dst may overlap, but they MUST be at least 8 bytes apart. + * The src buffer must be before the dst buffer. + */ MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE -void ZSTD_wildcopy(void* dst, const void* src, BYTE* oend_g, ptrdiff_t length, ZSTD_overlap_e ovtype) +void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e ovtype) { ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; const BYTE* ip = (const BYTE*)src; BYTE* op = (BYTE*)dst; BYTE* const oend = op + length; - assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8)); + assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff <= -WILDCOPY_VECLEN)); - if (length < VECLEN || (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN)) { - do - COPY8(op, ip) - while (op < oend); - } - else { - if (oend < oend_g-16) { - /* common case */ + if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) { + /* Handle short offset copies. */ do { - COPY16(op, ip); + COPY8(op, ip) + } while (op < oend); + } else { + assert(diff >= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN); + /* Separate out the first two COPY16() calls because the copy length is + * almost certain to be short, so the branches have different + * probabilities. + * On gcc-9 unrolling once is +1.6%, twice is +2%, thrice is +1.8%. + * On clang-8 unrolling once is +1.4%, twice is +3.3%, thrice is +3%. + */ + COPY16(op, ip); + if (op >= oend) return; + COPY16(op, ip); + if (op >= oend) return; + do { + COPY16(op, ip); } while (op < oend); - } - else { - do { - COPY8(op, ip); - } - while (op < oend); - } } } -/*! ZSTD_wildcopy_16min() : - * same semantics as ZSTD_wildcopy() except guaranteed to be able to copy 16 bytes at the start */ -MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE -void ZSTD_wildcopy_16min(void* dst, const void* src, BYTE* oend_g, ptrdiff_t length, ZSTD_overlap_e ovtype) -{ - ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; - const BYTE* ip = (const BYTE*)src; - BYTE* op = (BYTE*)dst; - BYTE* const oend = op + length; - - assert(length >= 8); - assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8)); - - if (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN) { - do { - COPY8(op, ip); - } - while (op < oend); - } - else { - if (oend < oend_g-16) { - /* common case */ - do { - COPY16(op, ip); - } - while (op < oend); - } - else { - do { - COPY8(op, ip); - } - while (op < oend); - } - } -} - -MEM_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, void* dstEnd) /* should be faster for decoding, but strangely, not verified on all platform */ +/*! ZSTD_wildcopy8() : + * The same as ZSTD_wildcopy(), but it can only overwrite 8 bytes, and works for + * overlapping buffers that are at least 8 bytes apart. + */ +MEM_STATIC void ZSTD_wildcopy8(void* dst, const void* src, ptrdiff_t length) { const BYTE* ip = (const BYTE*)src; BYTE* op = (BYTE*)dst; - BYTE* const oend = (BYTE*)dstEnd; - do + BYTE* const oend = (BYTE*)op + length; + do { COPY8(op, ip) - while (op < oend); + } while (op < oend); } diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 208a04c5..fefa8aff 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -359,7 +359,10 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v /* copy Literals */ assert(seqStorePtr->maxNbLit <= 128 KB); assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit); - ZSTD_wildcopy(seqStorePtr->lit, literals, seqStorePtr->lit + litLength + 8, (ptrdiff_t)litLength, ZSTD_no_overlap); + /* We are guaranteed at least 8 bytes of literals space because of HASH_READ_SIZE and + * MINMATCH. + */ + ZSTD_wildcopy8(seqStorePtr->lit, literals, (ptrdiff_t)litLength); seqStorePtr->lit += litLength; /* literal Length */ diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c index bab96fb7..e799a5c7 100644 --- a/lib/decompress/zstd_decompress_block.c +++ b/lib/decompress/zstd_decompress_block.c @@ -573,38 +573,118 @@ typedef struct { size_t pos; } seqState_t; +/*! ZSTD_overlapCopy8() : + * Copies 8 bytes from ip to op and updates op and ip where ip <= op. + * If the offset is < 8 then the offset is spread to at least 8 bytes. + * + * Precondition: *ip <= *op + * Postcondition: *op - *op >= 8 + */ +static void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) { + assert(*ip <= *op); + if (offset < 8) { + /* close range match, overlap */ + static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ + static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ + int const sub2 = dec64table[offset]; + (*op)[0] = (*ip)[0]; + (*op)[1] = (*ip)[1]; + (*op)[2] = (*ip)[2]; + (*op)[3] = (*ip)[3]; + *ip += dec32table[offset]; + ZSTD_copy4(*op+4, *ip); + *ip -= sub2; + } else { + ZSTD_copy8(*op, *ip); + } + *ip += 8; + *op += 8; + assert(*op - *ip >= 8); +} -/* ZSTD_execSequenceLast7(): - * exceptional case : decompress a match starting within last 7 bytes of output buffer. - * requires more careful checks, to ensure there is no overflow. - * performance does not matter though. - * note : this case is supposed to be never generated "naturally" by reference encoder, - * since in most cases it needs at least 8 bytes to look for a match. - * but it's allowed by the specification. */ +/*! ZSTD_safecopy() : + * Specialized version of memcpy() that is allowed to READ up to WILDCOPY_OVERLENGTH past the input buffer + * and write up to 16 bytes past oend_w (op >= oend_w is allowed). + * This function is only called in the uncommon case where the sequence is near the end of the block. It + * should be fast for a single long sequence, but can be slow for several short sequences. + * + * @param ovtype controls the overlap detection + * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart. + * - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart. + * The src buffer must be before the dst buffer. + */ +static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) { + ptrdiff_t const diff = op - ip; + BYTE* const oend = op + length; + + assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8)) || + (ovtype == ZSTD_overlap_src_before_dst && diff >= 0)); + + if (length < 8) { + /* Handle short lengths. */ + while (op < oend) *op++ = *ip++; + return; + } + if (ovtype == ZSTD_overlap_src_before_dst) { + /* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */ + assert(length >= 8); + ZSTD_overlapCopy8(&op, &ip, diff); + assert(op - ip >= 8); + assert(op <= oend); + } + + if (oend <= oend_w) { + /* No risk of overwrite. */ + ZSTD_wildcopy(op, ip, length, ovtype); + return; + } + if (op <= oend_w) { + /* Wildcopy until we get close to the end. */ + assert(oend > oend_w); + ZSTD_wildcopy(op, ip, oend_w - op, ovtype); + ip += oend_w - op; + op = oend_w; + } + /* Handle the leftovers. */ + while (op < oend) *op++ = *ip++; +} + +/* ZSTD_execSequenceEnd(): + * This version handles cases that are near the end of the output buffer. It requires + * more careful checks to make sure there is no overflow. By separating out these hard + * and unlikely cases, we can speed up the common cases. + * + * NOTE: This function needs to be fast for a single long sequence, but doesn't need + * to be optimized for many small sequences, since those fall into ZSTD_execSequence(). + */ FORCE_NOINLINE -size_t ZSTD_execSequenceLast7(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) +size_t ZSTD_execSequenceEnd(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) { BYTE* const oLitEnd = op + sequence.litLength; size_t const sequenceLength = sequence.litLength + sequence.matchLength; BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ const BYTE* const iLitEnd = *litPtr + sequence.litLength; const BYTE* match = oLitEnd - sequence.offset; + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; - /* check */ - RETURN_ERROR_IF(oMatchEnd>oend, dstSize_tooSmall, "last match must fit within dstBuffer"); + /* bounds checks */ + assert(oLitEnd < oMatchEnd); + RETURN_ERROR_IF(oMatchEnd > oend, dstSize_tooSmall, "last match must fit within dstBuffer"); RETURN_ERROR_IF(iLitEnd > litLimit, corruption_detected, "try to read beyond literal buffer"); /* copy literals */ - while (op < oLitEnd) *op++ = *(*litPtr)++; + ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap); + op = oLitEnd; + *litPtr = iLitEnd; /* copy Match */ - if (sequence.offset > (size_t)(oLitEnd - base)) { + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { /* offset beyond prefix */ - RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - vBase),corruption_detected); - match = dictEnd - (base-match); + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected); + match = dictEnd - (prefixStart-match); if (match + sequence.matchLength <= dictEnd) { memmove(oLitEnd, match, sequence.matchLength); return sequenceLength; @@ -614,13 +694,12 @@ size_t ZSTD_execSequenceLast7(BYTE* op, memmove(oLitEnd, match, length1); op = oLitEnd + length1; sequence.matchLength -= length1; - match = base; + match = prefixStart; } } - while (op < oMatchEnd) *op++ = *match++; + ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); return sequenceLength; } - HINT_INLINE size_t ZSTD_execSequence(BYTE* op, BYTE* const oend, seq_t sequence, @@ -634,20 +713,27 @@ size_t ZSTD_execSequence(BYTE* op, const BYTE* const iLitEnd = *litPtr + sequence.litLength; const BYTE* match = oLitEnd - sequence.offset; - /* check */ - RETURN_ERROR_IF(oMatchEnd>oend, dstSize_tooSmall, "last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend"); - RETURN_ERROR_IF(iLitEnd > litLimit, corruption_detected, "over-read beyond lit buffer"); - if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + /* Errors and uncommon cases handled here. */ + assert(oLitEnd < oMatchEnd); + if (iLitEnd > litLimit || oMatchEnd > oend_w) + return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); - /* copy Literals */ - if (sequence.litLength > 8) - ZSTD_wildcopy_16min(op, (*litPtr), oend, sequence.litLength, ZSTD_no_overlap); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ + assert(iLitEnd <= litLimit /* Literal length is in bounds */); + assert(oLitEnd <= oend_w /* Can wildcopy literals */); + assert(oMatchEnd <= oend_w /* Can wildcopy matches */); + + /* Copy Literals: + * Split out litLength <= 16 since it is nearly always true. +1% on gcc-9. + */ + if (sequence.litLength <= 16) + ZSTD_copy16(op, *litPtr); else - ZSTD_copy8(op, *litPtr); + ZSTD_wildcopy(op, (*litPtr), sequence.litLength, ZSTD_no_overlap); op = oLitEnd; *litPtr = iLitEnd; /* update for next sequence */ - /* copy Match */ + /* Copy Match */ if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { /* offset beyond prefix -> go into extDict */ RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected); @@ -662,123 +748,33 @@ size_t ZSTD_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = prefixStart; - if (op > oend_w || sequence.matchLength < MINMATCH) { - U32 i; - for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i]; - return sequenceLength; - } } } - /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */ + /* Match within prefix of 1 or more bytes */ + assert(op <= oMatchEnd); + assert(oMatchEnd <= oend_w); + assert(match >= prefixStart); + assert(sequence.matchLength >= 1); - /* match within prefix */ - if (sequence.offset < 8) { - /* close range match, overlap */ - static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ - static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ - int const sub2 = dec64table[sequence.offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[sequence.offset]; - ZSTD_copy4(op+4, match); - match -= sub2; - } else { - ZSTD_copy8(op, match); + /* Nearly all offsets are >= 16 bytes, which means we can use wildcopy + * without overlap checking. + */ + if (sequence.offset >= 16) { + /* Split out matchLength <= 16 since it is nearly always true. +1% on gcc-9. */ + if (sequence.matchLength <= 16) + ZSTD_copy16(op, match); + else + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); + return sequenceLength; } - op += 8; match += 8; + assert(sequence.offset < 16); - if (oMatchEnd > oend-(16-MINMATCH)) { - if (op < oend_w) { - ZSTD_wildcopy(op, match, oend, oend_w - op, ZSTD_overlap_src_before_dst); - match += oend_w - op; - op = oend_w; - } - while (op < oMatchEnd) *op++ = *match++; - } else { - ZSTD_wildcopy(op, match, oend, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); /* works even if matchLength < 8 */ - } - return sequenceLength; -} + /* Copy 8 bytes and spread the offset to be >= 8. */ + ZSTD_overlapCopy8(&op, &match, sequence.offset); - -HINT_INLINE -size_t ZSTD_execSequenceLong(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const prefixStart, const BYTE* const dictStart, const BYTE* const dictEnd) -{ - BYTE* const oLitEnd = op + sequence.litLength; - size_t const sequenceLength = sequence.litLength + sequence.matchLength; - BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ - BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; - const BYTE* const iLitEnd = *litPtr + sequence.litLength; - const BYTE* match = sequence.match; - - /* check */ - RETURN_ERROR_IF(oMatchEnd > oend, dstSize_tooSmall, "last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend"); - RETURN_ERROR_IF(iLitEnd > litLimit, corruption_detected, "over-read beyond lit buffer"); - if (oLitEnd > oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, dictStart, dictEnd); - - /* copy Literals */ - if (sequence.litLength > 8) - ZSTD_wildcopy_16min(op, *litPtr, oend, sequence.litLength, ZSTD_no_overlap); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ - else - ZSTD_copy8(op, *litPtr); /* note : op <= oLitEnd <= oend_w == oend - 8 */ - - op = oLitEnd; - *litPtr = iLitEnd; /* update for next sequence */ - - /* copy Match */ - if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { - /* offset beyond prefix */ - RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - dictStart), corruption_detected); - if (match + sequence.matchLength <= dictEnd) { - memmove(oLitEnd, match, sequence.matchLength); - return sequenceLength; - } - /* span extDict & currentPrefixSegment */ - { size_t const length1 = dictEnd - match; - memmove(oLitEnd, match, length1); - op = oLitEnd + length1; - sequence.matchLength -= length1; - match = prefixStart; - if (op > oend_w || sequence.matchLength < MINMATCH) { - U32 i; - for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i]; - return sequenceLength; - } - } } - assert(op <= oend_w); - assert(sequence.matchLength >= MINMATCH); - - /* match within prefix */ - if (sequence.offset < 8) { - /* close range match, overlap */ - static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ - static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ - int const sub2 = dec64table[sequence.offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[sequence.offset]; - ZSTD_copy4(op+4, match); - match -= sub2; - } else { - ZSTD_copy8(op, match); - } - op += 8; match += 8; - - if (oMatchEnd > oend-(16-MINMATCH)) { - if (op < oend_w) { - ZSTD_wildcopy(op, match, oend, oend_w - op, ZSTD_overlap_src_before_dst); - match += oend_w - op; - op = oend_w; - } - while (op < oMatchEnd) *op++ = *match++; - } else { - ZSTD_wildcopy(op, match, oend, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); /* works even if matchLength < 8 */ + /* If the match length is > 8 bytes, then continue with the wildcopy. */ + if (sequence.matchLength > 8) { + assert(op < oMatchEnd); + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); } return sequenceLength; } @@ -1098,7 +1094,7 @@ ZSTD_decompressSequencesLong_body( /* decode and decompress */ for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && (seqNb Date: Fri, 20 Sep 2019 00:52:15 -0700 Subject: [PATCH 110/144] Widen ZSTD_wildcopy to 32 bytes --- lib/common/zstd_internal.h | 6 +++--- lib/decompress/zstd_decompress_block.c | 28 ++++++++++++++------------ 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 007b03df..522c1fda 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -197,7 +197,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } static void ZSTD_copy16(void* dst, const void* src) { memcpy(dst, src, 16); } #define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; } -#define WILDCOPY_OVERLENGTH 16 +#define WILDCOPY_OVERLENGTH 32 #define WILDCOPY_VECLEN 16 typedef enum { @@ -237,11 +237,11 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e * On clang-8 unrolling once is +1.4%, twice is +3.3%, thrice is +3%. */ COPY16(op, ip); - if (op >= oend) return; COPY16(op, ip); if (op >= oend) return; do { COPY16(op, ip); + COPY16(op, ip); } while (op < oend); } @@ -257,7 +257,7 @@ MEM_STATIC void ZSTD_wildcopy8(void* dst, const void* src, ptrdiff_t length) BYTE* op = (BYTE*)dst; BYTE* const oend = (BYTE*)op + length; do { - COPY8(op, ip) + COPY8(op, ip); } while (op < oend); } diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c index e799a5c7..27ce137f 100644 --- a/lib/decompress/zstd_decompress_block.c +++ b/lib/decompress/zstd_decompress_block.c @@ -724,12 +724,14 @@ size_t ZSTD_execSequence(BYTE* op, assert(oMatchEnd <= oend_w /* Can wildcopy matches */); /* Copy Literals: - * Split out litLength <= 16 since it is nearly always true. +1% on gcc-9. + * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. + * We likely don't need the full 32-byte wildcopy. */ - if (sequence.litLength <= 16) - ZSTD_copy16(op, *litPtr); - else - ZSTD_wildcopy(op, (*litPtr), sequence.litLength, ZSTD_no_overlap); + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(op, (*litPtr)); + if (sequence.litLength > 16) { + ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap); + } op = oLitEnd; *litPtr = iLitEnd; /* update for next sequence */ @@ -755,18 +757,18 @@ size_t ZSTD_execSequence(BYTE* op, assert(match >= prefixStart); assert(sequence.matchLength >= 1); - /* Nearly all offsets are >= 16 bytes, which means we can use wildcopy + /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy * without overlap checking. */ - if (sequence.offset >= 16) { - /* Split out matchLength <= 16 since it is nearly always true. +1% on gcc-9. */ - if (sequence.matchLength <= 16) - ZSTD_copy16(op, match); - else - ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); + if (sequence.offset >= WILDCOPY_VECLEN) { + /* Split out matchLength <= 32 since it is nearly always true. +1% on gcc-9. + * We copy 32 bytes here since matches are generally longer than literals. + * In silesia, for example ~10% of matches are longer than 16 bytes. + */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); return sequenceLength; } - assert(sequence.offset < 16); + assert(sequence.offset < WILDCOPY_VECLEN); /* Copy 8 bytes and spread the offset to be >= 8. */ ZSTD_overlapCopy8(&op, &match, sequence.offset); From ddab2a94e86c059a613e99c8562afa6495600148 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 20 Sep 2019 00:52:55 -0700 Subject: [PATCH 111/144] Pass iend into ZSTD_storeSeq() to allow ZSTD_wildcopy() --- lib/compress/zstd_compress_internal.h | 9 +++++++-- lib/compress/zstd_double_fast.c | 18 +++++++++--------- lib/compress/zstd_fast.c | 18 +++++++++--------- lib/compress/zstd_lazy.c | 10 +++++----- lib/compress/zstd_ldm.c | 2 +- lib/compress/zstd_opt.c | 2 +- lib/decompress/zstd_decompress_block.c | 8 ++++---- 7 files changed, 36 insertions(+), 31 deletions(-) diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index fefa8aff..579bd5d4 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -344,8 +344,9 @@ MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat) * Store a sequence (litlen, litPtr, offCode and mlBase) into seqStore_t. * `offCode` : distance to match + ZSTD_REP_MOVE (values <= ZSTD_REP_MOVE are repCodes). * `mlBase` : matchLength - MINMATCH + * Allowed to overread literals up to litLimit. */ -MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offCode, size_t mlBase) +MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, const BYTE* litLimit, U32 offCode, size_t mlBase) { #if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6) static const BYTE* g_start = NULL; @@ -362,7 +363,11 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v /* We are guaranteed at least 8 bytes of literals space because of HASH_READ_SIZE and * MINMATCH. */ - ZSTD_wildcopy8(seqStorePtr->lit, literals, (ptrdiff_t)litLength); + assert(litLimit - literals >= HASH_READ_SIZE + MINMATCH); + if (litLimit - literals >= WILDCOPY_OVERLENGTH) + ZSTD_wildcopy(seqStorePtr->lit, literals, (ptrdiff_t)litLength, ZSTD_no_overlap); + else + ZSTD_wildcopy8(seqStorePtr->lit, literals, (ptrdiff_t)litLength); seqStorePtr->lit += litLength; /* literal Length */ diff --git a/lib/compress/zstd_double_fast.c b/lib/compress/zstd_double_fast.c index 54467cc3..a661a485 100644 --- a/lib/compress/zstd_double_fast.c +++ b/lib/compress/zstd_double_fast.c @@ -148,7 +148,7 @@ size_t ZSTD_compressBlock_doubleFast_generic( const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); goto _match_stored; } @@ -157,7 +157,7 @@ size_t ZSTD_compressBlock_doubleFast_generic( && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) { mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); goto _match_stored; } @@ -247,7 +247,7 @@ _match_found: offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); _match_stored: /* match found */ @@ -278,7 +278,7 @@ _match_stored: const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4; U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH); hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; ip += repLength2; @@ -297,7 +297,7 @@ _match_stored: U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); - ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, rLength-MINMATCH); ip += rLength; anchor = ip; continue; /* faster when present ... (?) */ @@ -411,7 +411,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); } else { if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend; @@ -422,7 +422,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) { size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); @@ -447,7 +447,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( } offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } else { ip += ((ip-anchor) >> kSearchStrength) + 1; @@ -479,7 +479,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH); hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; ip += repLength2; diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c index 75aec8a3..6dbefee6 100644 --- a/lib/compress/zstd_fast.c +++ b/lib/compress/zstd_fast.c @@ -136,7 +136,7 @@ _offset: /* Requires: ip0, match0 */ _match: /* Requires: ip0, match0, offcode */ /* Count the forward length */ mLength += ZSTD_count(ip0+mLength+4, match0+mLength+4, iend) + 4; - ZSTD_storeSeq(seqStore, (size_t)(ip0-anchor), anchor, offcode, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip0-anchor), anchor, iend, offcode, mLength-MINMATCH); /* match found */ ip0 += mLength; anchor = ip0; @@ -156,7 +156,7 @@ _match: /* Requires: ip0, match0, offcode */ hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); ip0 += rLength; ip1 = ip0 + 1; - ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, 0 /*offCode*/, rLength-MINMATCH); + ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, 0 /*offCode*/, rLength-MINMATCH); anchor = ip0; continue; /* faster when present (confirmed on gcc-8) ... (?) */ } @@ -261,7 +261,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); } else if ( (matchIndex <= prefixStartIndex) ) { size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls); U32 const dictMatchIndex = dictHashTable[dictHash]; @@ -281,7 +281,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } } else if (MEM_read32(match) != MEM_read32(ip)) { /* it's not a match, and we're not going to check the dictionary */ @@ -296,7 +296,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } /* match found */ @@ -321,7 +321,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH); hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; ip += repLength2; anchor = ip; @@ -411,7 +411,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; size_t const rLength = ZSTD_count_2segments(ip+1 +4, repMatch +4, iend, repMatchEnd, prefixStart) + 4; ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, rLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, rLength-MINMATCH); ip += rLength; anchor = ip; } else { @@ -427,7 +427,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( size_t mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; /* update offset history */ - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); ip += mLength; anchor = ip; } } @@ -446,7 +446,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; { U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; } /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, 0 /*offcode*/, repLength2-MINMATCH); + ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, 0 /*offcode*/, repLength2-MINMATCH); hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; ip += repLength2; anchor = ip; diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c index 0af41724..9ad7e03b 100644 --- a/lib/compress/zstd_lazy.c +++ b/lib/compress/zstd_lazy.c @@ -810,7 +810,7 @@ ZSTD_compressBlock_lazy_generic( /* store sequence */ _storeSequence: { size_t const litLength = start - anchor; - ZSTD_storeSeq(seqStore, litLength, anchor, (U32)offset, matchLength-MINMATCH); + ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offset, matchLength-MINMATCH); anchor = ip = start + matchLength; } @@ -828,7 +828,7 @@ _storeSequence: const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend; matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4; offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength-MINMATCH); ip += matchLength; anchor = ip; continue; @@ -843,7 +843,7 @@ _storeSequence: /* store sequence */ matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap repcodes */ - ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength-MINMATCH); ip += matchLength; anchor = ip; continue; /* faster when present ... (?) */ @@ -1051,7 +1051,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( /* store sequence */ _storeSequence: { size_t const litLength = start - anchor; - ZSTD_storeSeq(seqStore, litLength, anchor, (U32)offset, matchLength-MINMATCH); + ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offset, matchLength-MINMATCH); anchor = ip = start + matchLength; } @@ -1066,7 +1066,7 @@ _storeSequence: const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset history */ - ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength-MINMATCH); ip += matchLength; anchor = ip; continue; /* faster when present ... (?) */ diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index 3dcf86e6..fc3f4694 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -583,7 +583,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, rep[i] = rep[i-1]; rep[0] = sequence.offset; /* Store the sequence */ - ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, + ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, iend, sequence.offset + ZSTD_REP_MOVE, sequence.matchLength - MINMATCH); ip += sequence.matchLength; diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 2da363f9..2e50fca6 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -1098,7 +1098,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ assert(anchor + llen <= iend); ZSTD_updateStats(optStatePtr, llen, anchor, offCode, mlen); - ZSTD_storeSeq(seqStore, llen, anchor, offCode, mlen-MINMATCH); + ZSTD_storeSeq(seqStore, llen, anchor, iend, offCode, mlen-MINMATCH); anchor += advance; ip = anchor; } } diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c index 27ce137f..cbb66c8d 100644 --- a/lib/decompress/zstd_decompress_block.c +++ b/lib/decompress/zstd_decompress_block.c @@ -761,10 +761,10 @@ size_t ZSTD_execSequence(BYTE* op, * without overlap checking. */ if (sequence.offset >= WILDCOPY_VECLEN) { - /* Split out matchLength <= 32 since it is nearly always true. +1% on gcc-9. - * We copy 32 bytes here since matches are generally longer than literals. - * In silesia, for example ~10% of matches are longer than 16 bytes. - */ + /* We bet on a full wildcopy for matches, since we expect matches to be + * longer than literals (in general). In silesia, ~10% of matches are longer + * than 16 bytes. + */ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); return sequenceLength; } From e068bd01dfc9aec6a2ae49836b6c548ee796fff7 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 20 Sep 2019 01:09:47 -0700 Subject: [PATCH 112/144] [tests] Fix decodecorpus --- tests/decodecorpus.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index dbc27bc9..91873ba4 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -758,8 +758,8 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, DISPLAYLEVEL(7, " repeat offset: %d\n", (int)repIndex); } /* use libzstd sequence handling */ - ZSTD_storeSeq(seqStore, literalLen, literals, offsetCode, - matchLen - MINMATCH); + ZSTD_storeSeq(seqStore, literalLen, literals, literals + literalLen, + offsetCode, matchLen - MINMATCH); literalsSize -= literalLen; excessMatch -= (matchLen - MIN_SEQ_LEN); From 67b1f5fc72cd23433e9dbdfcdaae1773c1e805b8 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 20 Sep 2019 01:23:35 -0700 Subject: [PATCH 113/144] Fix too strict assert --- lib/compress/zstd_compress_internal.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 579bd5d4..bc654bcc 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -360,10 +360,8 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B /* copy Literals */ assert(seqStorePtr->maxNbLit <= 128 KB); assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit); - /* We are guaranteed at least 8 bytes of literals space because of HASH_READ_SIZE and - * MINMATCH. - */ - assert(litLimit - literals >= HASH_READ_SIZE + MINMATCH); + /* We are guaranteed at least 8 bytes of literals space because of HASH_READ_SIZE. */ + assert(litLimit - literals >= HASH_READ_SIZE); if (litLimit - literals >= WILDCOPY_OVERLENGTH) ZSTD_wildcopy(seqStorePtr->lit, literals, (ptrdiff_t)litLength, ZSTD_no_overlap); else From fde217df0440ed5b05fd6f0ceb53d73b4902820e Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 20 Sep 2019 08:25:12 -0700 Subject: [PATCH 114/144] Fix bounds check in ZSTD_storeSeq() --- lib/compress/zstd_compress_internal.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index bc654bcc..83221757 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -361,8 +361,8 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B assert(seqStorePtr->maxNbLit <= 128 KB); assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit); /* We are guaranteed at least 8 bytes of literals space because of HASH_READ_SIZE. */ - assert(litLimit - literals >= HASH_READ_SIZE); - if (litLimit - literals >= WILDCOPY_OVERLENGTH) + assert(literals + litLength + HASH_READ_SIZE <= litLimit); + if (literals + litLength + WILDCOPY_OVERLENGTH <= litLimit) ZSTD_wildcopy(seqStorePtr->lit, literals, (ptrdiff_t)litLength, ZSTD_no_overlap); else ZSTD_wildcopy8(seqStorePtr->lit, literals, (ptrdiff_t)litLength); From f7d9b36835aa7c5ac27c63b8847ba14543f919cc Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 20 Sep 2019 14:11:29 -0400 Subject: [PATCH 115/144] Update Comment on `ZSTD_estimateCCtxSize()` --- lib/zstd.h | 25 +++++++++++++++++-------- tests/fuzzer.c | 6 +++--- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/lib/zstd.h b/lib/zstd.h index 52266ab5..c2fbc3c9 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1221,14 +1221,23 @@ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); ***************************************/ /*! ZSTD_estimate*() : - * These functions make it possible to estimate memory usage - * of a future {D,C}Ctx, before its creation. - * ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one. - * It will also consider src size to be arbitrarily "large", which is worst case. - * If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation. - * ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. - * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. - * Note : CCtx size estimation is only correct for single-threaded compression. */ + * These functions make it possible to estimate memory usage of a future + * {D,C}Ctx, before its creation. + * + * ZSTD_estimateCCtxSize() will provide a budget large enough for any + * compression level up to selected one. Unlike ZSTD_estimateCStreamSize*(), + * this estimate does not include space for a window buffer, so this estimate + * is guaranteed to be enough for single-shot compressions, but not streaming + * compressions. It will however assume the input may be arbitrarily large, + * which is the worst case. If srcSize is known to always be small, + * ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation. + * ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with + * ZSTD_getCParams() to create cParams from compressionLevel. + * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with + * ZSTD_CCtxParams_setParameter(). + * + * Note: only single-threaded compression is supported. This function will + * return an error code if ZSTD_c_nbWorkers is >= 1. */ ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 0ae0b394..3baa9b07 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -2156,9 +2156,9 @@ static int basicUnitTests(U32 const seed, double compressibility) size_t approxIndex = 0; size_t maxIndex = ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)); /* ZSTD_CURRENT_MAX from zstd_compress_internal.h */ - /* vastly overprovision space in a static context so that we can do all - * this without ever reallocating, which would reset the indices */ - size_t const staticCCtxSize = 2 * ZSTD_estimateCCtxSize(22); + /* Provision enough space in a static context so that we can do all + * this without ever reallocating, which would reset the indices. */ + size_t const staticCCtxSize = ZSTD_estimateCStreamSize(22); void* const staticCCtxBuffer = malloc(staticCCtxSize); ZSTD_CCtx* cctx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize); From 44c65da97ea36e4ba8ca6968979123fafca5485c Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 20 Sep 2019 12:23:25 -0700 Subject: [PATCH 116/144] Remove literals overread in ZSTD_storeSeq() for ~neutral perf --- lib/common/zstd_internal.h | 2 +- lib/compress/zstd_compress_internal.h | 36 ++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 522c1fda..9dc9c09e 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -214,7 +214,7 @@ typedef enum { * The src buffer must be before the dst buffer. */ MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE -void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e ovtype) +void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e const ovtype) { ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; const BYTE* ip = (const BYTE*)src; diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 83221757..e80686cc 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -340,6 +340,21 @@ MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat) return (srcSize >> minlog) + 2; } +/*! ZSTD_safecopyLiterals() : + * memcpy() function that won't read beyond more than WILDCOPY_OVERLENGTH bytes past ilimit_w. + * Only called when the sequence ends past ilimit_w, so it only needs to be optimized for single + * large copies. + */ +static void ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE const* ilimit_w) { + assert(iend > ilimit_w); + if (ip <= ilimit_w) { + ZSTD_wildcopy(op, ip, ilimit_w - ip, ZSTD_no_overlap); + op += ilimit_w - ip; + ip = ilimit_w; + } + while (ip < iend) *op++ = *ip++; +} + /*! ZSTD_storeSeq() : * Store a sequence (litlen, litPtr, offCode and mlBase) into seqStore_t. * `offCode` : distance to match + ZSTD_REP_MOVE (values <= ZSTD_REP_MOVE are repCodes). @@ -348,6 +363,8 @@ MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat) */ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, const BYTE* litLimit, U32 offCode, size_t mlBase) { + BYTE const* const litLimit_w = litLimit - WILDCOPY_OVERLENGTH; + BYTE const* const litEnd = literals + litLength; #if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6) static const BYTE* g_start = NULL; if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */ @@ -360,12 +377,19 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B /* copy Literals */ assert(seqStorePtr->maxNbLit <= 128 KB); assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit); - /* We are guaranteed at least 8 bytes of literals space because of HASH_READ_SIZE. */ - assert(literals + litLength + HASH_READ_SIZE <= litLimit); - if (literals + litLength + WILDCOPY_OVERLENGTH <= litLimit) - ZSTD_wildcopy(seqStorePtr->lit, literals, (ptrdiff_t)litLength, ZSTD_no_overlap); - else - ZSTD_wildcopy8(seqStorePtr->lit, literals, (ptrdiff_t)litLength); + assert(literals + litLength <= litLimit); + if (litEnd <= litLimit_w) { + /* Common case we can use wildcopy. + * First copy 16 bytes, because literals are likely short. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(seqStorePtr->lit, literals); + if (litLength > 16) { + ZSTD_wildcopy(seqStorePtr->lit+16, literals+16, (ptrdiff_t)litLength-16, ZSTD_no_overlap); + } + } else { + ZSTD_safecopyLiterals(seqStorePtr->lit, literals, litEnd, litLimit_w); + } seqStorePtr->lit += litLength; /* literal Length */ From f3c4fd17e30465a4ec90152c9f858dc8ff674b7f Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Fri, 20 Sep 2019 15:50:58 -0700 Subject: [PATCH 117/144] Passing in dummy dst buffer of compressbound(srcSize) --- lib/compress/zstd_compress.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index ce352acd..e7ec1d61 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2319,6 +2319,9 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, size_t outSeqsSize, const void* src, size_t srcSize) { + const size_t dstCapacity = ZSTD_compressBound(srcSize); + void* dst = ZSTD_malloc(dstCapacity, ZSTD_defaultCMem); + SeqCollector seqCollector; seqCollector.collectSequences = 1; seqCollector.seqStart = outSeqs; @@ -2326,8 +2329,8 @@ size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, seqCollector.maxSequences = outSeqsSize; zc->seqCollector = seqCollector; - /* We never write to dst when collecing sequences so setting dst = src is harmless */ - ZSTD_compress2(zc, (void*)src, srcSize, src, srcSize); + ZSTD_compress2(zc, dst, dstCapacity, src, srcSize); + ZSTD_free(dst, ZSTD_defaultCMem); return zc->seqCollector.seqIndex; } From 5dc0a1d65921821eddfcf0a16ef4a3d79be0ba3f Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 20 Sep 2019 16:39:27 -0700 Subject: [PATCH 118/144] HINT_INLINE ZSTD_storeSeq() Clang on Mac wasn't inlining `ZSTD_storeSeq()` in level 1, which was causing a 5% performance regression. This fixes it. --- lib/compress/zstd_compress_internal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index e80686cc..42660e1e 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -361,7 +361,7 @@ static void ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const ie * `mlBase` : matchLength - MINMATCH * Allowed to overread literals up to litLimit. */ -MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, const BYTE* litLimit, U32 offCode, size_t mlBase) +HINT_INLINE void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, const BYTE* litLimit, U32 offCode, size_t mlBase) { BYTE const* const litLimit_w = litLimit - WILDCOPY_OVERLENGTH; BYTE const* const litEnd = literals + litLength; From 5cb7615f1f3313b4dbd8ab06dc54ad946db324d7 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 20 Sep 2019 21:37:13 -0700 Subject: [PATCH 119/144] Add UNUSED_ATTR to ZSTD_storeSeq() --- lib/common/compiler.h | 7 +++++++ lib/compress/zstd_compress_internal.h | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/common/compiler.h b/lib/common/compiler.h index 36584aa6..1877a0c1 100644 --- a/lib/common/compiler.h +++ b/lib/common/compiler.h @@ -61,6 +61,13 @@ # define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR #endif +/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */ +#if defined(__GNUC__) +# define UNUSED_ATTR __attribute__((unused)) +#else +# define UNUSED_ATTR +#endif + /* force no inlining */ #ifdef _MSC_VER # define FORCE_NOINLINE static __declspec(noinline) diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 42660e1e..1140945b 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -361,7 +361,8 @@ static void ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const ie * `mlBase` : matchLength - MINMATCH * Allowed to overread literals up to litLimit. */ -HINT_INLINE void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, const BYTE* litLimit, U32 offCode, size_t mlBase) +HINT_INLINE UNUSED_ATTR +void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, const BYTE* litLimit, U32 offCode, size_t mlBase) { BYTE const* const litLimit_w = litLimit - WILDCOPY_OVERLENGTH; BYTE const* const litEnd = literals + litLength; From 1f7228c040f614cde72ad0c48a819284f9ecd2f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Bolvansk=C3=BD?= Date: Mon, 23 Sep 2019 21:23:09 +0200 Subject: [PATCH 120/144] Use clz ^ 31 instead of 31 - clz; better codegen for GCC --- lib/common/bitstream.h | 2 +- lib/legacy/zstd_v01.c | 2 +- lib/legacy/zstd_v02.c | 2 +- lib/legacy/zstd_v03.c | 2 +- lib/legacy/zstd_v04.c | 2 +- lib/legacy/zstd_v05.c | 2 +- lib/legacy/zstd_v06.c | 2 +- lib/legacy/zstd_v07.c | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/common/bitstream.h b/lib/common/bitstream.h index 306e019c..1c294b80 100644 --- a/lib/common/bitstream.h +++ b/lib/common/bitstream.h @@ -164,7 +164,7 @@ MEM_STATIC unsigned BIT_highbit32 (U32 val) _BitScanReverse ( &r, val ); return (unsigned) r; # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ - return 31 - __builtin_clz (val); + return __builtin_clz (val) ^ 31; # elif defined(__ICCARM__) /* IAR Intrinsic */ return 31 - __CLZ(val); # else /* Software version */ diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c index ae8cba2a..8112527f 100644 --- a/lib/legacy/zstd_v01.c +++ b/lib/legacy/zstd_v01.c @@ -346,7 +346,7 @@ FORCE_INLINE unsigned FSE_highbit32 (U32 val) _BitScanReverse ( &r, val ); return (unsigned) r; # elif defined(__GNUC__) && (GCC_VERSION >= 304) /* GCC Intrinsic */ - return 31 - __builtin_clz (val); + return __builtin_clz (val) ^ 31; # else /* Software version */ static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; U32 v = val; diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c index de0a4bd6..c8783799 100644 --- a/lib/legacy/zstd_v02.c +++ b/lib/legacy/zstd_v02.c @@ -353,7 +353,7 @@ MEM_STATIC unsigned BIT_highbit32 (U32 val) _BitScanReverse ( &r, val ); return (unsigned) r; # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ - return 31 - __builtin_clz (val); + return __builtin_clz (val) ^ 31; # else /* Software version */ static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; U32 v = val; diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c index dbc83f1e..162bd630 100644 --- a/lib/legacy/zstd_v03.c +++ b/lib/legacy/zstd_v03.c @@ -356,7 +356,7 @@ MEM_STATIC unsigned BIT_highbit32 (U32 val) _BitScanReverse ( &r, val ); return (unsigned) r; # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ - return 31 - __builtin_clz (val); + return __builtin_clz (val) ^ 31; # else /* Software version */ static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; U32 v = val; diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index 201ce2b6..4dec3081 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -627,7 +627,7 @@ MEM_STATIC unsigned BIT_highbit32 (U32 val) _BitScanReverse ( &r, val ); return (unsigned) r; # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ - return 31 - __builtin_clz (val); + return __builtin_clz (val) ^ 31; # else /* Software version */ static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; U32 v = val; diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index e347b00d..570e0ff8 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -756,7 +756,7 @@ MEM_STATIC unsigned BITv05_highbit32 (U32 val) _BitScanReverse ( &r, val ); return (unsigned) r; # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ - return 31 - __builtin_clz (val); + return __builtin_clz (val) ^ 31; # else /* Software version */ static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; U32 v = val; diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index f907a3a7..2a08e8de 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -860,7 +860,7 @@ MEM_STATIC unsigned BITv06_highbit32 ( U32 val) _BitScanReverse ( &r, val ); return (unsigned) r; # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ - return 31 - __builtin_clz (val); + return __builtin_clz (val) ^ 31; # else /* Software version */ static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; U32 v = val; diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index a83ddc9a..a2eeff80 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -530,7 +530,7 @@ MEM_STATIC unsigned BITv07_highbit32 (U32 val) _BitScanReverse ( &r, val ); return (unsigned) r; # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ - return 31 - __builtin_clz (val); + return __builtin_clz (val) ^ 31; # else /* Software version */ static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; U32 v = val; From 1ab1a40c9ce8d189d2901a1a8536926863211ad6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Bolvansk=C3=BD?= Date: Mon, 23 Sep 2019 21:32:56 +0200 Subject: [PATCH 121/144] Fixed one more place --- lib/common/zstd_internal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 9dc9c09e..f791c5b3 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -314,7 +314,7 @@ MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus _BitScanReverse(&r, val); return (unsigned)r; # elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ - return 31 - __builtin_clz(val); + return __builtin_clz (val) ^ 31; # elif defined(__ICCARM__) /* IAR Intrinsic */ return 31 - __CLZ(val); # else /* Software version */ From be0bebd24e08833f757daaee3c172514c0cc811f Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Mon, 23 Sep 2019 15:08:18 -0700 Subject: [PATCH 122/144] Adding test and null check for malloc --- lib/compress/zstd_compress.c | 4 +++- tests/fuzzer.c | 26 ++++++++++++++++++++++---- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index e7ec1d61..8eb29289 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2321,8 +2321,10 @@ size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, { const size_t dstCapacity = ZSTD_compressBound(srcSize); void* dst = ZSTD_malloc(dstCapacity, ZSTD_defaultCMem); - SeqCollector seqCollector; + + assert(dst != NULL); + seqCollector.collectSequences = 1; seqCollector.seqStart = outSeqs; seqCollector.seqIndex = 0; diff --git a/tests/fuzzer.c b/tests/fuzzer.c index a513e1b4..349f0d19 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1962,11 +1962,29 @@ static int basicUnitTests(U32 const seed, double compressibility) { ZSTD_CCtx* cctx = ZSTD_createCCtx(); - assert(cctx != NULL); + size_t zerosLength = ZSTD_BLOCKSIZE_MAX * 2 - 1; + size_t expectedOffsets[] = {1, 1}; + size_t expectedLitLengths[] = {2, 1}; + size_t expectedMatchLengths[] = {ZSTD_BLOCKSIZE_MAX - 2, ZSTD_BLOCKSIZE_MAX - 2}; + size_t expectedReps[] = {1, 1}; + size_t expectedMatchPos[] = {2, 1}; + size_t expectedSequencesSize = 2; + size_t sequencesSize; + size_t i = 0; + ZSTD_Sequence* sequences = (ZSTD_Sequence*)compressedBuffer; DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences zeros : ", testNb++); - memset(CNBuffer, 0, 1000000); - assert(ZSTD_getSequences(cctx, (ZSTD_Sequence*)compressedBuffer, 1000000, - CNBuffer, 1000000) == 1000000 / 131071 + 1); + assert(cctx != NULL); + memset(CNBuffer, 0, zerosLength); + sequencesSize = ZSTD_getSequences(cctx, sequences, 10, + CNBuffer, zerosLength); + assert(sequencesSize == expectedSequencesSize); + for (i = 0; i < sequencesSize; ++i) { + assert(sequences[i].offset == expectedOffsets[i]); + assert(sequences[i].litLength == expectedLitLengths[i]); + assert(sequences[i].matchLength == expectedMatchLengths[i]); + assert(sequences[i].rep == expectedReps[i]); + assert(sequences[i].matchPos == expectedMatchPos[i]); + } ZSTD_freeCCtx(cctx); } From c04245b257e25c26cdb15507308575b1f16108e4 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Mon, 23 Sep 2019 15:42:16 -0700 Subject: [PATCH 123/144] Replacing assert with memory_allocation error code throw --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 8eb29289..7facbeff 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2323,7 +2323,7 @@ size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, void* dst = ZSTD_malloc(dstCapacity, ZSTD_defaultCMem); SeqCollector seqCollector; - assert(dst != NULL); + RETURN_ERROR_IF(dst == NULL, memory_allocation); seqCollector.collectSequences = 1; seqCollector.seqStart = outSeqs; From ad2a2785f7cf470ebe458e015671e6e8e1f010d2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 24 Sep 2019 15:15:33 -0700 Subject: [PATCH 124/144] bump version number to v1.4.4 so that future reports on `dev` branch use this number instead --- doc/zstd_manual.html | 41 +++++++++++++++++++++++++++++------------ lib/zstd.h | 2 +- programs/zstd.1 | 10 +++++++++- programs/zstdgrep.1 | 2 +- programs/zstdless.1 | 2 +- 5 files changed, 41 insertions(+), 16 deletions(-) diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index 26b204e1..79b9d023 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -1,10 +1,10 @@ -zstd 1.4.3 Manual +zstd 1.4.4 Manual -

zstd 1.4.3 Manual

+

zstd 1.4.4 Manual

Contents

    @@ -324,6 +324,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); * ZSTD_c_forceAttachDict * ZSTD_c_literalCompressionMode * ZSTD_c_targetCBlockSize + * ZSTD_c_srcSizeHint * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly; * also, the enums values themselves are unstable and can still change. @@ -334,6 +335,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); ZSTD_c_experimentalParam4=1001, ZSTD_c_experimentalParam5=1002, ZSTD_c_experimentalParam6=1003, + ZSTD_c_experimentalParam7=1004, } ZSTD_cParameter; 
    typedef struct {
@@ -1005,14 +1007,23 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
 size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
 size_t ZSTD_estimateDCtxSize(void);
-
      These functions make it possible to estimate memory usage
-  of a future {D,C}Ctx, before its creation.
-  ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one.
-  It will also consider src size to be arbitrarily "large", which is worst case.
-  If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation.
-  ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
-  ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
-  Note : CCtx size estimation is only correct for single-threaded compression. 
+
      These functions make it possible to estimate memory usage of a future
+  {D,C}Ctx, before its creation.
+
+  ZSTD_estimateCCtxSize() will provide a budget large enough for any
+  compression level up to selected one. Unlike ZSTD_estimateCStreamSize*(),
+  this estimate does not include space for a window buffer, so this estimate
+  is guaranteed to be enough for single-shot compressions, but not streaming
+  compressions. It will however assume the input may be arbitrarily large,
+  which is the worst case. If srcSize is known to always be small,
+  ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation.
+  ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with
+  ZSTD_getCParams() to create cParams from compressionLevel.
+  ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with
+  ZSTD_CCtxParams_setParameter().
+
+  Note: only single-threaded compression is supported. This function will
+  return an error code if ZSTD_c_nbWorkers is >= 1. 
 

    size_t ZSTD_estimateCStreamSize(int compressionLevel);
@@ -1318,7 +1329,10 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di
 /**! ZSTD_initCStream_advanced() :
  * This function is deprecated, and is approximately equivalent to:
  *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
- *     ZSTD_CCtx_setZstdParams(zcs, params); // Set the zstd params and leave the rest as-is
+ *     // Pseudocode: Set each zstd parameter and leave the rest as-is.
+ *     for ((param, value) : params) {
+ *         ZSTD_CCtx_setParameter(zcs, param, value);
+ *     }
  *     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
  *     ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
  *
@@ -1338,7 +1352,10 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
 /**! ZSTD_initCStream_usingCDict_advanced() :
  * This function is deprecated, and is approximately equivalent to:
  *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
- *     ZSTD_CCtx_setZstdFrameParams(zcs, fParams); // Set the zstd frame params and leave the rest as-is
+ *     // Pseudocode: Set each zstd frame parameter and leave the rest as-is.
+ *     for ((fParam, value) : fParams) {
+ *         ZSTD_CCtx_setParameter(zcs, fParam, value);
+ *     }
  *     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
  *     ZSTD_CCtx_refCDict(zcs, cdict);
  *
diff --git a/lib/zstd.h b/lib/zstd.h
index c2fbc3c9..42d4188c 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -72,7 +72,7 @@ extern "C" {
 /*------   Version   ------*/
 #define ZSTD_VERSION_MAJOR    1
 #define ZSTD_VERSION_MINOR    4
-#define ZSTD_VERSION_RELEASE  3
+#define ZSTD_VERSION_RELEASE  4
 
 #define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
 ZSTDLIB_API unsigned ZSTD_versionNumber(void);   /**< to check runtime library version */
diff --git a/programs/zstd.1 b/programs/zstd.1
index 4b7273ff..bb5103c6 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -1,5 +1,5 @@
 .
-.TH "ZSTD" "1" "August 2019" "zstd 1.4.3" "User Commands"
+.TH "ZSTD" "1" "September 2019" "zstd 1.4.4" "User Commands"
 .
 .SH "NAME"
 \fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@@ -127,6 +127,14 @@ Does not spawn a thread for compression, use a single thread for both I/O and co
 \fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. Adaptation can be constrained between supplied \fBmin\fR and \fBmax\fR levels\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\. \fInote\fR : at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
 .
 .TP
+\fB\-\-stream\-size=#\fR
+Sets the pledged source size of input coming from a stream\. This value must be exact, as it will be included in the produced frame header\. Incorrect stream sizes will cause an error\. This information will be used to better optimize compression parameters, resulting in better and potentially faster compression, especially for smaller source sizes\.
+.
+.TP
+\fB\-\-size\-hint=#\fR
+When handling input from a stream, \fBzstd\fR must guess how large the source size will be when optimizing compression parameters\. If the stream size is relatively small, this guess may be a poor one, resulting in a higher compression ratio than expected\. This feature allows for controlling the guess when needed\. Exact guesses result in better compression ratios\. Overestimates result in slightly degraded compression ratios, while underestimates may result in significant degradation\.
+.
+.TP
 \fB\-\-rsyncable\fR
 \fBzstd\fR will periodically synchronize the compression state to make the compressed file more rsync\-friendly\. There is a negligible impact to compression ratio, and the faster compression levels will see a small compression speed hit\. This feature does not work with \fB\-\-single\-thread\fR\. You probably don\'t want to use it with long range mode, since it will decrease the effectiveness of the synchronization points, but your milage may vary\.
 .
diff --git a/programs/zstdgrep.1 b/programs/zstdgrep.1
index 456298f8..06927ab7 100644
--- a/programs/zstdgrep.1
+++ b/programs/zstdgrep.1
@@ -1,5 +1,5 @@
 .
-.TH "ZSTDGREP" "1" "August 2019" "zstd 1.4.3" "User Commands"
+.TH "ZSTDGREP" "1" "September 2019" "zstd 1.4.4" "User Commands"
 .
 .SH "NAME"
 \fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files
diff --git a/programs/zstdless.1 b/programs/zstdless.1
index 42156fd2..d4904227 100644
--- a/programs/zstdless.1
+++ b/programs/zstdless.1
@@ -1,5 +1,5 @@
 .
-.TH "ZSTDLESS" "1" "August 2019" "zstd 1.4.3" "User Commands"
+.TH "ZSTDLESS" "1" "September 2019" "zstd 1.4.4" "User Commands"
 .
 .SH "NAME"
 \fBzstdless\fR \- view zstandard\-compressed files

From bb27472afcf5a8f6499638f30221e62850bba0e7 Mon Sep 17 00:00:00 2001
From: Bimba Shrestha 
Date: Thu, 26 Sep 2019 15:38:31 -0700
Subject: [PATCH 125/144] Adding more realistic test for get sequences

---
 tests/fuzzer.c | 63 ++++++++++++++++++++++++++++++++------------------
 1 file changed, 41 insertions(+), 22 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 349f0d19..66d1c7e0 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -304,6 +304,26 @@ static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
 
 #endif
 
+static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize, BYTE* src, size_t size)
+{
+    size_t i;
+    for(i = 0; i < seqsSize; ++i) {
+        assert(dst + seqs[i].litLength + seqs[i].matchLength < dst + size);
+        assert(src + seqs[i].litLength + seqs[i].matchLength < src + size);
+
+        memcpy(dst, src, seqs[i].litLength);
+        dst += seqs[i].litLength;
+        src += seqs[i].litLength;
+        size -= seqs[i].litLength;
+
+        memcpy(dst, dst-seqs[i].offset, seqs[i].matchLength);
+        dst += seqs[i].matchLength;
+        src += seqs[i].matchLength;
+        size -= seqs[i].matchLength;
+    }
+    memcpy(dst, src, size);
+}
+
 /*=============================================
 *   Unit tests
 =============================================*/
@@ -1960,32 +1980,31 @@ static int basicUnitTests(U32 const seed, double compressibility)
         DISPLAYLEVEL(3, "OK \n");
     }
 
+    DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences decode from sequences test : ", testNb++);
     {
+        size_t srcSize = sizeof(U32) * 1000;
+        BYTE* src = (BYTE*)CNBuffer;
+        BYTE* decoded = (BYTE*)compressedBuffer;
+
         ZSTD_CCtx* cctx = ZSTD_createCCtx();
-        size_t zerosLength = ZSTD_BLOCKSIZE_MAX * 2 - 1;
-        size_t expectedOffsets[] = {1, 1};
-        size_t expectedLitLengths[] = {2, 1};
-        size_t expectedMatchLengths[] = {ZSTD_BLOCKSIZE_MAX - 2, ZSTD_BLOCKSIZE_MAX - 2};
-        size_t expectedReps[] = {1, 1};
-        size_t expectedMatchPos[] = {2, 1};
-        size_t expectedSequencesSize = 2;
-        size_t sequencesSize;
-        size_t i = 0;
-        ZSTD_Sequence* sequences = (ZSTD_Sequence*)compressedBuffer;
-        DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences zeros : ", testNb++);
+        ZSTD_Sequence* seqs = malloc(srcSize * sizeof(ZSTD_Sequence));
+        size_t seqsSize; size_t i;
+        U32 randSeed = seed;
+
         assert(cctx != NULL);
-        memset(CNBuffer, 0, zerosLength);
-        sequencesSize = ZSTD_getSequences(cctx, sequences, 10,
-            CNBuffer, zerosLength);
-        assert(sequencesSize == expectedSequencesSize);
-        for (i = 0; i < sequencesSize; ++i) {
-            assert(sequences[i].offset == expectedOffsets[i]);
-            assert(sequences[i].litLength == expectedLitLengths[i]);
-            assert(sequences[i].matchLength == expectedMatchLengths[i]);
-            assert(sequences[i].rep == expectedReps[i]);
-            assert(sequences[i].matchPos == expectedMatchPos[i]);
-        }
+
+        /* Populate src with random data */
+        for (i = 0; i < srcSize / sizeof(U32); ++i) {*((U32*)src + i) = FUZ_rand(&randSeed);}
+
+        /* get the sequences */
+        seqsSize = ZSTD_getSequences(cctx, seqs, srcSize, src, srcSize);
+
+        /* "decode" and compare the sequences */
+        FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize);
+        assert(!memcmp(CNBuffer, compressedBuffer, 5));
+
         ZSTD_freeCCtx(cctx);
+        free(seqs);
     }
 
     /* Multiple blocks of zeros test */

From 75b128635455994ea5218036c5bbd187c67e1998 Mon Sep 17 00:00:00 2001
From: Bimba Shrestha 
Date: Thu, 26 Sep 2019 16:07:34 -0700
Subject: [PATCH 126/144] Fixing shortest failure

---
 tests/fuzzer.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 66d1c7e0..c3687974 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1994,7 +1994,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         assert(cctx != NULL);
 
         /* Populate src with random data */
-        for (i = 0; i < srcSize / sizeof(U32); ++i) {*((U32*)src + i) = FUZ_rand(&randSeed);}
+        for (i = 0; i < srcSize / sizeof(U32); ++i) {((U32*)CNBuffer)[i] = FUZ_rand(&randSeed);}
 
         /* get the sequences */
         seqsSize = ZSTD_getSequences(cctx, seqs, srcSize, src, srcSize);

From 91daee5c068c105d456331ff30fd6971ec9cd437 Mon Sep 17 00:00:00 2001
From: Bimba Shrestha 
Date: Thu, 26 Sep 2019 16:21:57 -0700
Subject: [PATCH 127/144] Fixing appveyor test

---
 tests/fuzzer.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index c3687974..0b935c82 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1987,10 +1987,11 @@ static int basicUnitTests(U32 const seed, double compressibility)
         BYTE* decoded = (BYTE*)compressedBuffer;
 
         ZSTD_CCtx* cctx = ZSTD_createCCtx();
-        ZSTD_Sequence* seqs = malloc(srcSize * sizeof(ZSTD_Sequence));
+        ZSTD_Sequence* seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
         size_t seqsSize; size_t i;
         U32 randSeed = seed;
 
+        if (seqs == NULL) goto _output_error;
         assert(cctx != NULL);
 
         /* Populate src with random data */

From b63a1e7ae526e7544c0bcfdfebf8744f133eb500 Mon Sep 17 00:00:00 2001
From: Bimba Shrestha 
Date: Fri, 27 Sep 2019 07:20:20 -0700
Subject: [PATCH 128/144] Typo fix

---
 tests/fuzzer.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 0b935c82..cf81230d 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -2002,7 +2002,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
         /* "decode" and compare the sequences */
         FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize);
-        assert(!memcmp(CNBuffer, compressedBuffer, 5));
+        assert(!memcmp(CNBuffer, compressedBuffer, srcSize));
 
         ZSTD_freeCCtx(cctx);
         free(seqs);

From f80437c58665294083c27ebba519b303352fa437 Mon Sep 17 00:00:00 2001
From: Sen Huang 
Date: Wed, 2 Oct 2019 11:08:20 -0400
Subject: [PATCH 129/144] Add support for --output-dir-flat

New flag to specify output directory destination for multiple files.
---
 build/LICENSE      |  0
 programs/fileio.c  |  2 +-
 programs/util.c    | 94 ++++++++++++++++++++++++++++++----------------
 programs/util.h    | 11 +++---
 programs/zstdcli.c | 27 ++++++++-----
 tests/playTests.sh | 19 +++++++++-
 6 files changed, 103 insertions(+), 50 deletions(-)
 create mode 100644 build/LICENSE

diff --git a/build/LICENSE b/build/LICENSE
new file mode 100644
index 00000000..e69de29b
diff --git a/programs/fileio.c b/programs/fileio.c
index cc8809f9..c0300f6a 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1453,7 +1453,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN
     }   }
 
     FIO_freeCResources(ress);
-    if (outDirName)
+    if (dstFileNamesTable)
         UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles);
         
     return error;
diff --git a/programs/util.c b/programs/util.c
index 920601fb..e5c93786 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -90,53 +90,81 @@ U32 UTIL_isDirectory(const char* infilename)
     return 0;
 }
 
-int UTIL_createDir(const char* outDirName)
-{
-    int r;
-    if (UTIL_isDirectory(outDirName))
-        return 0;   /* no need to create if directory already exists */
+int UTIL_compareStr(const void *p1, const void *p2) {
+    return strcmp(* (char * const *) p1, * (char * const *) p2);
+}
 
-#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__)
-    r = _mkdir(outDirName);
-    if (r || !UTIL_isDirectory(outDirName)) return 1;
-#else
-    r = mkdir(outDirName, S_IRWXU | S_IRWXG | S_IRWXO); /* dir has all permissions */
-    if (r || !UTIL_isDirectory(outDirName)) return 1;
-#endif
+int UTIL_checkFilenameCollisions(char** dstFilenameTable, unsigned nbFiles) {
+    char** dstFilenameTableSorted;
+    char* prevElem;
+    unsigned u;
+
+    dstFilenameTableSorted = (char**) malloc(sizeof(char*) * nbFiles);
+    if (!dstFilenameTableSorted) {
+        UTIL_DISPLAYLEVEL(1, "Unable to malloc new str array, not checking for name collisions\n");
+        return 1;
+    }
+
+    for (u = 0; u < nbFiles; ++u) {
+        dstFilenameTableSorted[u] = dstFilenameTable[u];
+    }
+    qsort(dstFilenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr);
+    prevElem = dstFilenameTableSorted[0];
+    for (u = 1; u < nbFiles; ++u) {
+        if (strcmp(prevElem, dstFilenameTableSorted[u]) == 0) {
+            UTIL_DISPLAYLEVEL(1, "WARNING: Two files have same filename as source : %s\n", prevElem);
+        }
+        prevElem = dstFilenameTableSorted[u];
+    }
+
+    free(dstFilenameTableSorted);
     return 0;
 }
 
-void UTIL_createDestinationDirTable(const char** filenameTable, unsigned nbFiles,
-                                   const char* outDirName, char** dstFilenameTable)
+void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filenameTable,
+        const unsigned nbFiles, const char* outDirName, const int compressing)
 {
     unsigned u;
-    char c;
-    c = '/';
+    const char* c;
+    #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
+    c = "\\";
+    #else
+    c = "/";
+    #endif
 
-    /* duplicate source file table */
     for (u = 0; u < nbFiles; ++u) {
-        const char* filename;
+        char* filename, *filenameBegin;
         size_t finalPathLen;
         finalPathLen = strlen(outDirName);
-        filename = strrchr(filenameTable[u], c);    /* filename is the last bit of string after '/' */
+        filenameBegin = strrchr(filenameTable[u], c[0]);
+        if (filenameBegin == NULL) {
+            filename = strdup(filenameTable[u]);
+        } else {
+            filename = strdup(filenameBegin+1);
+        }
+
         finalPathLen += strlen(filename);
-        dstFilenameTable[u] = (char*) malloc((finalPathLen+5) * sizeof(char)); /* extra 1 bit for \0, extra 4 for .zst if compressing*/
+        dstFilenameTable[u] = compressing ?
+            (char*) malloc((finalPathLen+6) * sizeof(char)) /* 4 more bytes for .zst suffix */
+          : (char*) malloc((finalPathLen+2) * sizeof(char));
+        if (!dstFilenameTable[u]) {
+            UTIL_DISPLAYLEVEL(1, "Unable to allocate space for file destination str\n");
+            continue;
+        }
+
         strcpy(dstFilenameTable[u], outDirName);
-        strcat(dstFilenameTable[u], filename);
-    } 
-}
+        if (outDirName[strlen(outDirName)-1] == c[0]) {
+            strcat(dstFilenameTable[u], filename);
+        } else {  
+            strcat(dstFilenameTable[u], c);
+            strcat(dstFilenameTable[u], filename);
+        }
 
-void UTIL_processMultipleFilenameDestinationDir(char** dstFilenameTable,
-                                              const char** filenameTable, unsigned filenameIdx,
-                                              const char* outFileName, const char* outDirName) {
-    int dirResult;
-    dirResult = UTIL_createDir(outDirName);
-    if (dirResult)
-        UTIL_DISPLAYLEVEL(1, "Directory creation unsuccessful\n");
+        free(filename);
+    }
 
-    UTIL_createDestinationDirTable(filenameTable, filenameIdx, outDirName, dstFilenameTable);
-    if (outFileName) {
-        outFileName = dstFilenameTable[0]; /* in case -O is called with single file */
+    if (UTIL_checkFilenameCollisions(dstFilenameTable, nbFiles)) {
+        UTIL_DISPLAYLEVEL(1, "Checking for filename collisions failed");
     }
 }
 
diff --git a/programs/util.h b/programs/util.h
index 9615504c..e90b251d 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -127,15 +127,14 @@ int UTIL_fileExist(const char* filename);
 int UTIL_isRegularFile(const char* infilename);
 int UTIL_setFileStat(const char* filename, stat_t* statbuf);
 U32 UTIL_isDirectory(const char* infilename);
-int UTIL_createDir(const char* outDirName);
 int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
 int UTIL_isSameFile(const char* file1, const char* file2);
-void UTIL_createDestinationDirTable(const char** filenameTable, unsigned filenameIdx,
-        const char* outDirName,  char** dstFilenameTable);
+int UTIL_compareStr(const void *p1, const void *p2);
+int UTIL_checkFilenameCollisions(char** dstFilenameTable, unsigned nbFiles);
+/* Populates dstFilenameTable using outDirName concatenated with entries from filenameTable */
+void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filenameTable, const unsigned nbFiles,
+    const char* outDirName, const int compressing);
 void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles);
-void UTIL_processMultipleFilenameDestinationDir(char** dstFilenameTable,
-                                              const char** filenameTable, unsigned filenameIdx,
-                                              const char* outFileName, const char* outDirName);
 
 U32 UTIL_isLink(const char* infilename);
 #define UTIL_FILESIZE_UNKNOWN  ((U64)(-1))
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 5fc6b8aa..6b761ffa 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -118,7 +118,6 @@ static int usage(const char* programName)
 #endif
     DISPLAY( " -D file: use `file` as Dictionary \n");
     DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n");
-    DISPLAY( " -O directory: result(s) stored into `directory`, creates one if non-existent \n");
     DISPLAY( " -f     : overwrite output without prompting and (de)compress links \n");
     DISPLAY( "--rm    : remove source file(s) after successful de/compression \n");
     DISPLAY( " -k     : preserve source file(s) (default) \n");
@@ -137,6 +136,7 @@ static int usage_advanced(const char* programName)
     DISPLAY( " -q     : suppress warnings; specify twice to suppress errors too\n");
     DISPLAY( " -c     : force write to standard output, even if it is the console\n");
     DISPLAY( " -l     : print information about zstd compressed files \n");
+    DISPLAY( " --output-dir-flat directory: results stored into `directory` top level \n");
 #ifndef ZSTD_NOCOMPRESS
     DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
     DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
@@ -690,6 +690,7 @@ int main(int argCount, const char* argv[])
                     if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(prefs, 0); continue; }
                     if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(prefs, 1); continue; }
                     if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
+                    if (!strcmp(argument, "--output-dir-flat")) {nextArgumentIsOutDirName=1; lastCommand=1; continue; }
                     if (!strcmp(argument, "--adapt")) { adapt = 1; continue; }
                     if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) CLEAN_RETURN(badusage(programName)); continue; }
                     if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; }
@@ -856,9 +857,6 @@ int main(int argCount, const char* argv[])
 
                         /* destination file name */
                     case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break;
-
-                         /* destination directory name */
-                    case 'O': nextArgumentIsOutDirName=1; lastCommand=1; argument++; break;
                    
                         /* limit decompression memory */
                     case 'M':
@@ -1178,9 +1176,14 @@ int main(int argCount, const char* argv[])
         if (adaptMax < cLevel) cLevel = adaptMax;
 
         if (outDirName) {
-            printf("ok\n");
-            dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*));
-            UTIL_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName);
+            if (UTIL_isDirectory(outDirName)) {
+                DISPLAY("Output of files will be in directory: %s\n", outDirName);
+                dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*));
+                UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName, 1);
+            } else {
+                DISPLAY("%s is not a directory!\n", outDirName);
+                CLEAN_RETURN(1);
+            }
         } else {
             dstFilenameTable = NULL;
         }
@@ -1205,8 +1208,14 @@ int main(int argCount, const char* argv[])
         FIO_setMemLimit(prefs, memLimit);
 
         if (outDirName) {
-            dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*));
-            UTIL_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName);
+            if (UTIL_isDirectory(outDirName)) {
+                DISPLAY("Output of files will be in directory: %s\n", outDirName);
+                dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*));
+                UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName, 1);
+            } else {
+                DISPLAY("%s is not a directory!\n", outDirName);
+                CLEAN_RETURN(1);
+            }
         } else {
             dstFilenameTable = NULL;
         }
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 19fc514f..8851da2f 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -264,6 +264,24 @@ if [ "$?" -eq 139 ]; then
 fi
 rm tmp*
 
+println "test : compress multiple files into an output directory, --output-dir-flat"
+println henlo > tmp1
+mkdir tmpInputTestDir
+mkdir tmpInputTestDir/we
+mkdir tmpInputTestDir/we/must
+mkdir tmpInputTestDir/we/must/go
+mkdir tmpInputTestDir/we/must/go/deeper
+println cool > tmpInputTestDir/we/must/go/deeper/tmp2
+mkdir tmpOutDir
+$ZSTD tmp1 tmpInputTestDir/we/must/go/deeper/tmp2 --output-dir-flat tmpOutDir
+test -f tmpOutDir/tmp1.zst
+test -f tmpOutDir/tmp2.zst
+println "test : decompress multiple files into an output directory, --output-dir-flat"
+mkdir tmpOutDirDecomp
+$ZSTD tmpOutDir/ -r -d --output-dir-flat tmpOutDirDecomp
+test -f tmpOutDirDecomp/tmp2
+test -f tmpOutDirDecomp/tmp1
+rm -rf tmp*
 
 println "\n===>  Advanced compression parameters "
 println "Hello world!" | $ZSTD --zstd=windowLog=21,      - -o tmp.zst && die "wrong parameters not detected!"
@@ -407,7 +425,6 @@ ls -ls tmp* # check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3))
 println "compress multiple files including a missing one (notHere) : "
 $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
 
-
 println "\n===>  stream-size mode"
 
 ./datagen -g11000 > tmp

From 1e4d602412241c142b7f57e1d7f4995e12a003e0 Mon Sep 17 00:00:00 2001
From: Sen Huang 
Date: Wed, 2 Oct 2019 11:11:35 -0400
Subject: [PATCH 130/144] Modified message

---
 programs/zstdcli.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 6b761ffa..63e71e7e 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -136,7 +136,7 @@ static int usage_advanced(const char* programName)
     DISPLAY( " -q     : suppress warnings; specify twice to suppress errors too\n");
     DISPLAY( " -c     : force write to standard output, even if it is the console\n");
     DISPLAY( " -l     : print information about zstd compressed files \n");
-    DISPLAY( " --output-dir-flat directory: results stored into `directory` top level \n");
+    DISPLAY( " --output-dir-flat directory: results stored into `directory`. Filename collisions mean first file will be compressed. With -f, the last file will be compressed.\n");
 #ifndef ZSTD_NOCOMPRESS
     DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
     DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);

From 147b7614544e411e2087c3e696f4c1be046986cf Mon Sep 17 00:00:00 2001
From: Sen Huang 
Date: Wed, 2 Oct 2019 11:18:14 -0400
Subject: [PATCH 131/144] Removed strdup() dependency

---
 programs/util.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/programs/util.c b/programs/util.c
index e5c93786..083eb12e 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -138,9 +138,11 @@ void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filena
         finalPathLen = strlen(outDirName);
         filenameBegin = strrchr(filenameTable[u], c[0]);
         if (filenameBegin == NULL) {
-            filename = strdup(filenameTable[u]);
+            filename = (char*) malloc((strlen(filenameTable[u])+1) * sizeof(char));
+            strcpy(filename, filenameTable[u]);
         } else {
-            filename = strdup(filenameBegin+1);
+            filename = (char*) malloc((strlen(filenameBegin+1)) * sizeof(char));
+            strcpy(filename, filenameBegin+1);
         }
 
         finalPathLen += strlen(filename);

From b93f1b2a30c55cc0d5938c71f33b9e83f28d2d7a Mon Sep 17 00:00:00 2001
From: Sen Huang 
Date: Wed, 2 Oct 2019 11:29:34 -0400
Subject: [PATCH 132/144] CI Tests fix

---
 programs/util.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/programs/util.c b/programs/util.c
index 083eb12e..aa769567 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -133,7 +133,8 @@ void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filena
     #endif
 
     for (u = 0; u < nbFiles; ++u) {
-        char* filename, *filenameBegin;
+        char* filename;
+        const char* filenameBegin;
         size_t finalPathLen;
         finalPathLen = strlen(outDirName);
         filenameBegin = strrchr(filenameTable[u], c[0]);

From c763457e0a4d70b6912bc14c5908a38c3239a0bf Mon Sep 17 00:00:00 2001
From: Sen Huang 
Date: Wed, 2 Oct 2019 15:30:24 -0400
Subject: [PATCH 133/144] Static analyze fix

---
 programs/util.c    | 7 +++----
 programs/util.h    | 2 +-
 programs/zstdcli.c | 4 ++--
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/programs/util.c b/programs/util.c
index aa769567..83fa70ba 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -122,7 +122,7 @@ int UTIL_checkFilenameCollisions(char** dstFilenameTable, unsigned nbFiles) {
 }
 
 void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filenameTable,
-        const unsigned nbFiles, const char* outDirName, const int compressing)
+        const unsigned nbFiles, const char* outDirName)
 {
     unsigned u;
     const char* c;
@@ -147,11 +147,10 @@ void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filena
         }
 
         finalPathLen += strlen(filename);
-        dstFilenameTable[u] = compressing ?
-            (char*) malloc((finalPathLen+6) * sizeof(char)) /* 4 more bytes for .zst suffix */
-          : (char*) malloc((finalPathLen+2) * sizeof(char));
+        dstFilenameTable[u] = (char*) malloc((finalPathLen+2) * sizeof(char));
         if (!dstFilenameTable[u]) {
             UTIL_DISPLAYLEVEL(1, "Unable to allocate space for file destination str\n");
+            free(filename);
             continue;
         }
 
diff --git a/programs/util.h b/programs/util.h
index e90b251d..558e7a9c 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -133,7 +133,7 @@ int UTIL_compareStr(const void *p1, const void *p2);
 int UTIL_checkFilenameCollisions(char** dstFilenameTable, unsigned nbFiles);
 /* Populates dstFilenameTable using outDirName concatenated with entries from filenameTable */
 void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filenameTable, const unsigned nbFiles,
-    const char* outDirName, const int compressing);
+    const char* outDirName);
 void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles);
 
 U32 UTIL_isLink(const char* infilename);
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 63e71e7e..d9e1be66 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -1179,7 +1179,7 @@ int main(int argCount, const char* argv[])
             if (UTIL_isDirectory(outDirName)) {
                 DISPLAY("Output of files will be in directory: %s\n", outDirName);
                 dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*));
-                UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName, 1);
+                UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName);
             } else {
                 DISPLAY("%s is not a directory!\n", outDirName);
                 CLEAN_RETURN(1);
@@ -1211,7 +1211,7 @@ int main(int argCount, const char* argv[])
             if (UTIL_isDirectory(outDirName)) {
                 DISPLAY("Output of files will be in directory: %s\n", outDirName);
                 dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*));
-                UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName, 1);
+                UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName);
             } else {
                 DISPLAY("%s is not a directory!\n", outDirName);
                 CLEAN_RETURN(1);

From 4dc604cab8f5d5e89454d2afeffe32f2001c41aa Mon Sep 17 00:00:00 2001
From: Sen Huang 
Date: Wed, 2 Oct 2019 18:34:42 -0400
Subject: [PATCH 134/144] Addressed comments on malloc

---
 programs/fileio.c  |  6 ------
 programs/util.c    | 12 ++++++++++--
 programs/zstdcli.c |  5 +++++
 3 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index c0300f6a..6006af01 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1453,9 +1453,6 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN
     }   }
 
     FIO_freeCResources(ress);
-    if (dstFileNamesTable)
-        UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles);
-        
     return error;
 }
 
@@ -2276,9 +2273,6 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
     }
 
     FIO_freeDResources(ress);
-    if (outDirName)
-        UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles);
-
     return error;
 }
 
diff --git a/programs/util.c b/programs/util.c
index 83fa70ba..c12a3188 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -140,14 +140,22 @@ void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filena
         filenameBegin = strrchr(filenameTable[u], c[0]);
         if (filenameBegin == NULL) {
             filename = (char*) malloc((strlen(filenameTable[u])+1) * sizeof(char));
+            if (!filename) {
+                UTIL_DISPLAYLEVEL(1, "Unable to allocate space for filename str\n");
+                continue;
+            }
             strcpy(filename, filenameTable[u]);
         } else {
-            filename = (char*) malloc((strlen(filenameBegin+1)) * sizeof(char));
+            filename = (char*) malloc((strlen(filenameBegin+1)+1) * sizeof(char));
+            if (!filename) {
+                UTIL_DISPLAYLEVEL(1, "Unable to allocate space for filename str\n");
+                continue;
+            }
             strcpy(filename, filenameBegin+1);
         }
 
         finalPathLen += strlen(filename);
-        dstFilenameTable[u] = (char*) malloc((finalPathLen+2) * sizeof(char));
+        dstFilenameTable[u] = (char*) malloc((finalPathLen+3) * sizeof(char));
         if (!dstFilenameTable[u]) {
             UTIL_DISPLAYLEVEL(1, "Unable to allocate space for file destination str\n");
             free(filename);
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index d9e1be66..91e67a2d 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -1192,6 +1192,9 @@ int main(int argCount, const char* argv[])
           operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams);
         else
           operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, dstFilenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
+        
+        if (dstFilenameTable)
+            UTIL_freeDestinationFilenameTable(dstFilenameTable, filenameIdx);
 #else
         (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */
         DISPLAY("Compression not supported \n");
@@ -1224,6 +1227,8 @@ int main(int argCount, const char* argv[])
             operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName);
         else
             operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, dstFilenameTable, outFileName, dictFileName);
+        if (dstFilenameTable)
+            UTIL_freeDestinationFilenameTable(dstFilenameTable, filenameIdx);
 #else
         DISPLAY("Decompression not supported \n");
 #endif

From 61ec4c2e7f7a20856933978b79dcf55a3c68ff74 Mon Sep 17 00:00:00 2001
From: Bimba Shrestha 
Date: Thu, 3 Oct 2019 06:42:40 -0700
Subject: [PATCH 135/144] Cleaning sequence parsing logic

---
 lib/compress/zstd_compress.c | 59 +++++++++++++++---------------------
 1 file changed, 25 insertions(+), 34 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 7facbeff..4e2fbd85 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2272,46 +2272,37 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc)
     size_t seqsSize = seqStore->sequences - seqs;
 
     ZSTD_Sequence* outSeqs = &zc->seqCollector.seqStart[zc->seqCollector.seqIndex];
-    size_t i; size_t position; int repIdx;
+    int i;
 
     assert(zc->seqCollector.seqIndex + 1 < zc->seqCollector.maxSequences);
-    for (i = 0, position = 0; i < seqsSize; ++i) {
-        outSeqs[i].offset = seqs[i].offset;
+    for (i = 0; i < (int)seqsSize; ++i) {
+        unsigned int offsetValue = seqs[i].offset;
         outSeqs[i].litLength = seqs[i].litLength;
-        outSeqs[i].matchLength = seqs[i].matchLength + MINMATCH;
-
-        if (i == seqStore->longLengthPos) {
-            if (seqStore->longLengthID == 1) {
-                outSeqs[i].litLength += 0x10000;
-            } else if (seqStore->longLengthID == 2) {
-                outSeqs[i].matchLength += 0x10000;
-            }
-        }
-
-        if (outSeqs[i].offset <= ZSTD_REP_NUM) {
-            outSeqs[i].rep = outSeqs[i].offset;
-            repIdx = (unsigned int)i - outSeqs[i].offset;
-
-            if (outSeqs[i].litLength == 0) {
-                if (outSeqs[i].offset < 3) {
-                    --repIdx;
-                } else {
-                    repIdx = (unsigned int)i - 1;
-                }
-                ++outSeqs[i].rep;
-            }
-            assert(repIdx >= -3);
-            outSeqs[i].offset = repIdx >= 0 ? outSeqs[repIdx].offset : repStartValue[-repIdx - 1];
-            if (outSeqs[i].rep == 4) {
-                --outSeqs[i].offset;
-            }
+        outSeqs[i].matchLength = seqs[i].matchLength;
+        if (offsetValue > 3) {
+            outSeqs[i].offset = offsetValue - 3;
         } else {
-            outSeqs[i].offset -= ZSTD_REP_NUM;
+            /* special repeat offset case */
+            unsigned int repeatOffset1 = i - 1 >= 0 ? outSeqs[i - 1].offset : 1;
+            unsigned int repeatOffset2 = 1 - 2 >= 0 ? outSeqs[i - 2].offset : 4;
+            unsigned int repeatOffset3 = i - 3 >= 0 ? outSeqs[i - 3].offset : 8;
+            if (seqs[i].litLength != 0) {
+                switch (offsetValue) {
+                    case 1: outSeqs[i].offset = repeatOffset1; break;
+                    case 2: outSeqs[i].offset = repeatOffset2; break;
+                    case 3: outSeqs[i].offset = repeatOffset3; break;
+                }
+            } else {
+                /* offsets shifted by one */
+                switch (offsetValue) {
+                    case 1: outSeqs[i].offset = repeatOffset2; break;
+                    case 2: outSeqs[i].offset = repeatOffset3; break;
+                    /* corner case where offsetValue == 3 */
+                    case 3: outSeqs[i].offset = repeatOffset1 - 1; break;
+                }
+            }
         }
 
-        position += outSeqs[i].litLength;
-        outSeqs[i].matchPos = (unsigned int)position;
-        position += outSeqs[i].matchLength;
     }
     zc->seqCollector.seqIndex += seqsSize;
 }

From 36528b96c4ae2dff700e63851f710aa11c1d0bc8 Mon Sep 17 00:00:00 2001
From: Bimba Shrestha 
Date: Thu, 3 Oct 2019 09:26:51 -0700
Subject: [PATCH 136/144] Manually moving instead of memcpy on decoder and
 using genBuffer()

---
 lib/compress/zstd_compress.c | 59 +++++++++++++++++++++---------------
 tests/fuzzer.c               | 13 ++++----
 2 files changed, 41 insertions(+), 31 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 4e2fbd85..7facbeff 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2272,37 +2272,46 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc)
     size_t seqsSize = seqStore->sequences - seqs;
 
     ZSTD_Sequence* outSeqs = &zc->seqCollector.seqStart[zc->seqCollector.seqIndex];
-    int i;
+    size_t i; size_t position; int repIdx;
 
     assert(zc->seqCollector.seqIndex + 1 < zc->seqCollector.maxSequences);
-    for (i = 0; i < (int)seqsSize; ++i) {
-        unsigned int offsetValue = seqs[i].offset;
+    for (i = 0, position = 0; i < seqsSize; ++i) {
+        outSeqs[i].offset = seqs[i].offset;
         outSeqs[i].litLength = seqs[i].litLength;
-        outSeqs[i].matchLength = seqs[i].matchLength;
-        if (offsetValue > 3) {
-            outSeqs[i].offset = offsetValue - 3;
-        } else {
-            /* special repeat offset case */
-            unsigned int repeatOffset1 = i - 1 >= 0 ? outSeqs[i - 1].offset : 1;
-            unsigned int repeatOffset2 = 1 - 2 >= 0 ? outSeqs[i - 2].offset : 4;
-            unsigned int repeatOffset3 = i - 3 >= 0 ? outSeqs[i - 3].offset : 8;
-            if (seqs[i].litLength != 0) {
-                switch (offsetValue) {
-                    case 1: outSeqs[i].offset = repeatOffset1; break;
-                    case 2: outSeqs[i].offset = repeatOffset2; break;
-                    case 3: outSeqs[i].offset = repeatOffset3; break;
-                }
-            } else {
-                /* offsets shifted by one */
-                switch (offsetValue) {
-                    case 1: outSeqs[i].offset = repeatOffset2; break;
-                    case 2: outSeqs[i].offset = repeatOffset3; break;
-                    /* corner case where offsetValue == 3 */
-                    case 3: outSeqs[i].offset = repeatOffset1 - 1; break;
-                }
+        outSeqs[i].matchLength = seqs[i].matchLength + MINMATCH;
+
+        if (i == seqStore->longLengthPos) {
+            if (seqStore->longLengthID == 1) {
+                outSeqs[i].litLength += 0x10000;
+            } else if (seqStore->longLengthID == 2) {
+                outSeqs[i].matchLength += 0x10000;
             }
         }
 
+        if (outSeqs[i].offset <= ZSTD_REP_NUM) {
+            outSeqs[i].rep = outSeqs[i].offset;
+            repIdx = (unsigned int)i - outSeqs[i].offset;
+
+            if (outSeqs[i].litLength == 0) {
+                if (outSeqs[i].offset < 3) {
+                    --repIdx;
+                } else {
+                    repIdx = (unsigned int)i - 1;
+                }
+                ++outSeqs[i].rep;
+            }
+            assert(repIdx >= -3);
+            outSeqs[i].offset = repIdx >= 0 ? outSeqs[repIdx].offset : repStartValue[-repIdx - 1];
+            if (outSeqs[i].rep == 4) {
+                --outSeqs[i].offset;
+            }
+        } else {
+            outSeqs[i].offset -= ZSTD_REP_NUM;
+        }
+
+        position += outSeqs[i].litLength;
+        outSeqs[i].matchPos = (unsigned int)position;
+        position += outSeqs[i].matchLength;
     }
     zc->seqCollector.seqIndex += seqsSize;
 }
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index cf81230d..a22dafa3 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -307,7 +307,8 @@ static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
 static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize, BYTE* src, size_t size)
 {
     size_t i;
-    for(i = 0; i < seqsSize; ++i) {
+    size_t j;
+    for(i = 0; i < seqsSize - 1; ++i) {
         assert(dst + seqs[i].litLength + seqs[i].matchLength < dst + size);
         assert(src + seqs[i].litLength + seqs[i].matchLength < src + size);
 
@@ -316,7 +317,8 @@ static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize,
         src += seqs[i].litLength;
         size -= seqs[i].litLength;
 
-        memcpy(dst, dst-seqs[i].offset, seqs[i].matchLength);
+        for (j = 0; j < seqs[i].matchLength; ++j)
+            dst[j] = dst[j - seqs[i].offset];
         dst += seqs[i].matchLength;
         src += seqs[i].matchLength;
         size -= seqs[i].matchLength;
@@ -1982,20 +1984,19 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
     DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences decode from sequences test : ", testNb++);
     {
-        size_t srcSize = sizeof(U32) * 1000;
+        size_t srcSize = 100 KB;
         BYTE* src = (BYTE*)CNBuffer;
         BYTE* decoded = (BYTE*)compressedBuffer;
 
         ZSTD_CCtx* cctx = ZSTD_createCCtx();
         ZSTD_Sequence* seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
-        size_t seqsSize; size_t i;
-        U32 randSeed = seed;
+        size_t seqsSize;
 
         if (seqs == NULL) goto _output_error;
         assert(cctx != NULL);
 
         /* Populate src with random data */
-        for (i = 0; i < srcSize / sizeof(U32); ++i) {((U32*)CNBuffer)[i] = FUZ_rand(&randSeed);}
+        RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed);
 
         /* get the sequences */
         seqsSize = ZSTD_getSequences(cctx, seqs, srcSize, src, srcSize);

From 64bc441d7d06b267aba52cfdae681a5582d4cfe7 Mon Sep 17 00:00:00 2001
From: Sen Huang 
Date: Thu, 3 Oct 2019 13:53:04 -0400
Subject: [PATCH 137/144] Now constructs final destination path without
 allocating new table

---
 programs/fileio.c  | 184 +++++++++++++++++++++++++++++++++++----------
 programs/fileio.h  |  14 ++--
 programs/util.c    |  93 -----------------------
 programs/util.h    |   5 --
 programs/zstdcli.c |  37 +--------
 5 files changed, 157 insertions(+), 176 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 6006af01..9988e2d6 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -628,6 +628,104 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
     return (size_t)fileSize;
 }
 
+
+
+/* FIO_checkFilenameCollisions() :
+ * Checks for and warns if there are any files that would have the same output path
+ */
+int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
+    const char** filenameTableSorted;
+    const char* c, *prevElem;
+    char* filename;
+    unsigned u;
+
+    #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
+    c = "\\";
+    #else
+    c = "/";
+    #endif
+
+    filenameTableSorted = (const char**) malloc(sizeof(char*) * nbFiles);
+    if (!filenameTableSorted) {
+        DISPLAY("Unable to malloc new str array, not checking for name collisions\n");
+        return 1;
+    }
+    
+    for (u = 0; u < nbFiles; ++u) {
+        filename = strrchr(filenameTable[u], c[0]);
+        if (filename == NULL) {
+            filenameTableSorted[u] = filenameTable[u];
+        } else {
+            filenameTableSorted[u] = filename+1;
+        }
+    }
+
+    qsort(filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr);
+    prevElem = filenameTableSorted[0];
+    for (u = 1; u < nbFiles; ++u) {
+        if (strcmp(prevElem, filenameTableSorted[u]) == 0) {
+            DISPLAY("WARNING: Two files have same filename: %s\n", prevElem);
+        }
+        prevElem = filenameTableSorted[u];
+    }
+
+    free(filenameTableSorted);
+    return 0;
+}
+
+/* FIO_determineDstFilenameOutdir() :
+ * Takes a source file name and specified output directory, and
+ * allocates memory for and returns a pointer to final path.
+ * This function never returns an error (it may abort() in case of pb)
+ */
+static char*
+FIO_determineDstFilenameOutdir(const char* srcFilename, const char* outDirName, const size_t suffixLen)
+{
+    const char* c, *filenameBegin;
+    char* filename, *result;
+    size_t finalPathLen;
+
+    #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
+    c = "\\";
+    #else
+    c = "/";
+    #endif
+
+    finalPathLen = strlen(outDirName);
+    filenameBegin = strrchr(srcFilename, c[0]);
+    if (filenameBegin == NULL) {
+        filename = (char*) malloc((strlen(srcFilename)+1) * sizeof(char));
+        if (!filename) {
+            EXM_THROW(30, "zstd: %s", strerror(errno));
+        }
+        strcpy(filename, srcFilename);
+    } else {
+        filename = (char*) malloc((strlen(filenameBegin+1)+1) * sizeof(char));
+        if (!filename) {
+            EXM_THROW(30, "zstd: %s", strerror(errno));
+        }
+        strcpy(filename, filenameBegin+1);
+    }
+
+    finalPathLen += strlen(filename);
+    result = (char*) malloc((finalPathLen+suffixLen+30) * sizeof(char));
+    if (!result) {
+        free(filename);
+        EXM_THROW(30, "zstd: %s", strerror(errno));
+    }
+
+    strcpy(result, outDirName);
+    if (outDirName[strlen(outDirName)-1] == c[0]) {
+        strcat(result, filename);
+    } else {  
+        strcat(result, c);
+        strcat(result, filename);
+    }
+
+    free(filename);
+    return result;
+}
+
 #ifndef ZSTD_NOCOMPRESS
 
 /* **********************************************************************
@@ -1379,19 +1477,25 @@ int FIO_compressFilename(FIO_prefs_t* const prefs, const char* dstFileName,
     return result;
 }
 
-
 /* FIO_determineCompressedName() :
  * create a destination filename for compressed srcFileName.
  * @return a pointer to it.
  * This function never returns an error (it may abort() in case of pb)
  */
 static const char*
-FIO_determineCompressedName(const char* srcFileName, const char* suffix)
+FIO_determineCompressedName(const char* srcFileName, const char* outDirName, const char* suffix)
 {
     static size_t dfnbCapacity = 0;
     static char* dstFileNameBuffer = NULL;   /* using static allocation : this function cannot be multi-threaded */
-    size_t const sfnSize = strlen(srcFileName);
+    char* outDirFilename = NULL;
+    size_t sfnSize = strlen(srcFileName);
     size_t const suffixSize = strlen(suffix);
+    if (outDirName) {
+        outDirFilename = FIO_determineDstFilenameOutdir(srcFileName, outDirName, suffixSize);
+        sfnSize = strlen(outDirFilename);
+        assert(outDirFilename != NULL);
+    }
+    
     if (dfnbCapacity <= sfnSize+suffixSize+1) {
         /* resize buffer for dstName */
         free(dstFileNameBuffer);
@@ -1399,9 +1503,16 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix)
         dstFileNameBuffer = (char*)malloc(dfnbCapacity);
         if (!dstFileNameBuffer) {
             EXM_THROW(30, "zstd: %s", strerror(errno));
-    }   }
+        }
+    }
     assert(dstFileNameBuffer != NULL);
-    memcpy(dstFileNameBuffer, srcFileName, sfnSize);
+
+    if (outDirFilename) {
+        memcpy(dstFileNameBuffer, outDirFilename, sfnSize);
+        free(outDirFilename);
+    } else {
+        memcpy(dstFileNameBuffer, srcFileName, sfnSize);
+    }
     memcpy(dstFileNameBuffer+sfnSize, suffix, suffixSize+1 /* Include terminating null */);
     return dstFileNameBuffer;
 }
@@ -1414,24 +1525,17 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix)
  * or into a destination folder (specified with -O)
  */
 int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable,
-                                  const char* outDirName, char** dstFileNamesTable, 
-                                  unsigned nbFiles, const char* outFileName,
-                                  const char* suffix, const char* dictFileName,
-                                  int compressionLevel, ZSTD_compressionParameters comprParams)
+                                  const char* outDirName, unsigned nbFiles, 
+                                  const char* outFileName, const char* suffix, 
+                                  const char* dictFileName, int compressionLevel,
+                                  ZSTD_compressionParameters comprParams)
 {
     int error = 0;
     cRess_t ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams);
 
     /* init */
     assert(outFileName != NULL || suffix != NULL);
-    if (outDirName != NULL) {   /* output into a particular folder */
-        unsigned u;
-        for (u = 0; u < nbFiles; ++u) {
-            const char* const srcFileName = inFileNamesTable[u];
-            const char* const dstFileName = FIO_determineCompressedName(dstFileNamesTable[u], suffix);
-            error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
-        }
-    } else if (outFileName != NULL) {   /* output into a single destination (stdout typically) */
+    if (outFileName != NULL) {   /* output into a single destination (stdout typically) */
         ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName);
         if (ress.dstFile == NULL) {  /* could not open outFileName */
             error = 1;
@@ -1448,9 +1552,11 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN
         unsigned u;
         for (u=0; u= 1 */
-    char** dstFilenameTable;
     unsigned filenameIdx = 0;
     const char* programName = argv[0];
     const char* outFileName = NULL;
@@ -1175,26 +1174,10 @@ int main(int argCount, const char* argv[])
         if (adaptMin > cLevel) cLevel = adaptMin;
         if (adaptMax < cLevel) cLevel = adaptMax;
 
-        if (outDirName) {
-            if (UTIL_isDirectory(outDirName)) {
-                DISPLAY("Output of files will be in directory: %s\n", outDirName);
-                dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*));
-                UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName);
-            } else {
-                DISPLAY("%s is not a directory!\n", outDirName);
-                CLEAN_RETURN(1);
-            }
-        } else {
-            dstFilenameTable = NULL;
-        }
-
         if ((filenameIdx==1) && outFileName)
           operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams);
         else
-          operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, dstFilenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
-        
-        if (dstFilenameTable)
-            UTIL_freeDestinationFilenameTable(dstFilenameTable, filenameIdx);
+          operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
 #else
         (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */
         DISPLAY("Compression not supported \n");
@@ -1209,26 +1192,10 @@ int main(int argCount, const char* argv[])
             }
         }
         FIO_setMemLimit(prefs, memLimit);
-
-        if (outDirName) {
-            if (UTIL_isDirectory(outDirName)) {
-                DISPLAY("Output of files will be in directory: %s\n", outDirName);
-                dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*));
-                UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName);
-            } else {
-                DISPLAY("%s is not a directory!\n", outDirName);
-                CLEAN_RETURN(1);
-            }
-        } else {
-            dstFilenameTable = NULL;
-        }
-        
         if (filenameIdx==1 && outFileName)
             operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName);
         else
-            operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, dstFilenameTable, outFileName, dictFileName);
-        if (dstFilenameTable)
-            UTIL_freeDestinationFilenameTable(dstFilenameTable, filenameIdx);
+            operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, outFileName, dictFileName);
 #else
         DISPLAY("Decompression not supported \n");
 #endif

From 6b81bfb257523f4638e0e1e6005d30776cdffb6e Mon Sep 17 00:00:00 2001
From: Sen Huang 
Date: Thu, 3 Oct 2019 15:23:49 -0400
Subject: [PATCH 138/144] Changed function name, fix test

---
 programs/fileio.c | 24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 9988e2d6..6f1a2d67 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -634,9 +634,7 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
  * Checks for and warns if there are any files that would have the same output path
  */
 int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
-    const char** filenameTableSorted;
-    const char* c, *prevElem;
-    char* filename;
+    const char **filenameTableSorted, *c, *prevElem, *filename;
     unsigned u;
 
     #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
@@ -673,13 +671,13 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
     return 0;
 }
 
-/* FIO_determineDstFilenameOutdir() :
+/* FIO_createFilename_fromOutDir() :
  * Takes a source file name and specified output directory, and
  * allocates memory for and returns a pointer to final path.
  * This function never returns an error (it may abort() in case of pb)
  */
 static char*
-FIO_determineDstFilenameOutdir(const char* srcFilename, const char* outDirName, const size_t suffixLen)
+FIO_createFilename_fromOutDir(const char* srcFilename, const char* outDirName, const size_t suffixLen)
 {
     const char* c, *filenameBegin;
     char* filename, *result;
@@ -1491,7 +1489,7 @@ FIO_determineCompressedName(const char* srcFileName, const char* outDirName, con
     size_t sfnSize = strlen(srcFileName);
     size_t const suffixSize = strlen(suffix);
     if (outDirName) {
-        outDirFilename = FIO_determineDstFilenameOutdir(srcFileName, outDirName, suffixSize);
+        outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, suffixSize);
         sfnSize = strlen(outDirFilename);
         assert(outDirFilename != NULL);
     }
@@ -2279,7 +2277,7 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName)
 {
     static size_t dfnbCapacity = 0;
     static char* dstFileNameBuffer = NULL;   /* using static allocation : this function cannot be multi-threaded */
-    char* dstFilenameOutDir = NULL;
+    char* outDirFilename = NULL;
     size_t sfnSize = strlen(srcFileName);
     size_t suffixSize;
     
@@ -2321,9 +2319,9 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName)
         return NULL;
     }
     if (outDirName) {
-        dstFilenameOutDir = FIO_determineDstFilenameOutdir(srcFileName, outDirName, 0);
-        sfnSize = strlen(dstFilenameOutDir);
-        assert(dstFilenameOutDir != NULL);
+        outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, 0);
+        sfnSize = strlen(outDirFilename);
+        assert(outDirFilename != NULL);
     }
 
     if (dfnbCapacity+suffixSize <= sfnSize+1) {
@@ -2337,9 +2335,9 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName)
 
     /* return dst name == src name truncated from suffix */
     assert(dstFileNameBuffer != NULL);
-    if (dstFilenameOutDir) {
-        memcpy(dstFileNameBuffer, dstFilenameOutDir, sfnSize - suffixSize);
-        free(dstFilenameOutDir);
+    if (outDirFilename) {
+        memcpy(dstFileNameBuffer, outDirFilename, sfnSize - suffixSize);
+        free(outDirFilename);
     } else {
         memcpy(dstFileNameBuffer, srcFileName, sfnSize - suffixSize);
     }

From f9b1da6f62b265f38603e916fd6621198b862a20 Mon Sep 17 00:00:00 2001
From: Sen Huang 
Date: Tue, 8 Oct 2019 00:15:28 -0400
Subject: [PATCH 139/144] Hopefully fix VS compiler warning

---
 programs/fileio.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 6f1a2d67..cc495b85 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -658,7 +658,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
         }
     }
 
-    qsort(filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr);
+    qsort((const char**)filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr);
     prevElem = filenameTableSorted[0];
     for (u = 1; u < nbFiles; ++u) {
         if (strcmp(prevElem, filenameTableSorted[u]) == 0) {
@@ -667,7 +667,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
         prevElem = filenameTableSorted[u];
     }
 
-    free(filenameTableSorted);
+    free((const char**)filenameTableSorted);
     return 0;
 }
 

From 332b5988d8f080aca68901e371b70451f7061fd8 Mon Sep 17 00:00:00 2001
From: Sen Huang 
Date: Tue, 8 Oct 2019 09:44:24 -0400
Subject: [PATCH 140/144] Suppress C4090 warning

---
 programs/fileio.c | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index cc495b85..9ac65abc 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -658,7 +658,13 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
         }
     }
 
-    qsort((const char**)filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr);
+    /* Silence VS Compiler bug with warning C4090 */
+    #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__)
+    #pragma warning (push)
+    #pragma warning (disable : 4090)
+    #endif
+
+    qsort(filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr);
     prevElem = filenameTableSorted[0];
     for (u = 1; u < nbFiles; ++u) {
         if (strcmp(prevElem, filenameTableSorted[u]) == 0) {
@@ -667,7 +673,11 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
         prevElem = filenameTableSorted[u];
     }
 
-    free((const char**)filenameTableSorted);
+    free(filenameTableSorted);
+
+    #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__)
+    #pragma warning (pop)
+    #endif
     return 0;
 }
 

From 6e406b55eeeaa3ce1175459fda4044cbf248e0d1 Mon Sep 17 00:00:00 2001
From: Sen Huang 
Date: Tue, 8 Oct 2019 09:54:59 -0400
Subject: [PATCH 141/144] Casting to void* to avoid C4090 warnings

---
 programs/fileio.c | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 9ac65abc..81633947 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -658,13 +658,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
         }
     }
 
-    /* Silence VS Compiler bug with warning C4090 */
-    #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__)
-    #pragma warning (push)
-    #pragma warning (disable : 4090)
-    #endif
-
-    qsort(filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr);
+    qsort((void*)filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr);
     prevElem = filenameTableSorted[0];
     for (u = 1; u < nbFiles; ++u) {
         if (strcmp(prevElem, filenameTableSorted[u]) == 0) {
@@ -673,11 +667,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
         prevElem = filenameTableSorted[u];
     }
 
-    free(filenameTableSorted);
-
-    #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__)
-    #pragma warning (pop)
-    #endif
+    free((void*)filenameTableSorted);
     return 0;
 }
 

From 8826f3b433f65294414697df2cef23511211f39e Mon Sep 17 00:00:00 2001
From: Hasnain Lakhani 
Date: Tue, 8 Oct 2019 14:13:45 -0700
Subject: [PATCH 142/144] Add OSS-Fuzz fuzzing badge

This is basically redoing https://github.com/facebook/zstd/pull/1818 by @inferno-chromium from the OSS-Fuzz team
---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index 290341cc..9c5f9201 100644
--- a/README.md
+++ b/README.md
@@ -15,6 +15,7 @@ a list of known ports and bindings is provided on [Zstandard homepage](http://ww
 [![Build status][AppveyorDevBadge]][AppveyorLink]
 [![Build status][CircleDevBadge]][CircleLink]
 [![Build status][CirrusDevBadge]][CirrusLink]
+[![Fuzzing Status][OSSFuzzBadge]][OSSFuzzLink]
 
 [travisDevBadge]: https://travis-ci.org/facebook/zstd.svg?branch=dev "Continuous Integration test suite"
 [travisLink]: https://travis-ci.org/facebook/zstd
@@ -24,6 +25,8 @@ a list of known ports and bindings is provided on [Zstandard homepage](http://ww
 [CircleLink]: https://circleci.com/gh/facebook/zstd
 [CirrusDevBadge]: https://api.cirrus-ci.com/github/facebook/zstd.svg?branch=dev
 [CirrusLink]: https://cirrus-ci.com/github/facebook/zstd
+[OSSFuzzBadge]: https://oss-fuzz-build-logs.storage.googleapis.com/badges/zstd.svg
+[OSSFuzzLink]: https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:zstd
 
 ## Benchmarks
 

From c5ebb370510b5a2c7fbeac332a2480f51e0c9070 Mon Sep 17 00:00:00 2001
From: Sen Huang 
Date: Wed, 9 Oct 2019 09:39:52 -0400
Subject: [PATCH 143/144] Only check for filename collisions when using
 --output-dir-flat

---
 programs/fileio.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 81633947..39cc70b9 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1553,7 +1553,8 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN
             const char* const dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix);  /* cannot fail */
             error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
         }
-        FIO_checkFilenameCollisions(inFileNamesTable ,nbFiles);
+        if (outDirName)
+            FIO_checkFilenameCollisions(inFileNamesTable ,nbFiles);
     }
 
     FIO_freeCResources(ress);
@@ -2375,7 +2376,8 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
 
             error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName);
         }
-        FIO_checkFilenameCollisions(srcNamesTable ,nbFiles);
+        if (outDirName)
+            FIO_checkFilenameCollisions(srcNamesTable ,nbFiles);
     }
 
     FIO_freeDResources(ress);

From 46ee10dfb5944e4a277e402da6ec2e1168ece697 Mon Sep 17 00:00:00 2001
From: Eric van Gyzen 
Date: Thu, 10 Oct 2019 09:21:29 -0500
Subject: [PATCH 144/144] Include errno in all fwrite error messages in
 fileio.c

---
 programs/fileio.c | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 8a45563d..09687c67 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -769,7 +769,7 @@ FIO_compressGzFrame(cRess_t* ress,
         {   size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
             if (decompBytes) {
                 if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes)
-                    EXM_THROW(73, "Write error : cannot write to output file");
+                    EXM_THROW(73, "Write error : cannot write to output file : %s", strerror(errno));
                 outFileSize += decompBytes;
                 strm.next_out = (Bytef*)ress->dstBuffer;
                 strm.avail_out = (uInt)ress->dstBufferSize;
@@ -1523,7 +1523,7 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
     if (!prefs->sparseFileSupport) {  /* normal write */
         size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
         if (sizeCheck != bufferSize)
-            EXM_THROW(70, "Write error : %s (cannot write decoded block)",
+            EXM_THROW(70, "Write error : cannot write decoded block : %s",
                             strerror(errno));
         return 0;
     }
@@ -1554,7 +1554,8 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
             ptrT += nb0T;
             {   size_t const sizeCheck = fwrite(ptrT, sizeof(size_t), seg0SizeT, file);
                 if (sizeCheck != seg0SizeT)
-                    EXM_THROW(73, "Write error : cannot write decoded block");
+                    EXM_THROW(73, "Write error : cannot write decoded block : %s",
+                            strerror(errno));
         }   }
         ptrT += seg0SizeT;
     }
@@ -1575,7 +1576,8 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
                 storedSkips = 0;
                 {   size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file);
                     if (sizeCheck != (size_t)(restEnd - restPtr))
-                        EXM_THROW(75, "Write error : cannot write decoded end of block");
+                        EXM_THROW(75, "Write error : cannot write decoded end of block : %s",
+                            strerror(errno));
     }   }   }   }
 
     return storedSkips;
@@ -1593,7 +1595,7 @@ FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
          * so that skipped ones get implicitly translated as zero by FS */
         {   const char lastZeroByte[1] = { 0 };
             if (fwrite(lastZeroByte, 1, 1, file) != 1)
-                EXM_THROW(69, "Write error : cannot write last zero");
+                EXM_THROW(69, "Write error : cannot write last zero : %s", strerror(errno));
     }   }
 }
 
@@ -1612,7 +1614,7 @@ static int FIO_passThrough(FIO_prefs_t* const prefs,
     /* assumption : ress->srcBufferLoaded bytes already loaded and stored within buffer */
     {   size_t const sizeCheck = fwrite(buffer, 1, alreadyLoaded, foutput);
         if (sizeCheck != alreadyLoaded) {
-            DISPLAYLEVEL(1, "Pass-through write error \n");
+            DISPLAYLEVEL(1, "Pass-through write error : %s\n", strerror(errno));
             return 1;
     }   }
 
@@ -1783,7 +1785,7 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress,
         {   size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
             if (decompBytes) {
                 if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) {
-                    DISPLAYLEVEL(1, "zstd: %s \n", strerror(errno));
+                    DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno));
                     decodingError = 1; break;
                 }
                 outFileSize += decompBytes;
@@ -1858,7 +1860,7 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile,
         {   size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
             if (decompBytes) {
                 if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) {
-                    DISPLAYLEVEL(1, "zstd: %s \n", strerror(errno));
+                    DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno));
                     decodingError = 1; break;
                 }
                 outFileSize += decompBytes;
@@ -1929,7 +1931,7 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
             /* Write Block */
             if (decodedBytes) {
                 if (fwrite(ress->dstBuffer, 1, decodedBytes, ress->dstFile) != decodedBytes) {
-                    DISPLAYLEVEL(1, "zstd: %s \n", strerror(errno));
+                    DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno));
                     decodingError = 1; nextToLoad = 0; break;
                 }
                 filesize += decodedBytes;