From 343a75d2ef095e6d0c823b02ef1ed7ce8551f8b4 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 1 Dec 2020 22:33:45 -0800 Subject: [PATCH 1/7] simplified test makefile removed gzstd target: relevant tests are unused and broken anyway --- tests/Makefile | 42 +++++------------------------------------- 1 file changed, 5 insertions(+), 37 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 7e7d5534..c073543c 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -94,23 +94,20 @@ allnothread: fullbench fuzzer paramgrill datagen decodecorpus # note : broken : requires symbols unavailable from dynamic library dll: fuzzer-dll zstreamtest-dll -PHONY: zstd zstd32 zstd-nolegacy # phony: only external makefile knows how to build or update them +.PHONY: zstd zstd32 zstd-nolegacy # only external makefile knows how to build or update them zstd zstd32 zstd-nolegacy: - $(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)" DEBUGLEVEL=$(DEBUGLEVEL) - -gzstd: - $(MAKE) -C $(PRGDIR) $@ HAVE_ZLIB=1 MOREFLAGS+="$(DEBUGFLAGS)" DEBUGLEVEL=$(DEBUGLEVEL) + $(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)" .PHONY: libzstd libzstd : - $(MAKE) -C $(ZSTDDIR) libzstd DEBUGLEVEL=$(DEBUGLEVEL) + $(MAKE) -C $(ZSTDDIR) libzstd %-dll : libzstd -%-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd +%-dll : LDFLAGS += -L$(ZSTDDIR) -lzstd .PHONY: zstd-staticLib zstd-staticLib : - $(MAKE) -C $(ZSTDDIR) libzstd.a DEBUGLEVEL=$(DEBUGLEVEL) + $(MAKE) -C $(ZSTDDIR) libzstd.a zstdm_%.o : $(ZSTDDIR)/common/%.c $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ @@ -327,35 +324,6 @@ test-zstd test-zstd32 test-zstd-nolegacy: datagen file $(ZSTD) EXE_PREFIX="$(QEMU_SYS)" ZSTD_BIN="$(ZSTD)" DATAGEN_BIN=./datagen ./playTests.sh $(ZSTDRTTEST) - -test-gzstd: gzstd - $(PRGDIR)/zstd -f README.md test-zstd-speed.py - gzip -f README.md test-zstd-speed.py - cat README.md.zst test-zstd-speed.py.gz >zstd_gz.zst - cat README.md.gz test-zstd-speed.py.zst >gz_zstd.gz - $(PRGDIR)/zstd -df README.md.gz -o README2.md - $(PRGDIR)/zstd -df README.md.gz test-zstd-speed.py.gz - $(PRGDIR)/zstd -df zstd_gz.zst gz_zstd.gz - $(DIFF) -q zstd_gz gz_zstd - echo Hello World ZSTD | $(PRGDIR)/zstd -c - >hello.zst - echo Hello World GZIP | gzip -c - >hello.gz - echo Hello World TEXT >hello.txt - cat hello.zst hello.gz hello.txt >hello_zst_gz_txt.gz - $(PRGDIR)/zstd -dcf hello.* - $(PRGDIR)/zstd -dcf - test.txt && $(PRGDIR)/zstd test.txt - env ZCAT=/tmp/zstdcat $(PRGDIR)/zstdgrep hello test.txt.zst - env ZCAT=/tmp/zstdcat $(PRGDIR)/zstdgrep weird test.txt.zst && return 1 || return 0 - -echo 'hello' > pattern.txt - env ZCAT=/tmp/zstdcat $(PRGDIR)/zstdgrep -f pattern.txt test.txt.zst - $(RM) test.txt test.txt.zst pattern.txt - test-fullbench: fullbench datagen $(QEMU_SYS) ./fullbench -i1 $(QEMU_SYS) ./fullbench -i1 -P0 From 37165f66b7f5ee6d8e3426aa06b247cf2152da44 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 1 Dec 2020 23:36:05 -0800 Subject: [PATCH 2/7] better usage of default build rules --- tests/Makefile | 68 +++++++++++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index c073543c..0a362882 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -25,6 +25,7 @@ PYTHON ?= python3 TESTARTEFACT := versionsTest DEBUGLEVEL ?= 1 +export DEBUGLEVEL # transmit value to sub-makefiles DEBUGFLAGS = -g -DDEBUGLEVEL=$(DEBUGLEVEL) CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) @@ -77,21 +78,23 @@ FUZZERTEST ?= -T200s ZSTDRTTEST = --test-large-data DECODECORPUS_TESTTIME ?= -T30 -.PHONY: default all all32 allnothread dll clean test test32 test-all versionsTest - +.PHONY: default default: fullbench - @echo $(ZSTDMT_OBJECTS) +.PHONY: all all: fullbench fuzzer zstreamtest paramgrill datagen decodecorpus roundTripCrash \ fullbench-lib poolTests +.PHONY: all32 all32: fullbench32 fuzzer32 zstreamtest32 +.PHONY: allnothread allnothread: MULTITHREAD_CPP= allnothread: MULTITHREAD_LD= allnothread: fullbench fuzzer paramgrill datagen decodecorpus # note : broken : requires symbols unavailable from dynamic library +.PHONY: dll dll: fuzzer-dll zstreamtest-dll .PHONY: zstd zstd32 zstd-nolegacy # only external makefile knows how to build or update them @@ -100,13 +103,13 @@ zstd zstd32 zstd-nolegacy: .PHONY: libzstd libzstd : - $(MAKE) -C $(ZSTDDIR) libzstd + $(MAKE) -C $(ZSTDDIR) libzstd MOREFLAGS+="$(DEBUGFLAGS)" %-dll : libzstd %-dll : LDFLAGS += -L$(ZSTDDIR) -lzstd -.PHONY: zstd-staticLib -zstd-staticLib : +.PHONY: $(ZSTDDIR)/libzstd.a +$(ZSTDDIR)/libzstd.a : $(MAKE) -C $(ZSTDDIR) libzstd.a zstdm_%.o : $(ZSTDDIR)/common/%.c @@ -135,12 +138,11 @@ fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD) fullbench fullbench32 : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements fullbench fullbench32 : $(ZSTD_FILES) fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c fullbench.c - $(CC) $(FLAGS) $^ -o $@$(EXT) + $(LINK.c) $^ -o $@$(EXT) fullbench-lib : CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -fullbench-lib : zstd-staticLib -fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c fullbench.c - $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) $(ZSTDDIR)/libzstd.a +fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(ZSTDDIR)/libzstd.a fullbench.c + $(LINK.c) $^ -o $@$(EXT) # note : broken : requires symbols unavailable from dynamic library fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR)/timefn.c fullbench.c @@ -153,7 +155,7 @@ fuzzer32: CFLAGS += -m32 fuzzer : $(ZSTDMT_OBJECTS) fuzzer32: $(ZSTD_FILES) fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c - $(CC) $(FLAGS) $^ -o $@$(EXT) + $(LINK.c) $^ -o $@$(EXT) # note : broken : requires symbols unavailable from dynamic library fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c @@ -177,66 +179,65 @@ zstreamtest zstreamtest32 : LDFLAGS += $(MULTITHREAD_LD) zstreamtest : $(ZSTDMT_OBJECTS) $(ZSTREAM_PROPER_FILES) zstreamtest32 : $(ZSTREAMFILES) zstreamtest zstreamtest32 : - $(CC) $(FLAGS) $^ -o $@$(EXT) + $(LINK.c) $^ -o $@$(EXT) zstreamtest_asan : CFLAGS += -fsanitize=address zstreamtest_asan : $(ZSTREAMFILES) - $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) + $(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT) zstreamtest_tsan : CFLAGS += -fsanitize=thread zstreamtest_tsan : $(ZSTREAMFILES) - $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) + $(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT) # note : broken : requires symbols unavailable from dynamic library zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c # xxh symbols not exposed from dll zstreamtest-dll : $(ZSTREAM_LOCAL_FILES) $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) -paramgrill : DEBUGFLAGS = # turn off assert() by default for speed measurements +paramgrill : DEBUGFLAGS = # turn off debug for speed measurements +paramgrill : LDLIBS += -lm paramgrill : $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(PRGDIR)/benchzstd.c $(PRGDIR)/datagen.c paramgrill.c - $(CC) $(FLAGS) $^ -lm -o $@$(EXT) datagen : $(PRGDIR)/datagen.c datagencli.c - $(CC) $(FLAGS) $^ -o $@$(EXT) + $(LINK.c) $^ -o $@$(EXT) +roundTripCrash: CFLAGS += $(MULTITHREAD) roundTripCrash : $(ZSTD_OBJECTS) roundTripCrash.c - $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) -longmatch : $(ZSTD_OBJECTS) longmatch.c - $(CC) $(FLAGS) $^ -o $@$(EXT) +longmatch : $(ZSTD_OBJECTS) longmatch.c +bigdict: CFLAGS += $(MULTITHREAD) bigdict: $(ZSTDMT_OBJECTS) $(PRGDIR)/datagen.c bigdict.c - $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) invalidDictionaries : $(ZSTD_OBJECTS) invalidDictionaries.c - $(CC) $(FLAGS) $^ -o $@$(EXT) legacy : CPPFLAGS += -I$(ZSTDDIR)/legacy -DZSTD_LEGACY_SUPPORT=4 legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c - $(CC) $(FLAGS) $^ -o $@$(EXT) +decodecorpus : LDLIBS += -lm decodecorpus : $(filter-out zstdc_zstd_compress.o, $(ZSTD_OBJECTS)) $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c decodecorpus.c - $(CC) $(FLAGS) $^ -o $@$(EXT) -lm poolTests : $(PRGDIR)/util.c $(PRGDIR)/timefn.c poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c - $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) + $(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT) .PHONY: versionsTest versionsTest: clean $(PYTHON) test-zstd-versions.py +.PHONY: automated_benchmarking automated_benchmarking: clean $(PYTHON) automated_benchmarking.py -checkTag: checkTag.c $(ZSTDDIR)/zstd.h - $(CC) $(FLAGS) $< -o $@$(EXT) +# make checkTag +checkTag.o : $(ZSTDDIR)/zstd.h +.PHONY: clean clean: $(MAKE) -C $(ZSTDDIR) clean $(MAKE) -C $(PRGDIR) clean - @$(RM) -fR $(TESTARTEFACT) - @$(RM) -rf tmp* # some test directories are named tmp* - @$(RM) core *.o *.tmp result* *.gcda dictionary *.zst \ + $(RM) -fR $(TESTARTEFACT) + $(RM) -rf tmp* # some test directories are named tmp* + $(RM) core *.o *.tmp result* *.gcda dictionary *.zst \ $(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \ fullbench-lib$(EXT) fullbench-dll$(EXT) \ @@ -293,9 +294,12 @@ list: @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs .PHONY: shortest -shortest: ZSTDRTTEST= +shortest: ZSTDRTTEST= # remove long tests shortest: test-zstd +.PHONY: check +check: shortest + .PHONY: fuzztest fuzztest: test-fuzzer test-zstream test-decodecorpus @@ -305,8 +309,10 @@ ifeq ($(QEMU_SYS),) test: test-pool endif +.PHONY: test32 test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32 +.PHONY: test-all test-all: test test32 valgrindTest test-decodecorpus-cli From f8d0b46a9f095c1a46c8a1194b8259bf4c687486 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 1 Dec 2020 23:44:16 -0800 Subject: [PATCH 3/7] streamline fuzzer from fuzzer32 --- tests/Makefile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 0a362882..69db5987 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -149,12 +149,12 @@ fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR # $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -fuzzer : CPPFLAGS += $(MULTITHREAD_CPP) fuzzer : LDFLAGS += $(MULTITHREAD_LD) -fuzzer32: CFLAGS += -m32 fuzzer : $(ZSTDMT_OBJECTS) -fuzzer32: $(ZSTD_FILES) fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c + +fuzzer32: CFLAGS += -m32 +fuzzer32: $(ZSTD_FILES) $(LINK.c) $^ -o $@$(EXT) # note : broken : requires symbols unavailable from dynamic library @@ -315,7 +315,6 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32 .PHONY: test-all test-all: test test32 valgrindTest test-decodecorpus-cli - .PHONY: test-zstd test-zstd32 test-zstd-nolegacy test-zstdgrep test-zstd: ZSTD = $(PRGDIR)/zstd test-zstd: zstd From 9f8b180d5daebe54b33e0f6e02a5186dd578f838 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 2 Dec 2020 00:15:07 -0800 Subject: [PATCH 4/7] fixed API documentation --- doc/zstd_manual.html | 308 ++++++++++++++++++++++++++++++------------- lib/zstd.h | 66 +++++----- tests/Makefile | 8 +- 3 files changed, 258 insertions(+), 124 deletions(-) diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index 77d4b8e6..d26268b5 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -1,10 +1,10 @@ -zstd 1.4.5 Manual +zstd 1.4.6 Manual -

zstd 1.4.5 Manual

+

zstd 1.4.6 Manual

Contents

    @@ -27,16 +27,10 @@
  1. Advanced compression functions
  2. Advanced decompression functions
  3. Advanced streaming functions
    4. -
  4. ! ZSTD_initCStream_usingDict() :
    5. -
  5. ! ZSTD_initCStream_advanced() :
    6. -
  6. ! ZSTD_initCStream_usingCDict() :
    7. -
  7. ! ZSTD_initCStream_usingCDict_advanced() :
    8. -
  8. This function is deprecated, and is equivalent to:
    9. -
  9. This function is deprecated, and is equivalent to:
    10. -
  10. Buffer-less and synchronous inner streaming functions
    11. -
  11. Buffer-less streaming compression (synchronous mode)
    12. -
  12. Buffer-less streaming decompression (synchronous mode)
    13. -
  13. Block level API
    14. +
  14. Buffer-less and synchronous inner streaming functions
    15. +
  15. Buffer-less streaming compression (synchronous mode)
    16. +
  16. Buffer-less streaming decompression (synchronous mode)
    17. +
  17. Block level API

Introduction

@@ -72,8 +66,14 @@
 
 
Version

 
-
unsigned ZSTD_versionNumber(void);   /**< to check runtime library version */
-


+
unsigned ZSTD_versionNumber(void);
+
  Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). 
+


+
+
const char* ZSTD_versionString(void);
+
  Return runtime library version, like "1.4.5". Requires v1.3.0+. 
+


+
 
Simple API

 
 
size_t ZSTD_compress( void* dst, size_t dstCapacity,
@@ -277,7 +277,9 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
                                      * for large inputs, by finding large matches at long distance.
                                      * It increases memory usage and window size.
                                      * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB
-                                     * except when expressly set to a different value. */
+                                     * except when expressly set to a different value.
+                                     * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and
+                                     * compression strategy >= ZSTD_btopt (== compression level 16+) */
     ZSTD_c_ldmHashLog=161,   /* Size of the table for long distance matching, as a power of 2.
                               * Larger values increase memory usage and compression ratio,
                               * but decrease compression speed.
@@ -308,16 +310,20 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
     ZSTD_c_dictIDFlag=202,   /* When applicable, dictionary's ID is written into frame header (default:1) */
 
     /* multi-threading parameters */
-    /* These parameters are only useful if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
-     * They return an error otherwise. */
+    /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
+     * Otherwise, trying to set any other value than default (0) will be a no-op and return an error.
+     * In a situation where it's unknown if the linked library supports multi-threading or not,
+     * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property.
+     */
     ZSTD_c_nbWorkers=400,    /* Select how many threads will be spawned to compress in parallel.
-                              * When nbWorkers >= 1, triggers asynchronous mode when used with ZSTD_compressStream*() :
+                              * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() :
                               * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
-                              * while compression work is performed in parallel, within worker threads.
+                              * while compression is performed in parallel, within worker thread(s).
                               * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
                               *  in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
                               * More workers improve speed, but also increase memory usage.
-                              * Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */
+                              * Default value is `0`, aka "single-threaded mode" : no worker is spawned,
+                              * compression is performed inside Caller's thread, and all invocations are blocking */
     ZSTD_c_jobSize=401,      /* Size of a compression job. This value is enforced only when nbWorkers >= 1.
                               * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
                               * 0 means default, which is dynamically determined based on compression parameters.
@@ -346,6 +352,11 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
      * ZSTD_c_literalCompressionMode
      * ZSTD_c_targetCBlockSize
      * ZSTD_c_srcSizeHint
+     * ZSTD_c_enableDedicatedDictSearch
+     * ZSTD_c_stableInBuffer
+     * ZSTD_c_stableOutBuffer
+     * ZSTD_c_blockDelimiters
+     * ZSTD_c_validateSequences
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly;
      *        also, the enums values themselves are unstable and can still change.
@@ -356,7 +367,12 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
      ZSTD_c_experimentalParam4=1001,
      ZSTD_c_experimentalParam5=1002,
      ZSTD_c_experimentalParam6=1003,
-     ZSTD_c_experimentalParam7=1004
+     ZSTD_c_experimentalParam7=1004,
+     ZSTD_c_experimentalParam8=1005,
+     ZSTD_c_experimentalParam9=1006,
+     ZSTD_c_experimentalParam10=1007,
+     ZSTD_c_experimentalParam11=1008,
+     ZSTD_c_experimentalParam12=1009
 } ZSTD_cParameter;
 


 
typedef struct {
@@ -456,11 +472,13 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
      * At the time of this writing, they include :
      * ZSTD_d_format
      * ZSTD_d_stableOutBuffer
+     * ZSTD_d_forceIgnoreChecksum
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly
      */
      ZSTD_d_experimentalParam1=1000,
-     ZSTD_d_experimentalParam2=1001
+     ZSTD_d_experimentalParam2=1001,
+     ZSTD_d_experimentalParam3=1002
 
 } ZSTD_dParameter;
 


@@ -591,8 +609,9 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
   - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
   - output->pos must be <= dstCapacity, input->pos must be <= srcSize
   - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
+  - endOp must be a valid directive
   - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
-  - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available,
+  - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available,
                                                   and then immediately returns, just indicating that there is some data remaining to be flushed.
                                                   The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
   - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
@@ -895,21 +914,40 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 

 
 
typedef struct {
-    unsigned int matchPos; /* Match pos in dst */
-    /* If seqDef.offset > 3, then this is seqDef.offset - 3
-     * If seqDef.offset < 3, then this is the corresponding repeat offset
-     * But if seqDef.offset < 3 and litLength == 0, this is the
-     *   repeat offset before the corresponding repeat offset
-     * And if seqDef.offset == 3 and litLength == 0, this is the
-     *   most recent repeat offset - 1
-     */
-    unsigned int offset;
-    unsigned int litLength; /* Literal length */
-    unsigned int matchLength; /* Match length */
-    /* 0 when seq not rep and seqDef.offset otherwise
-     * when litLength == 0 this will be <= 4, otherwise <= 3 like normal
-     */
-    unsigned int rep;
+    unsigned int offset;      /* The offset of the match. (NOT the same as the offset code)
+                               * If offset == 0 and matchLength == 0, this sequence represents the last
+                               * literals in the block of litLength size.
+                               */
+
+    unsigned int litLength;   /* Literal length of the sequence. */
+    unsigned int matchLength; /* Match length of the sequence. */
+
+                              /* Note: Users of this API may provide a sequence with matchLength == litLength == offset == 0.
+                               * In this case, we will treat the sequence as a marker for a block boundary.
+                               */
+
+    unsigned int rep;         /* Represents which repeat offset is represented by the field 'offset'.
+                               * Ranges from [0, 3].
+                               *
+                               * Repeat offsets are essentially previous offsets from previous sequences sorted in
+                               * recency order. For more detail, see doc/zstd_compression_format.md
+                               *
+                               * If rep == 0, then 'offset' does not contain a repeat offset.
+                               * If rep > 0:
+                               *  If litLength != 0:
+                               *      rep == 1 --> offset == repeat_offset_1
+                               *      rep == 2 --> offset == repeat_offset_2
+                               *      rep == 3 --> offset == repeat_offset_3
+                               *  If litLength == 0:
+                               *      rep == 1 --> offset == repeat_offset_2
+                               *      rep == 2 --> offset == repeat_offset_3
+                               *      rep == 3 --> offset == repeat_offset_1 - 1
+                               *
+                               * Note: This field is optional. ZSTD_generateSequences() will calculate the value of
+                               * 'rep', but repeat offsets do not necessarily need to be calculated from an external
+                               * sequence provider's perspective. For example, ZSTD_compressSequences() does not
+                               * use this 'rep' field at all (as of now).
+                               */
 } ZSTD_Sequence;
 


 
typedef struct {
@@ -951,6 +989,12 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
                                  * Decoder cannot recognise automatically this format, requiring this instruction. */
 } ZSTD_format_e;
 


+
typedef enum {
+    /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */
+    ZSTD_d_validateChecksum = 0,
+    ZSTD_d_ignoreChecksum = 1
+} ZSTD_forceIgnoreChecksum_e;
+


 
typedef enum {
     /* Note: this enum and the behavior it controls are effectively internal
      * implementation details of the compressor. They are expected to continue
@@ -1045,12 +1089,69 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
            or an error code (if srcSize is too small) 
 


 
-
size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
-    size_t outSeqsSize, const void* src, size_t srcSize);
-
 Extract sequences from the sequence store
+
typedef enum {
+  ZSTD_sf_noBlockDelimiters = 0,         /* Representation of ZSTD_Sequence has no block delimiters, sequences only */
+  ZSTD_sf_explicitBlockDelimiters = 1    /* Representation of ZSTD_Sequence contains explicit block delimiters */
+} ZSTD_sequenceFormat_e;
+


+
 Generate sequences using ZSTD_compress2, given a source buffer.
+
+ Each block will end with a dummy sequence
+ with offset == 0, matchLength == 0, and litLength == length of last literals.
+ litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0)
+ simply acts as a block delimiter.
+
  zc can be used to insert custom compression params.
  This function invokes ZSTD_compress2
- @return : number of sequences extracted
+
+ The output of this function can be fed into ZSTD_compressSequences() with CCtx
+ setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters
+ @return : number of sequences generated
+ 
+


+
+
size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize);
+
 Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals
+ by merging them into into the literals of the next sequence.
+
+ As such, the final generated result has no explicit representation of block boundaries,
+ and the final last literals segment is not represented in the sequences.
+
+ The output of this function can be fed into ZSTD_compressSequences() with CCtx
+ setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters
+ @return : number of sequences left after merging
+ 
+


+
+
size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize,
+                      const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
+                      const void* src, size_t srcSize);
+
 Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst.
+ If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
+ The entire source is compressed into a single frame.
+
+ The compression behavior changes based on cctx params. In particular:
+    If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain
+    no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on
+    the block size derived from the cctx, and sequences may be split. This is the default setting.
+
+    If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain
+    block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided.
+
+    If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined
+    behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for
+    specifics regarding offset/matchlength requirements) then the function will bail out and return an error.
+
+    In addition to the two adjustable experimental params, there are other important cctx params.
+    - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN.
+    - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression.
+    - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset
+      is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md
+
+ Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused.
+ Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly,
+         and cannot emit an RLE block that disagrees with the repcode history
+ @return : final compressed size or a ZSTD error.
  
 


 
@@ -1141,7 +1242,11 @@ ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize);
 
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
 typedef void  (*ZSTD_freeFunction) (void* opaque, void* address);
 typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
-static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  /**< this constant defers to stdlib's functions */
+static
+#ifdef __GNUC__
+__attribute__((__unused__))
+#endif
+ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  /**< this constant defers to stdlib's functions */
 
  These prototypes make it possible to pass your own allocation/free functions.
   ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below.
   All allocation/free operations will be completed using these custom variants instead of regular  ones.
@@ -1270,8 +1375,10 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
 
size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
 
  Similar to ZSTD_CCtx_setParameter.
   Set one compression parameter, selected by enum ZSTD_cParameter.
-  Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().
- @result : 0, or an error code (which can be tested with ZSTD_isError()).
+  Parameters must be applied to a ZSTD_CCtx using
+  ZSTD_CCtx_setParametersUsingCCtxParams().
+ @result : a code representing success or failure (which can be tested with
+           ZSTD_isError()).
  
 


 
@@ -1348,6 +1455,13 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
  
 


 
+
size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value);
+
  Get the requested decompression parameter value, selected by enum ZSTD_dParameter,
+  and store it into int* value.
+ @return : 0, or an error code (which can be tested with ZSTD_isError()).
+ 
+


+
 
size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
 
  Instruct the decoder context about what kind of data to decode next.
   This instruction is mandatory to decode data without a fully-formed header,
@@ -1371,24 +1485,29 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
   redundant functions will be deprecated, and then at some point removed.
 

 
-
Advanced Streaming compression functions
/**! ZSTD_initCStream_srcSize() :
- * This function is deprecated, and equivalent to:
- *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
- *     ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
- *     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
- *     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
- *
- * pledgedSrcSize must be correct. If it is not known at init time, use
- * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
- * "0" also disables frame content size field. It may be enabled in the future.
- * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
- */
-size_t
+
Advanced Streaming compression functions


+
size_t
 ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
-                         int compressionLevel,
-                         unsigned long long pledgedSrcSize);
-


-
! ZSTD_initCStream_usingDict() :
 This function is deprecated, and is equivalent to:
+             int compressionLevel,
+             unsigned long long pledgedSrcSize);
+
 This function is deprecated, and equivalent to:
+     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+     ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
+     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+
+ pledgedSrcSize must be correct. If it is not known at init time, use
+ ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
+ "0" also disables frame content size field. It may be enabled in the future.
+ Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
+ 
+


+
+
size_t
+ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
+         const void* dict, size_t dictSize,
+               int compressionLevel);
+
 This function is deprecated, and is equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
      ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
      ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
@@ -1399,9 +1518,14 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
  it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
  Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  
-

+


 
-
! ZSTD_initCStream_advanced() :
 This function is deprecated, and is approximately equivalent to:
+
size_t
+ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
+        const void* dict, size_t dictSize,
+              ZSTD_parameters params,
+              unsigned long long pledgedSrcSize);
+
 This function is deprecated, and is approximately equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
      // Pseudocode: Set each zstd parameter and leave the rest as-is.
      for ((param, value) : params) {
@@ -1415,18 +1539,24 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
  If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
  Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  
-

+


 
-
! ZSTD_initCStream_usingCDict() :
 This function is deprecated, and equivalent to:
+
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
+
 This function is deprecated, and equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
      ZSTD_CCtx_refCDict(zcs, cdict);
 
  note : cdict will just be referenced, and must outlive compression session
  Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  
-

+


 
-
! ZSTD_initCStream_usingCDict_advanced() :
   This function is DEPRECATED, and is approximately equivalent to:
+
size_t
+ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
+                   const ZSTD_CDict* cdict,
+                         ZSTD_frameParameters fParams,
+                         unsigned long long pledgedSrcSize);
+
   This function is DEPRECATED, and is approximately equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
      // Pseudocode: Set each zstd frame parameter and leave the rest as-is.
      for ((fParam, value) : fParams) {
@@ -1440,7 +1570,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
  value ZSTD_CONTENTSIZE_UNKNOWN.
  Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  
-

+


 
 
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
 
 This function is deprecated, and is equivalent to:
@@ -1483,42 +1613,44 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
  
 


 
-
Advanced Streaming decompression functions
/**
- * This function is deprecated, and is equivalent to:
- *
- *     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
- *     ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
- *
- * note: no dictionary will be used if dict == NULL or dictSize < 8
- * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
- */
-size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
-


-
This function is deprecated, and is equivalent to:
+
Advanced Streaming decompression functions


+
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
+

+     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+     ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
+
+ note: no dictionary will be used if dict == NULL or dictSize < 8
+ Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
+ 
+


+
+
size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
+

      ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
      ZSTD_DCtx_refDDict(zds, ddict);
 
  note : ddict is referenced, it must outlive decompression session
  Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  
-

+


 
-
This function is deprecated, and is equivalent to:
+
size_t ZSTD_resetDStream(ZSTD_DStream* zds);
+

      ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
 
  re-use decompression parameters from previous init; saves dictionary loading
  Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  
-

+


 
-
Buffer-less and synchronous inner streaming functions
+
Buffer-less and synchronous inner streaming functions
   This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
   But it's also a complex one, with several restrictions, documented below.
   Prefer normal streaming API for an easier experience.
  
 

 
-
Buffer-less streaming compression (synchronous mode)
+
Buffer-less streaming compression (synchronous mode)
   A ZSTD_CCtx object is required to track streaming operations.
   Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
   ZSTD_CCtx object can be re-used multiple times within successive compression operations.
@@ -1554,7 +1686,7 @@ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
 size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);   /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
 size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
 


-
Buffer-less streaming decompression (synchronous mode)
+
Buffer-less streaming decompression (synchronous mode)
   A ZSTD_DCtx object is required to track streaming operations.
   Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
   A ZSTD_DCtx object can be re-used multiple times.
@@ -1650,7 +1782,7 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
 
 
typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
 


-
Block level API

+
Block level API

 
 
    Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
     But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
diff --git a/lib/zstd.h b/lib/zstd.h
index b99ce975..1a235393 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -339,7 +339,7 @@ typedef enum {
                                      * for large inputs, by finding large matches at long distance.
                                      * It increases memory usage and window size.
                                      * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB
-                                     * except when expressly set to a different value. 
+                                     * except when expressly set to a different value.
                                      * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and
                                      * compression strategy >= ZSTD_btopt (== compression level 16+) */
     ZSTD_c_ldmHashLog=161,   /* Size of the table for long distance matching, as a power of 2.
@@ -1135,13 +1135,13 @@ typedef struct {
                               /* Note: Users of this API may provide a sequence with matchLength == litLength == offset == 0.
                                * In this case, we will treat the sequence as a marker for a block boundary.
                                */
-    
+
     unsigned int rep;         /* Represents which repeat offset is represented by the field 'offset'.
                                * Ranges from [0, 3].
-                               * 
+                               *
                                * Repeat offsets are essentially previous offsets from previous sequences sorted in
                                * recency order. For more detail, see doc/zstd_compression_format.md
-                               * 
+                               *
                                * If rep == 0, then 'offset' does not contain a repeat offset.
                                * If rep > 0:
                                *  If litLength != 0:
@@ -1152,7 +1152,7 @@ typedef struct {
                                *      rep == 1 --> offset == repeat_offset_2
                                *      rep == 2 --> offset == repeat_offset_3
                                *      rep == 3 --> offset == repeat_offset_1 - 1
-                               * 
+                               *
                                * Note: This field is optional. ZSTD_generateSequences() will calculate the value of
                                * 'rep', but repeat offsets do not necessarily need to be calculated from an external
                                * sequence provider's perspective. For example, ZSTD_compressSequences() does not
@@ -1309,15 +1309,15 @@ typedef enum {
 
 /*! ZSTD_generateSequences() :
  * Generate sequences using ZSTD_compress2, given a source buffer.
- * 
+ *
  * Each block will end with a dummy sequence
  * with offset == 0, matchLength == 0, and litLength == length of last literals.
  * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0)
  * simply acts as a block delimiter.
- * 
+ *
  * zc can be used to insert custom compression params.
  * This function invokes ZSTD_compress2
- * 
+ *
  * The output of this function can be fed into ZSTD_compressSequences() with CCtx
  * setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters
  * @return : number of sequences generated
@@ -1329,10 +1329,10 @@ ZSTDLIB_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
 /*! ZSTD_mergeBlockDelimiters() :
  * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals
  * by merging them into into the literals of the next sequence.
- * 
+ *
  * As such, the final generated result has no explicit representation of block boundaries,
  * and the final last literals segment is not represented in the sequences.
- * 
+ *
  * The output of this function can be fed into ZSTD_compressSequences() with CCtx
  * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters
  * @return : number of sequences left after merging
@@ -1342,26 +1342,26 @@ ZSTDLIB_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t se
 /*! ZSTD_compressSequences() :
  * Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst.
  * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
- * The entire source is compressed into a single frame. 
- * 
+ * The entire source is compressed into a single frame.
+ *
  * The compression behavior changes based on cctx params. In particular:
  *    If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain
  *    no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on
  *    the block size derived from the cctx, and sequences may be split. This is the default setting.
- * 
+ *
  *    If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain
  *    block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided.
- * 
+ *
  *    If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined
  *    behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for
  *    specifics regarding offset/matchlength requirements) then the function will bail out and return an error.
- * 
+ *
  *    In addition to the two adjustable experimental params, there are other important cctx params.
  *    - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN.
  *    - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression.
  *    - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset
  *      is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md
- * 
+ *
  * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused.
  * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly,
  *         and cannot emit an RLE block that disagrees with the repcode history
@@ -1513,7 +1513,7 @@ ZSTDLIB_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads);
 ZSTDLIB_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool);
 ZSTDLIB_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool);
 
-/**
+/*
  * This API is temporary and is expected to change or disappear in the future!
  */
 ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2(
@@ -1771,9 +1771,9 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
 
 /* ZSTD_c_blockDelimiters
  * Default is 0 == ZSTD_sf_noBlockDelimiters.
- * 
+ *
  * For use with sequence compression API: ZSTD_compressSequences().
- * 
+ *
  * Designates whether or not the given array of ZSTD_Sequence contains block delimiters
  * and last literals, which are defined as sequences with offset == 0 and matchLength == 0.
  * See the definition of ZSTD_Sequence for more specifics.
@@ -1782,18 +1782,18 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
 
 /* ZSTD_c_validateSequences
  * Default is 0 == disabled. Set to 1 to enable sequence validation.
- * 
+ *
  * For use with sequence compression API: ZSTD_compressSequences().
  * Designates whether or not we validate sequences provided to ZSTD_compressSequences()
  * during function execution.
- * 
+ *
  * Without validation, providing a sequence that does not conform to the zstd spec will cause
  * undefined behavior, and may produce a corrupted block.
- * 
+ *
  * With validation enabled, a if sequence is invalid (see doc/zstd_compression_format.md for
  * specifics regarding offset/matchlength requirements) then the function will bail out and
  * return an error.
- * 
+ *
  */
 #define ZSTD_c_validateSequences ZSTD_c_experimentalParam12
 
@@ -2010,7 +2010,8 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs (
 ********************************************************************/
 
 /*=====   Advanced Streaming compression functions  =====*/
-/**! ZSTD_initCStream_srcSize() :
+
+/*! ZSTD_initCStream_srcSize() :
  * This function is deprecated, and equivalent to:
  *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
  *     ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
@@ -2027,7 +2028,7 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
                          int compressionLevel,
                          unsigned long long pledgedSrcSize);
 
-/**! ZSTD_initCStream_usingDict() :
+/*! ZSTD_initCStream_usingDict() :
  * This function is deprecated, and is equivalent to:
  *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
  *     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
@@ -2044,7 +2045,7 @@ ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
                      const void* dict, size_t dictSize,
                            int compressionLevel);
 
-/**! ZSTD_initCStream_advanced() :
+/*! ZSTD_initCStream_advanced() :
  * This function is deprecated, and is approximately equivalent to:
  *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
  *     // Pseudocode: Set each zstd parameter and leave the rest as-is.
@@ -2065,7 +2066,7 @@ ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
                           ZSTD_parameters params,
                           unsigned long long pledgedSrcSize);
 
-/**! ZSTD_initCStream_usingCDict() :
+/*! ZSTD_initCStream_usingCDict() :
  * This function is deprecated, and equivalent to:
  *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
  *     ZSTD_CCtx_refCDict(zcs, cdict);
@@ -2075,7 +2076,7 @@ ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
  */
 ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
 
-/**! ZSTD_initCStream_usingCDict_advanced() :
+/*! ZSTD_initCStream_usingCDict_advanced() :
  *   This function is DEPRECATED, and is approximately equivalent to:
  *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
  *     // Pseudocode: Set each zstd frame parameter and leave the rest as-is.
@@ -2148,7 +2149,8 @@ ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
 
 
 /*=====   Advanced Streaming decompression functions  =====*/
-/**
+
+/*!
  * This function is deprecated, and is equivalent to:
  *
  *     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
@@ -2159,7 +2161,7 @@ ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
  */
 ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
 
-/**
+/*!
  * This function is deprecated, and is equivalent to:
  *
  *     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
@@ -2170,7 +2172,7 @@ ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dic
  */
 ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
 
-/**
+/*!
  * This function is deprecated, and is equivalent to:
  *
  *     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
@@ -2232,7 +2234,7 @@ ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstC
 ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 
 
-/*-
+/**
   Buffer-less streaming decompression (synchronous mode)
 
   A ZSTD_DCtx object is required to track streaming operations.
diff --git a/tests/Makefile b/tests/Makefile
index 69db5987..c3f52e6d 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -149,12 +149,12 @@ fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR
 #	$(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll
 	$(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT)
 
-fuzzer  : LDFLAGS += $(MULTITHREAD_LD)
-fuzzer  : $(ZSTDMT_OBJECTS)
+fuzzer : LDFLAGS += $(MULTITHREAD_LD)
+fuzzer : $(ZSTDMT_OBJECTS)
 fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c
 
-fuzzer32: CFLAGS += -m32
-fuzzer32: $(ZSTD_FILES)
+fuzzer32 : CFLAGS += -m32
+fuzzer32 : $(ZSTD_FILES)
 	$(LINK.c) $^ -o $@$(EXT)
 
 # note : broken : requires symbols unavailable from dynamic library

From f69d8c027d0eb420d3c7129eecde8f5d1af027fe Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 2 Dec 2020 00:21:29 -0800
Subject: [PATCH 5/7] removed fullbench-lib from tests/all

this build works fine on all my systems,
but since to fail on CI environment.
Unclear why there is a difference.
This build test is not relevant anyway.
---
 tests/Makefile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/Makefile b/tests/Makefile
index c3f52e6d..a4521b60 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -82,8 +82,7 @@ DECODECORPUS_TESTTIME ?= -T30
 default: fullbench
 
 .PHONY: all
-all: fullbench fuzzer zstreamtest paramgrill datagen decodecorpus roundTripCrash \
-     fullbench-lib poolTests
+all: fullbench fuzzer zstreamtest paramgrill datagen decodecorpus roundTripCrash poolTests
 
 .PHONY: all32
 all32: fullbench32 fuzzer32 zstreamtest32

From db21d383b5db2e63cc0f4fcd0c8792b8d2ccd93f Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 2 Dec 2020 09:13:55 -0800
Subject: [PATCH 6/7] fixed fuzzer32 to support multithreading tests

though it still fails on test33:
`test 33: superblock uncompressible data, too many nocompress superblocks`
---
 tests/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/Makefile b/tests/Makefile
index a4521b60..83ebca93 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -152,7 +152,7 @@ fuzzer : LDFLAGS += $(MULTITHREAD_LD)
 fuzzer : $(ZSTDMT_OBJECTS)
 fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c
 
-fuzzer32 : CFLAGS += -m32
+fuzzer32 : CFLAGS += -m32 $(MULTITHREAD)
 fuzzer32 : $(ZSTD_FILES)
 	$(LINK.c) $^ -o $@$(EXT)
 

From 5de5c1d759494f234830bc64235239016d41e137 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 2 Dec 2020 10:34:12 -0800
Subject: [PATCH 7/7] fixed fuzzer multithreading tests

---
 tests/Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/Makefile b/tests/Makefile
index 83ebca93..42bc353c 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -148,6 +148,7 @@ fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR
 #	$(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll
 	$(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT)
 
+fuzzer : CPPFLAGS += $(MULTITHREAD_CPP)
 fuzzer : LDFLAGS += $(MULTITHREAD_LD)
 fuzzer : $(ZSTDMT_OBJECTS)
 fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c