Fix off-by-one error in superblock mode (#3221)

Fixes #3212.

Long literal and match lengths had an off-by-one error in ZSTD_getSequenceLength.
Fix the off-by-one error, and add a golden compression test that catches the bug.
Also run all the golden tests in the cli-tests framework.

lib/common/zstd_internal.h

@@ -324,10 +324,10 @@ MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore
     seqLen.matchLength = seq->mlBase + MINMATCH;
     if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) {
         if (seqStore->longLengthType == ZSTD_llt_literalLength) {
-            seqLen.litLength += 0xFFFF;
+            seqLen.litLength += 0x10000;
         }
         if (seqStore->longLengthType == ZSTD_llt_matchLength) {
-            seqLen.matchLength += 0xFFFF;
+            seqLen.matchLength += 0x10000;
         }
     }
     return seqLen;

lib/decompress/zstd_decompress_block.c

@@ -1578,7 +1578,7 @@ ZSTD_decompressSequences_body(ZSTD_DCtx* dctx,
     const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart);
     const BYTE* const vBase = (const BYTE*)(dctx->virtualStart);
     const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd);
-    DEBUGLOG(5, "ZSTD_decompressSequences_body");
+    DEBUGLOG(5, "ZSTD_decompressSequences_body: nbSeq = %d", nbSeq);
     (void)frame;
 
     /* Regen sequences */

tests/cli-tests/compression/golden.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+set -e
+
+GOLDEN_DIR="$ZSTD_REPO_DIR/tests/golden-compression/"
+cp -r "$GOLDEN_DIR" golden/
+
+zstd -rf golden/ --output-dir-mirror golden-compressed/
+zstd -r -t golden-compressed/
+
+zstd --target-compressed-block-size=1024 -rf golden/ --output-dir-mirror golden-compressed/
+zstd -r -t golden-compressed/

tests/cli-tests/decompression/golden.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+GOLDEN_DIR="$ZSTD_REPO_DIR/tests/golden-decompression/"
+
+zstd -r -t "$GOLDEN_DIR"

tests/cli-tests/dictionaries/golden.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -e
+
+GOLDEN_COMP_DIR="$ZSTD_REPO_DIR/tests/golden-compression/"
+GOLDEN_DICT_DIR="$ZSTD_REPO_DIR/tests/golden-dictionaries/"
+
+zstd -D "$GOLDEN_DICT_DIR/http-dict-missing-symbols" "$GOLDEN_COMP_DIR/http" -o http.zst
+zstd -D "$GOLDEN_DICT_DIR/http-dict-missing-symbols" -t http.zst

tests/cli-tests/run.py

@@ -699,6 +699,7 @@ if __name__ == "__main__":
     if args.exec_prefix is not None:
         env["EXEC_PREFIX"] = args.exec_prefix
     env["ZSTD_SYMLINK_DIR"] = zstd_symlink_dir
+    env["ZSTD_REPO_DIR"] = os.path.abspath(REPO_DIR)
     env["DATAGEN_BIN"] = os.path.abspath(args.datagen)
     env["ZSTDGREP_BIN"] = os.path.abspath(args.zstdgrep)
     env["ZSTDLESS_BIN"] = os.path.abspath(args.zstdless)