From 2a907bf4aa31cd45e24985f4728b59233fb0117a Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Thu, 25 Mar 2021 07:30:53 -0700 Subject: [PATCH 1/2] Move lastCountSize into a returned struct, fix MSAN error --- lib/compress/zstd_compress.c | 25 ++++++++++++++----------- lib/compress/zstd_compress_internal.h | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 9a2486d1..32872072 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2231,6 +2231,7 @@ typedef struct { U32 Offtype; U32 MLtype; size_t size; + size_t lastCountSize; /* Accounts for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */ } ZSTD_symbolEncodingTypeStats_t; /* ZSTD_buildSequencesStatistics(): @@ -2240,7 +2241,7 @@ typedef struct { * entropyWkspSize must be of size at least ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1)*sizeof(U32) */ static ZSTD_symbolEncodingTypeStats_t -ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq, size_t* lastCountSize, +ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq, const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy, BYTE* dst, const BYTE* const dstEnd, ZSTD_strategy strategy, unsigned* countWorkspace, @@ -2256,6 +2257,7 @@ ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq, size_t* las const BYTE* const mlCodeTable = seqStorePtr->mlCode; ZSTD_symbolEncodingTypeStats_t stats; + stats.lastCountSize = 0; /* convert length/distances into codes */ ZSTD_seqToCodes(seqStorePtr); assert(op <= oend); @@ -2285,7 +2287,7 @@ ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq, size_t* las return stats; } if (stats.LLtype == set_compressed) - *lastCountSize = countSize; + stats.lastCountSize = countSize; op += countSize; assert(op <= oend); } } @@ -2317,7 +2319,7 @@ ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq, size_t* las return stats; } if (stats.Offtype == set_compressed) - *lastCountSize = countSize; + stats.lastCountSize = countSize; op += countSize; assert(op <= oend); } } @@ -2347,7 +2349,7 @@ ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq, size_t* las return stats; } if (stats.MLtype == set_compressed) - *lastCountSize = countSize; + stats.lastCountSize = countSize; op += countSize; assert(op <= oend); } } @@ -2382,7 +2384,7 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr, BYTE* const ostart = (BYTE*)dst; BYTE* const oend = ostart + dstCapacity; BYTE* op = ostart; - size_t lastCountSize = 0; + size_t lastCountSize; entropyWorkspace = count + (MaxSeq + 1); entropyWkspSize -= (MaxSeq + 1) * sizeof(*count); @@ -2431,13 +2433,14 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr, ZSTD_symbolEncodingTypeStats_t stats; BYTE* seqHead = op++; /* build stats for sequences */ - stats = ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, &lastCountSize, - &prevEntropy->fse, &nextEntropy->fse, - op, oend, - strategy, count, - entropyWorkspace, entropyWkspSize); + stats = ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, + &prevEntropy->fse, &nextEntropy->fse, + op, oend, + strategy, count, + entropyWorkspace, entropyWkspSize); FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!"); *seqHead = (BYTE)((stats.LLtype<<6) + (stats.Offtype<<4) + (stats.MLtype<<2)); + lastCountSize = stats.lastCountSize; op += stats.size; } @@ -2936,7 +2939,6 @@ static size_t ZSTD_buildBlockEntropyStats_sequences(seqStore_t* seqStorePtr, DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_sequences (nbSeq=%zu)", nbSeq); stats = ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, - &fseMetadata->lastCountSize, prevEntropy, nextEntropy, op, oend, strategy, countWorkspace, entropyWorkspace, entropyWorkspaceSize); @@ -2944,6 +2946,7 @@ static size_t ZSTD_buildBlockEntropyStats_sequences(seqStore_t* seqStorePtr, fseMetadata->llType = (symbolEncodingType_e) stats.LLtype; fseMetadata->ofType = (symbolEncodingType_e) stats.Offtype; fseMetadata->mlType = (symbolEncodingType_e) stats.MLtype; + fseMetadata->lastCountSize = stats.lastCountSize; return stats.size; } diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index f27b2bff..4338274e 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -106,7 +106,7 @@ typedef struct { symbolEncodingType_e mlType; BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE]; size_t fseTablesSize; - size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_entropyCompressSequences_internal() */ + size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */ } ZSTD_fseCTablesMetadata_t; typedef struct { From ef4e26bda5287a31552b263f0aa35d71ff58ee31 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Thu, 25 Mar 2021 09:11:43 -0700 Subject: [PATCH 2/2] Add clang msan fuzz test to github actions --- .github/workflows/generic-dev.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/generic-dev.yml b/.github/workflows/generic-dev.yml index 14e2e281..f5a12c9a 100644 --- a/.github/workflows/generic-dev.yml +++ b/.github/workflows/generic-dev.yml @@ -126,6 +126,16 @@ jobs: make libc6install CFLAGS="-O2 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest + clang-msan-fuzz: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: clang + MSan + Fuzz Test + run: | + sudo apt-get update + sudo apt-get install clang + CC=clang FUZZER_FLAGS="--long-tests" make clean msan-fuzztest + asan-ubsan-msan-regression: runs-on: ubuntu-latest steps: