From 100d8ad6be8ecfe0ec1dc31f1567887e18c891cc Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 8 Nov 2017 12:33:06 -0800 Subject: [PATCH] lib/compress: created ZSTD_LLcode() and ZSTD_MLcode() transform length into code. Since transformation is needed in several places throughout the code, better write the logic in one place. --- lib/compress/zstd_compress.c | 6 +-- lib/compress/zstd_compress_internal.h | 55 +++++++++++++++++---------- lib/compress/zstd_opt.c | 12 ++---- 3 files changed, 40 insertions(+), 33 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index a1162e3a..582e3e50 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1248,8 +1248,6 @@ static size_t ZSTD_compressLiterals (ZSTD_entropyCTables_t * entropy, void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) { - BYTE const LL_deltaCode = 19; - BYTE const ML_deltaCode = 36; const seqDef* const sequences = seqStorePtr->sequencesStart; BYTE* const llCodeTable = seqStorePtr->llCode; BYTE* const ofCodeTable = seqStorePtr->ofCode; @@ -1259,9 +1257,9 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) for (u=0; u 63) ? (BYTE)ZSTD_highbit32(llv) + LL_deltaCode : LL_Code[llv]; + llCodeTable[u] = (BYTE)ZSTD_LLcode(llv); ofCodeTable[u] = (BYTE)ZSTD_highbit32(sequences[u].offset); - mlCodeTable[u] = (mlv>127) ? (BYTE)ZSTD_highbit32(mlv) + ML_deltaCode : ML_Code[mlv]; + mlCodeTable[u] = (BYTE)ZSTD_MLcode(mlv); } if (seqStorePtr->longLengthID==1) llCodeTable[seqStorePtr->longLengthPos] = MaxLL; diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 7501f7aa..e29a61b7 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -195,30 +195,43 @@ struct ZSTD_CCtx_s { }; -static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 16, 17, 17, 18, 18, 19, 19, - 20, 20, 20, 20, 21, 21, 21, 21, - 22, 22, 22, 22, 22, 22, 22, 22, - 23, 23, 23, 23, 23, 23, 23, 23, - 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24 }; +MEM_STATIC U32 ZSTD_LLcode(U32 litLength) +{ + static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 20, 20, 21, 21, 21, 21, + 22, 22, 22, 22, 22, 22, 22, 22, + 23, 23, 23, 23, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24 }; + static const U32 LL_deltaCode = 19; + return (litLength > 63) ? ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; +} -static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, - 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, - 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, - 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; +/* ZSTD_MLcode() : + * note : mlBase = matchLength - MINMATCH; + * because it's the format it's stored in seqStore->sequences */ +MEM_STATIC U32 ZSTD_MLcode(U32 mlBase) +{ + static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; + static const U32 ML_deltaCode = 36; + return (mlBase > 127) ? ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase]; +} /*! ZSTD_storeSeq() : * Store a sequence (literal length, literals, offset code and match length code) into seqStore_t. * `offsetCode` : distance to match + 3 (values 1-3 are repCodes). - * `matchCode` : matchLength - MINMATCH + * `mlBase` : matchLength - MINMATCH */ -MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t matchCode) +MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t mlBase) { #if defined(ZSTD_DEBUG) && (ZSTD_DEBUG >= 6) static const BYTE* g_start = NULL; @@ -226,7 +239,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */ if ((pos > 0) && (pos < 1000000000)) DEBUGLOG(6, "Cpos %6u :%5u literals & match %3u bytes at distance %6u", - pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); + pos, (U32)litLength, (U32)mlBase+MINMATCH, (U32)offsetCode); #endif /* copy Literals */ assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + 128 KB); @@ -245,12 +258,12 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v seqStorePtr->sequences[0].offset = offsetCode + 1; /* match Length */ - if (matchCode>0xFFFF) { + if (mlBase>0xFFFF) { assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */ seqStorePtr->longLengthID = 2; seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); } - seqStorePtr->sequences[0].matchLength = (U16)matchCode; + seqStorePtr->sequences[0].matchLength = (U16)mlBase; seqStorePtr->sequences++; } diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index dab53835..5e42039a 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -129,8 +129,7 @@ static U32 ZSTD_getLiteralPrice(optState_t* optPtr, U32 litLength, const BYTE* l } /* literal Length */ - { BYTE const LL_deltaCode = 19; - BYTE const llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; + { U32 const llCode = ZSTD_LLcode(litLength); price += LL_bits[llCode] + optPtr->log2litLengthSum - ZSTD_highbit32(optPtr->litLengthFreq[llCode]+1); } @@ -151,8 +150,7 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_getPrice(optState_t* optPtr, U32 litLength, const if (!ultra && offCode >= 20) price += (offCode-19)*2; /* handicap for long matches, to favor decompression speed */ /* match Length */ - { BYTE const ML_deltaCode = 36; - BYTE const mlCode = (mlBase>127) ? (BYTE)ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase]; + { U32 const mlCode = ZSTD_MLcode(mlBase); price += ML_bits[mlCode] + optPtr->log2matchLengthSum - ZSTD_highbit32(optPtr->matchLengthFreq[mlCode]+1); } @@ -170,8 +168,7 @@ static void ZSTD_updatePrice(optState_t* optPtr, U32 litLength, const BYTE* lite optPtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD; /* literal Length */ - { const BYTE LL_deltaCode = 19; - const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; + { const U32 llCode = ZSTD_LLcode(litLength); optPtr->litLengthFreq[llCode]++; optPtr->litLengthSum++; } @@ -184,8 +181,7 @@ static void ZSTD_updatePrice(optState_t* optPtr, U32 litLength, const BYTE* lite /* match Length */ { U32 const mlBase = matchLength - MINMATCH; - const BYTE ML_deltaCode = 36; - const BYTE mlCode = (mlBase>127) ? (BYTE)ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase]; + U32 const mlCode = ZSTD_MLcode(mlBase); optPtr->matchLengthFreq[mlCode]++; optPtr->matchLengthSum++; }