facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

lib/compress: created ZSTD_LLcode() and ZSTD_MLcode()

Browse Source 
transform length into code.
Since transformation is needed in several places throughout the code,
better write the logic in one place.
This commit is contained in:
Yann Collet 2017-11-08 12:33:06 -08:00
parent 5aa0352742
commit 100d8ad6be
3 changed files with 40 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lib/compress/zstd_compress.c
View File

@ -1248,8 +1248,6 @@ static size_t ZSTD_compressLiterals (ZSTD_entropyCTables_t * entropy,
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
{ {
BYTE const LL_deltaCode = 19;
BYTE const ML_deltaCode = 36;
const seqDef* const sequences = seqStorePtr->sequencesStart; const seqDef* const sequences = seqStorePtr->sequencesStart;
BYTE* const llCodeTable = seqStorePtr->llCode; BYTE* const llCodeTable = seqStorePtr->llCode;
BYTE* const ofCodeTable = seqStorePtr->ofCode; BYTE* const ofCodeTable = seqStorePtr->ofCode;
@ -1259,9 +1257,9 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
for (u=0; u<nbSeq; u++) { for (u=0; u<nbSeq; u++) {
U32 const llv = sequences[u].litLength; U32 const llv = sequences[u].litLength;
U32 const mlv = sequences[u].matchLength; U32 const mlv = sequences[u].matchLength;
llCodeTable[u] = (llv> 63) ? (BYTE)ZSTD_highbit32(llv) + LL_deltaCode : LL_Code[llv]; llCodeTable[u] = (BYTE)ZSTD_LLcode(llv);
ofCodeTable[u] = (BYTE)ZSTD_highbit32(sequences[u].offset); ofCodeTable[u] = (BYTE)ZSTD_highbit32(sequences[u].offset);
mlCodeTable[u] = (mlv>127) ? (BYTE)ZSTD_highbit32(mlv) + ML_deltaCode : ML_Code[mlv]; mlCodeTable[u] = (BYTE)ZSTD_MLcode(mlv);
} }
if (seqStorePtr->longLengthID==1) if (seqStorePtr->longLengthID==1)
llCodeTable[seqStorePtr->longLengthPos] = MaxLL; llCodeTable[seqStorePtr->longLengthPos] = MaxLL;

27
lib/compress/zstd_compress_internal.h
View File

@ -195,7 +195,9 @@ struct ZSTD_CCtx_s {
}; };
static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, MEM_STATIC U32 ZSTD_LLcode(U32 litLength)
{
static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7,
8, 9, 10, 11, 12, 13, 14, 15, 8, 9, 10, 11, 12, 13, 14, 15,
16, 16, 17, 17, 18, 18, 19, 19, 16, 16, 17, 17, 18, 18, 19, 19,
20, 20, 20, 20, 21, 21, 21, 21, 20, 20, 20, 20, 21, 21, 21, 21,
@ -203,8 +205,16 @@ static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24 }; 24, 24, 24, 24, 24, 24, 24, 24 };
static const U32 LL_deltaCode = 19;
return (litLength > 63) ? ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
}
static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, /* ZSTD_MLcode() :
* note : mlBase = matchLength - MINMATCH;
* because it's the format it's stored in seqStore->sequences */
MEM_STATIC U32 ZSTD_MLcode(U32 mlBase)
{
static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39,
@ -212,13 +222,16 @@ static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11
41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
static const U32 ML_deltaCode = 36;
return (mlBase > 127) ? ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase];
}
/*! ZSTD_storeSeq() : /*! ZSTD_storeSeq() :
* Store a sequence (literal length, literals, offset code and match length code) into seqStore_t. * Store a sequence (literal length, literals, offset code and match length code) into seqStore_t.
* `offsetCode` : distance to match + 3 (values 1-3 are repCodes). * `offsetCode` : distance to match + 3 (values 1-3 are repCodes).
* `matchCode` : matchLength - MINMATCH * `mlBase` : matchLength - MINMATCH
*/ */
MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t matchCode) MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t mlBase)
{ {
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG >= 6) #if defined(ZSTD_DEBUG) && (ZSTD_DEBUG >= 6)
static const BYTE* g_start = NULL; static const BYTE* g_start = NULL;
@ -226,7 +239,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v
if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */ if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */
if ((pos > 0) && (pos < 1000000000)) if ((pos > 0) && (pos < 1000000000))
DEBUGLOG(6, "Cpos %6u :%5u literals & match %3u bytes at distance %6u", DEBUGLOG(6, "Cpos %6u :%5u literals & match %3u bytes at distance %6u",
pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); pos, (U32)litLength, (U32)mlBase+MINMATCH, (U32)offsetCode);
#endif #endif
/* copy Literals */ /* copy Literals */
assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + 128 KB); assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + 128 KB);
@ -245,12 +258,12 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v
seqStorePtr->sequences[0].offset = offsetCode + 1; seqStorePtr->sequences[0].offset = offsetCode + 1;
/* match Length */ /* match Length */
if (matchCode>0xFFFF) { if (mlBase>0xFFFF) {
assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */ assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */
seqStorePtr->longLengthID = 2; seqStorePtr->longLengthID = 2;
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
} }
seqStorePtr->sequences[0].matchLength = (U16)matchCode; seqStorePtr->sequences[0].matchLength = (U16)mlBase;
seqStorePtr->sequences++; seqStorePtr->sequences++;
} }

12
lib/compress/zstd_opt.c
View File

@ -129,8 +129,7 @@ static U32 ZSTD_getLiteralPrice(optState_t* optPtr, U32 litLength, const BYTE* l
} }
/* literal Length */ /* literal Length */
{ BYTE const LL_deltaCode = 19; { U32 const llCode = ZSTD_LLcode(litLength);
BYTE const llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
price += LL_bits[llCode] + optPtr->log2litLengthSum - ZSTD_highbit32(optPtr->litLengthFreq[llCode]+1); price += LL_bits[llCode] + optPtr->log2litLengthSum - ZSTD_highbit32(optPtr->litLengthFreq[llCode]+1);
} }
@ -151,8 +150,7 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_getPrice(optState_t* optPtr, U32 litLength, const
if (!ultra && offCode >= 20) price += (offCode-19)*2; /* handicap for long matches, to favor decompression speed */ if (!ultra && offCode >= 20) price += (offCode-19)*2; /* handicap for long matches, to favor decompression speed */
/* match Length */ /* match Length */
{ BYTE const ML_deltaCode = 36; { U32 const mlCode = ZSTD_MLcode(mlBase);
BYTE const mlCode = (mlBase>127) ? (BYTE)ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase];
price += ML_bits[mlCode] + optPtr->log2matchLengthSum - ZSTD_highbit32(optPtr->matchLengthFreq[mlCode]+1); price += ML_bits[mlCode] + optPtr->log2matchLengthSum - ZSTD_highbit32(optPtr->matchLengthFreq[mlCode]+1);
} }
@ -170,8 +168,7 @@ static void ZSTD_updatePrice(optState_t* optPtr, U32 litLength, const BYTE* lite
optPtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD; optPtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD;
/* literal Length */ /* literal Length */
{ const BYTE LL_deltaCode = 19; { const U32 llCode = ZSTD_LLcode(litLength);
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
optPtr->litLengthFreq[llCode]++; optPtr->litLengthFreq[llCode]++;
optPtr->litLengthSum++; optPtr->litLengthSum++;
} }
@ -184,8 +181,7 @@ static void ZSTD_updatePrice(optState_t* optPtr, U32 litLength, const BYTE* lite
/* match Length */ /* match Length */
{ U32 const mlBase = matchLength - MINMATCH; { U32 const mlBase = matchLength - MINMATCH;
const BYTE ML_deltaCode = 36; U32 const mlCode = ZSTD_MLcode(mlBase);
const BYTE mlCode = (mlBase>127) ? (BYTE)ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase];
optPtr->matchLengthFreq[mlCode]++; optPtr->matchLengthFreq[mlCode]++;
optPtr->matchLengthSum++; optPtr->matchLengthSum++;
} }