facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

btopt : minor adjustment of update frequencies

dev
Yann Collet 2018-05-10 16:32:36 -07:00
parent ac6105463a
commit 74b1c75d64
2 changed files with 23 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/compress/zstd_compress_internal.h
View File

@ -76,7 +76,7 @@ typedef struct {
U32 rep[ZSTD_REP_NUM]; U32 rep[ZSTD_REP_NUM];
} ZSTD_optimal_t; } ZSTD_optimal_t;
typedef enum { zop_none=0, zop_predef, zop_static } ZSTD_OptPrice_e; typedef enum { zop_dynamic=0, zop_predef, zop_static } ZSTD_OptPrice_e;
typedef struct { typedef struct {
/* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */ /* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */

37
lib/compress/zstd_opt.c
View File

@ -12,7 +12,7 @@
#include "zstd_opt.h" #include "zstd_opt.h"
#define ZSTD_LITFREQ_ADD 2 /* scaling factor for litFreq, so that frequencies adapt faster to new stats. Also used for matchSum (?) */ #define ZSTD_LITFREQ_ADD 2 /* scaling factor for litFreq, so that frequencies adapt faster to new stats */
#define ZSTD_FREQ_DIV 4 /* log factor when using previous stats to init next stats */ #define ZSTD_FREQ_DIV 4 /* log factor when using previous stats to init next stats */
#define ZSTD_MAX_PRICE (1<<30) #define ZSTD_MAX_PRICE (1<<30)
@ -32,24 +32,31 @@ static void ZSTD_setLog2Prices(optState_t* optPtr)
static void ZSTD_rescaleFreqs(optState_t* const optPtr, static void ZSTD_rescaleFreqs(optState_t* const optPtr,
const BYTE* const src, size_t const srcSize) const BYTE* const src, size_t const srcSize)
{ {
optPtr->priceType = zop_none; optPtr->priceType = zop_dynamic;
if (optPtr->litLengthSum == 0) { /* first init */ if (optPtr->litLengthSum == 0) { /* first block : init */
unsigned u; unsigned u;
if (srcSize <= 1024) optPtr->priceType = zop_predef; if (srcSize <= 1024) /* heuristic */
optPtr->priceType = zop_predef;
assert(optPtr->symbolCosts != NULL); assert(optPtr->symbolCosts != NULL);
if (optPtr->symbolCosts->hufCTable_repeatMode == HUF_repeat_valid) { /* huffman table presumed generated by dictionary */ if (optPtr->symbolCosts->hufCTable_repeatMode == HUF_repeat_valid) { /* huffman table presumed generated by dictionary */
optPtr->priceType = zop_static; if (srcSize <= 8192) /* heuristic */
optPtr->priceType = zop_static;
else {
assert(optPtr->priceType == zop_dynamic);
}
} }
assert(optPtr->litFreq!=NULL); assert(optPtr->litFreq != NULL);
for (u=0; u<=MaxLit; u++) { unsigned max = MaxLit;
optPtr->litFreq[u] = 0; FSE_count(optPtr->litFreq, &max, src, srcSize); /* use raw first block to init statistics */
for (u=0; u<srcSize; u++) }
optPtr->litFreq[src[u]]++;
optPtr->litSum = 0; optPtr->litSum = 0;
for (u=0; u<=MaxLit; u++) { for (u=0; u<=MaxLit; u++) {
optPtr->litFreq[u] = 1 + (optPtr->litFreq[u] >> ZSTD_FREQ_DIV); optPtr->litFreq[u] = 1 + (optPtr->litFreq[u] >> (ZSTD_FREQ_DIV+1));
optPtr->litSum += optPtr->litFreq[u]; optPtr->litSum += optPtr->litFreq[u];
} }
@ -63,7 +70,7 @@ static void ZSTD_rescaleFreqs(optState_t* const optPtr,
optPtr->offCodeFreq[u] = 1; optPtr->offCodeFreq[u] = 1;
optPtr->offCodeSum = (MaxOff+1); optPtr->offCodeSum = (MaxOff+1);
} else { } else { /* new block : re-use previous statistics, scaled down */
unsigned u; unsigned u;
optPtr->litSum = 0; optPtr->litSum = 0;
@ -73,17 +80,17 @@ static void ZSTD_rescaleFreqs(optState_t* const optPtr,
} }
optPtr->litLengthSum = 0; optPtr->litLengthSum = 0;
for (u=0; u<=MaxLL; u++) { for (u=0; u<=MaxLL; u++) {
optPtr->litLengthFreq[u] = 1 + (optPtr->litLengthFreq[u]>>(ZSTD_FREQ_DIV+1)); optPtr->litLengthFreq[u] = 1 + (optPtr->litLengthFreq[u] >> ZSTD_FREQ_DIV);
optPtr->litLengthSum += optPtr->litLengthFreq[u]; optPtr->litLengthSum += optPtr->litLengthFreq[u];
} }
optPtr->matchLengthSum = 0; optPtr->matchLengthSum = 0;
for (u=0; u<=MaxML; u++) { for (u=0; u<=MaxML; u++) {
optPtr->matchLengthFreq[u] = 1 + (optPtr->matchLengthFreq[u]>>ZSTD_FREQ_DIV); optPtr->matchLengthFreq[u] = 1 + (optPtr->matchLengthFreq[u] >> ZSTD_FREQ_DIV);
optPtr->matchLengthSum += optPtr->matchLengthFreq[u]; optPtr->matchLengthSum += optPtr->matchLengthFreq[u];
} }
optPtr->offCodeSum = 0; optPtr->offCodeSum = 0;
for (u=0; u<=MaxOff; u++) { for (u=0; u<=MaxOff; u++) {
optPtr->offCodeFreq[u] = 1 + (optPtr->offCodeFreq[u]>>ZSTD_FREQ_DIV); optPtr->offCodeFreq[u] = 1 + (optPtr->offCodeFreq[u] >> ZSTD_FREQ_DIV);
optPtr->offCodeSum += optPtr->offCodeFreq[u]; optPtr->offCodeSum += optPtr->offCodeFreq[u];
} }
} }