adding some debug functions to observe statistics
This commit is contained in:
parent
af3da079d1
commit
a95e9e80d1
@ -1561,13 +1561,14 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
|
|||||||
mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
|
mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
typedef enum {
|
typedef enum {
|
||||||
ZSTD_defaultDisallowed = 0,
|
ZSTD_defaultDisallowed = 0,
|
||||||
ZSTD_defaultAllowed = 1
|
ZSTD_defaultAllowed = 1
|
||||||
} ZSTD_defaultPolicy_e;
|
} ZSTD_defaultPolicy_e;
|
||||||
|
|
||||||
MEM_STATIC
|
MEM_STATIC symbolEncodingType_e
|
||||||
symbolEncodingType_e ZSTD_selectEncodingType(
|
ZSTD_selectEncodingType(
|
||||||
FSE_repeat* repeatMode, size_t const mostFrequent, size_t nbSeq,
|
FSE_repeat* repeatMode, size_t const mostFrequent, size_t nbSeq,
|
||||||
U32 defaultNormLog, ZSTD_defaultPolicy_e const isDefaultAllowed)
|
U32 defaultNormLog, ZSTD_defaultPolicy_e const isDefaultAllowed)
|
||||||
{
|
{
|
||||||
@ -1605,17 +1606,17 @@ symbolEncodingType_e ZSTD_selectEncodingType(
|
|||||||
return set_compressed;
|
return set_compressed;
|
||||||
}
|
}
|
||||||
|
|
||||||
MEM_STATIC
|
MEM_STATIC size_t
|
||||||
size_t ZSTD_buildCTable(void* dst, size_t dstCapacity,
|
ZSTD_buildCTable(void* dst, size_t dstCapacity,
|
||||||
FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type,
|
FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type,
|
||||||
U32* count, U32 max,
|
U32* count, U32 max,
|
||||||
BYTE const* codeTable, size_t nbSeq,
|
const BYTE* codeTable, size_t nbSeq,
|
||||||
S16 const* defaultNorm, U32 defaultNormLog, U32 defaultMax,
|
const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax,
|
||||||
FSE_CTable const* prevCTable, size_t prevCTableSize,
|
const FSE_CTable* prevCTable, size_t prevCTableSize,
|
||||||
void* workspace, size_t workspaceSize)
|
void* workspace, size_t workspaceSize)
|
||||||
{
|
{
|
||||||
BYTE* op = (BYTE*)dst;
|
BYTE* op = (BYTE*)dst;
|
||||||
BYTE const* const oend = op + dstCapacity;
|
const BYTE* const oend = op + dstCapacity;
|
||||||
|
|
||||||
switch (type) {
|
switch (type) {
|
||||||
case set_rle:
|
case set_rle:
|
||||||
@ -1917,9 +1918,9 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
|
|||||||
}
|
}
|
||||||
|
|
||||||
MEM_STATIC size_t ZSTD_compressSequences(seqStore_t* seqStorePtr,
|
MEM_STATIC size_t ZSTD_compressSequences(seqStore_t* seqStorePtr,
|
||||||
ZSTD_entropyCTables_t const* prevEntropy,
|
const ZSTD_entropyCTables_t* prevEntropy,
|
||||||
ZSTD_entropyCTables_t* nextEntropy,
|
ZSTD_entropyCTables_t* nextEntropy,
|
||||||
ZSTD_CCtx_params const* cctxParams,
|
const ZSTD_CCtx_params* cctxParams,
|
||||||
void* dst, size_t dstCapacity,
|
void* dst, size_t dstCapacity,
|
||||||
size_t srcSize, U32* workspace, int bmi2)
|
size_t srcSize, U32* workspace, int bmi2)
|
||||||
{
|
{
|
||||||
@ -1934,7 +1935,7 @@ MEM_STATIC size_t ZSTD_compressSequences(seqStore_t* seqStorePtr,
|
|||||||
if (ZSTD_isError(cSize)) return cSize;
|
if (ZSTD_isError(cSize)) return cSize;
|
||||||
|
|
||||||
/* Check compressibility */
|
/* Check compressibility */
|
||||||
{ size_t const maxCSize = srcSize - ZSTD_minGain(srcSize); /* note : fixed formula, maybe should depend on compression level, or strategy */
|
{ size_t const maxCSize = srcSize - ZSTD_minGain(srcSize); /* note : fixed formula; to be refined, depending on compression level, or strategy */
|
||||||
if (cSize >= maxCSize) return 0; /* block not compressed */
|
if (cSize >= maxCSize) return 0; /* block not compressed */
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -633,6 +633,33 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
|
|||||||
return contiguous;
|
return contiguous;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/* debug functions */
|
||||||
|
|
||||||
|
MEM_STATIC double ZSTD_fWeight(U32 rawStat)
|
||||||
|
{
|
||||||
|
U32 const fp_accuracy = 8;
|
||||||
|
U32 const fp_multiplier = (1 << fp_accuracy);
|
||||||
|
U32 const stat = rawStat + 1;
|
||||||
|
U32 const hb = ZSTD_highbit32(stat);
|
||||||
|
U32 const BWeight = hb * fp_multiplier;
|
||||||
|
U32 const FWeight = (stat << fp_accuracy) >> hb;
|
||||||
|
U32 const weight = BWeight + FWeight;
|
||||||
|
assert(hb + fp_accuracy < 31);
|
||||||
|
return (double)weight / fp_multiplier;
|
||||||
|
}
|
||||||
|
|
||||||
|
MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max)
|
||||||
|
{
|
||||||
|
unsigned u, sum;
|
||||||
|
for (u=0, sum=0; u<=max; u++) sum += table[u];
|
||||||
|
DEBUGLOG(2, "total nb elts: %u", sum);
|
||||||
|
for (u=0; u<=max; u++) {
|
||||||
|
DEBUGLOG(2, "%2u: %5u (%.2f)",
|
||||||
|
u, table[u], ZSTD_fWeight(sum) - ZSTD_fWeight(table[u]) );
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#if defined (__cplusplus)
|
#if defined (__cplusplus)
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
@ -1057,7 +1057,6 @@ size_t ZSTD_compressBlock_btultra(
|
|||||||
/* re-inforce weight of collected statistics */
|
/* re-inforce weight of collected statistics */
|
||||||
ZSTD_upscaleStats(&ms->opt);
|
ZSTD_upscaleStats(&ms->opt);
|
||||||
}
|
}
|
||||||
DEBUGLOG(5, "base=%p, src=%p, src-base=%zi", ms->window.base, src, (const BYTE*)src - (const BYTE*)ms->window.base);
|
|
||||||
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 2 /*optLevel*/, 0 /*extDict*/);
|
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, cParams, src, srcSize, 2 /*optLevel*/, 0 /*extDict*/);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
x
Reference in New Issue
Block a user