new initializer for ll price

dev
Yann Collet 2021-09-03 16:07:31 -07:00
parent f0fc8cb3e1
commit 27a8bbe265
3 changed files with 102 additions and 74 deletions

View File

@ -24,11 +24,11 @@
* Price functions for optimal parser
***************************************/
#if 0 /* approximation at bit level */
#if 0 /* approximation at bit level (for tests) */
# define BITCOST_ACCURACY 0
# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
# define WEIGHT(stat) ((void)opt, ZSTD_bitWeight(stat))
#elif 0 /* fractional bit accuracy */
# define WEIGHT(stat, opt) ((void)opt, ZSTD_bitWeight(stat))
#elif 0 /* fractional bit accuracy (for tests) */
# define BITCOST_ACCURACY 8
# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
# define WEIGHT(stat,opt) ((void)opt, ZSTD_fracWeight(stat))
@ -94,10 +94,21 @@ static U32 ZSTD_downscaleStat(unsigned* table, U32 lastEltIndex, int malus)
return sum;
}
static U32 sum_u32(const unsigned table[], size_t nbElts)
{
size_t n;
U32 total = 0;
for (n=0; n<nbElts; n++) {
total += table[n];
}
return total;
}
/* ZSTD_rescaleFreqs() :
* if first block (detected by optPtr->litLengthSum == 0) : init statistics
* take hints from dictionary if there is one
* or init from zero, using src for literals stats, or flat 1 for match symbols
* and init from zero if there is none,
* using src for literals stats, and baseline stats for sequence symbols
* otherwise downscale existing stats, to be used as seed for next block.
*/
static void
@ -177,11 +188,16 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1);
}
{ unsigned ll;
for (ll=0; ll<=MaxLL; ll++)
optPtr->litLengthFreq[ll] = 1;
{ unsigned const baseLLfreqs[MaxLL+1] = {
4, 2, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1
};
memcpy(optPtr->litLengthFreq, baseLLfreqs, sizeof(baseLLfreqs)); optPtr->litLengthSum = sum_u32(baseLLfreqs, MaxLL+1);
//unsigned ll; for (ll=0; ll<=MaxLL; ll++) optPtr->litLengthFreq[ll] = 1; optPtr->litLengthSum = MaxLL+1;
}
optPtr->litLengthSum = MaxLL+1;
{ unsigned ml;
for (ml=0; ml<=MaxML; ml++)
@ -189,11 +205,16 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
}
optPtr->matchLengthSum = MaxML+1;
{ unsigned of;
for (of=0; of<=MaxOff; of++)
optPtr->offCodeFreq[of] = 1;
{ unsigned const baseOFCfreqs[MaxOff+1] = {
8, 4, 3, 3, 3, 4, 4, 4,
4, 4, 3, 3, 3, 2, 2, 2,
1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1,
};
//memcpy(optPtr->offCodeFreq, baseOFCfreqs, sizeof(baseOFCfreqs)); optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1);
unsigned of; for (of=0; of<=MaxOff; of++) optPtr->offCodeFreq[of] = 1; optPtr->offCodeSum = MaxOff+1;
}
optPtr->offCodeSum = MaxOff+1;
}
@ -901,11 +922,11 @@ static void ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm, ZSTD_match_
ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock);
}
/*-*******************************
* Optimal parser
*********************************/
static U32 ZSTD_totalLen(ZSTD_optimal_t sol)
{
return sol.litlen + sol.mlen;

View File

@ -139,7 +139,7 @@ static unsigned ZDICT_NbCommonBytes (size_t val)
_BitScanForward64( &r, (U64)val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_ctzll((U64)val) >> 3);
return (unsigned)(__builtin_ctzll((U64)val) >> 3);
# else
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
@ -150,7 +150,7 @@ static unsigned ZDICT_NbCommonBytes (size_t val)
_BitScanForward( &r, (U32)val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_ctz((U32)val) >> 3);
return (unsigned)(__builtin_ctz((U32)val) >> 3);
# else
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
@ -163,7 +163,7 @@ static unsigned ZDICT_NbCommonBytes (size_t val)
_BitScanReverse64( &r, val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_clzll(val) >> 3);
return (unsigned)(__builtin_clzll(val) >> 3);
# else
unsigned r;
const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
@ -178,7 +178,7 @@ static unsigned ZDICT_NbCommonBytes (size_t val)
_BitScanReverse( &r, (unsigned long)val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_clz((U32)val) >> 3);
return (unsigned)(__builtin_clz((U32)val) >> 3);
# else
unsigned r;
if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
@ -235,7 +235,7 @@ static dictItem ZDICT_analyzePos(
U32 savings[LLIMIT] = {0};
const BYTE* b = (const BYTE*)buffer;
size_t maxLength = LLIMIT;
size_t pos = suffix[start];
size_t pos = (size_t)suffix[start];
U32 end = start;
dictItem solution;
@ -369,7 +369,7 @@ static dictItem ZDICT_analyzePos(
savings[i] = savings[i-1] + (lengthList[i] * (i-3));
DISPLAYLEVEL(4, "Selected dict at position %u, of length %u : saves %u (ratio: %.2f) \n",
(unsigned)pos, (unsigned)maxLength, (unsigned)savings[maxLength], (double)savings[maxLength] / maxLength);
(unsigned)pos, (unsigned)maxLength, (unsigned)savings[maxLength], (double)savings[maxLength] / (double)maxLength);
solution.pos = (U32)pos;
solution.length = (U32)maxLength;
@ -379,7 +379,7 @@ static dictItem ZDICT_analyzePos(
{ U32 id;
for (id=start; id<end; id++) {
U32 p, pEnd, length;
U32 const testedPos = suffix[id];
U32 const testedPos = (U32)suffix[id];
if (testedPos == pos)
length = solution.length;
else {
@ -442,7 +442,7 @@ static U32 ZDICT_tryMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, const
if ((table[u].pos + table[u].length >= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */
/* append */
int const addedLength = (int)eltEnd - (table[u].pos + table[u].length);
int const addedLength = (int)eltEnd - (int)(table[u].pos + table[u].length);
table[u].savings += elt.length / 8; /* rough approx bonus */
if (addedLength > 0) { /* otherwise, elt fully included into existing */
table[u].length += addedLength;
@ -766,6 +766,13 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
pos += fileSizes[u];
}
if (notificationLevel >= 4) {
/* writeStats */
DISPLAYLEVEL(4, "Offset Code Frequencies : \n");
for (u=0; u<=offcodeMax; u++) {
DISPLAYLEVEL(4, "%2u :%7u \n", u, offcodeCount[u]);
} }
/* analyze, build stats, starting with literals */
{ size_t maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog);
if (HUF_isError(maxNbBits)) {
@ -872,7 +879,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
MEM_writeLE32(dstPtr+8, bestRepOffset[2].offset);
#else
/* at this stage, we don't use the result of "most common first offset",
as the impact of statistics is not properly evaluated */
* as the impact of statistics is not properly evaluated */
MEM_writeLE32(dstPtr+0, repStartValue[0]);
MEM_writeLE32(dstPtr+4, repStartValue[1]);
MEM_writeLE32(dstPtr+8, repStartValue[2]);

View File

@ -11,10 +11,10 @@ silesia.tar, level 6, compress
silesia.tar, level 7, compress simple, 4576828
silesia.tar, level 9, compress simple, 4552584
silesia.tar, level 13, compress simple, 4502955
silesia.tar, level 16, compress simple, 4356834
silesia.tar, level 19, compress simple, 4264388
silesia.tar, level 16, compress simple, 4356970
silesia.tar, level 19, compress simple, 4264314
silesia.tar, uncompressed literals, compress simple, 4861423
silesia.tar, uncompressed literals optimal, compress simple, 4264388
silesia.tar, uncompressed literals optimal, compress simple, 4264314
silesia.tar, huffman literals, compress simple, 6186042
github.tar, level -5, compress simple, 46856
github.tar, level -3, compress simple, 43754
@ -29,9 +29,9 @@ github.tar, level 7, compress
github.tar, level 9, compress simple, 36767
github.tar, level 13, compress simple, 35501
github.tar, level 16, compress simple, 40255
github.tar, level 19, compress simple, 32837
github.tar, level 19, compress simple, 32820
github.tar, uncompressed literals, compress simple, 38441
github.tar, uncompressed literals optimal, compress simple, 32837
github.tar, uncompressed literals optimal, compress simple, 32820
github.tar, huffman literals, compress simple, 42490
silesia, level -5, compress cctx, 6737607
silesia, level -3, compress cctx, 6444677
@ -45,8 +45,8 @@ silesia, level 6, compress
silesia, level 7, compress cctx, 4567203
silesia, level 9, compress cctx, 4543311
silesia, level 13, compress cctx, 4493990
silesia, level 16, compress cctx, 4360251
silesia, level 19, compress cctx, 4283236
silesia, level 16, compress cctx, 4360072
silesia, level 19, compress cctx, 4282792
silesia, long distance mode, compress cctx, 4849551
silesia, multithreaded, compress cctx, 4849551
silesia, multithreaded long distance mode, compress cctx, 4849551
@ -55,7 +55,7 @@ silesia, small hash log, compress
silesia, small chain log, compress cctx, 4912199
silesia, explicit params, compress cctx, 4794480
silesia, uncompressed literals, compress cctx, 4849551
silesia, uncompressed literals optimal, compress cctx, 4283236
silesia, uncompressed literals optimal, compress cctx, 4282792
silesia, huffman literals, compress cctx, 6178460
silesia, multithreaded with advanced params, compress cctx, 4849551
github, level -5, compress cctx, 205285
@ -109,8 +109,8 @@ silesia, level 6, zstdcli,
silesia, level 7, zstdcli, 4567251
silesia, level 9, zstdcli, 4543359
silesia, level 13, zstdcli, 4494038
silesia, level 16, zstdcli, 4360299
silesia, level 19, zstdcli, 4283284
silesia, level 16, zstdcli, 4360120
silesia, level 19, zstdcli, 4282840
silesia, long distance mode, zstdcli, 4840807
silesia, multithreaded, zstdcli, 4849599
silesia, multithreaded long distance mode, zstdcli, 4840807
@ -119,7 +119,7 @@ silesia, small hash log, zstdcli,
silesia, small chain log, zstdcli, 4912247
silesia, explicit params, zstdcli, 4795856
silesia, uncompressed literals, zstdcli, 5128030
silesia, uncompressed literals optimal, zstdcli, 4317944
silesia, uncompressed literals optimal, zstdcli, 4318048
silesia, huffman literals, zstdcli, 5326317
silesia, multithreaded with advanced params, zstdcli, 5128030
silesia.tar, level -5, zstdcli, 6738934
@ -134,8 +134,8 @@ silesia.tar, level 6, zstdcli,
silesia.tar, level 7, zstdcli, 4578883
silesia.tar, level 9, zstdcli, 4553498
silesia.tar, level 13, zstdcli, 4502959
silesia.tar, level 16, zstdcli, 4356838
silesia.tar, level 19, zstdcli, 4264392
silesia.tar, level 16, zstdcli, 4356974
silesia.tar, level 19, zstdcli, 4264318
silesia.tar, no source size, zstdcli, 4861507
silesia.tar, long distance mode, zstdcli, 4853225
silesia.tar, multithreaded, zstdcli, 4861511
@ -213,7 +213,7 @@ github.tar, level 13, zstdcli,
github.tar, level 13 with dict, zstdcli, 38730
github.tar, level 16, zstdcli, 40259
github.tar, level 16 with dict, zstdcli, 33643
github.tar, level 19, zstdcli, 32841
github.tar, level 19, zstdcli, 32824
github.tar, level 19 with dict, zstdcli, 32899
github.tar, no source size, zstdcli, 38442
github.tar, no source size with dict, zstdcli, 38004
@ -225,7 +225,7 @@ github.tar, small hash log, zstdcli,
github.tar, small chain log, zstdcli, 41673
github.tar, explicit params, zstdcli, 41227
github.tar, uncompressed literals, zstdcli, 41126
github.tar, uncompressed literals optimal, zstdcli, 35392
github.tar, uncompressed literals optimal, zstdcli, 35498
github.tar, huffman literals, zstdcli, 38781
github.tar, multithreaded with advanced params, zstdcli, 41126
silesia, level -5, advanced one pass, 6737607
@ -248,8 +248,8 @@ silesia, level 11 row 2, advanced
silesia, level 12 row 1, advanced one pass, 4503117
silesia, level 12 row 2, advanced one pass, 4505152
silesia, level 13, advanced one pass, 4493990
silesia, level 16, advanced one pass, 4360251
silesia, level 19, advanced one pass, 4283236
silesia, level 16, advanced one pass, 4360072
silesia, level 19, advanced one pass, 4282792
silesia, no source size, advanced one pass, 4849551
silesia, long distance mode, advanced one pass, 4840738
silesia, multithreaded, advanced one pass, 4849551
@ -259,7 +259,7 @@ silesia, small hash log, advanced
silesia, small chain log, advanced one pass, 4912199
silesia, explicit params, advanced one pass, 4795856
silesia, uncompressed literals, advanced one pass, 5127982
silesia, uncompressed literals optimal, advanced one pass, 4317896
silesia, uncompressed literals optimal, advanced one pass, 4318000
silesia, huffman literals, advanced one pass, 5326269
silesia, multithreaded with advanced params, advanced one pass, 5127982
silesia.tar, level -5, advanced one pass, 6738593
@ -282,8 +282,8 @@ silesia.tar, level 11 row 2, advanced
silesia.tar, level 12 row 1, advanced one pass, 4513603
silesia.tar, level 12 row 2, advanced one pass, 4514568
silesia.tar, level 13, advanced one pass, 4502955
silesia.tar, level 16, advanced one pass, 4356834
silesia.tar, level 19, advanced one pass, 4264388
silesia.tar, level 16, advanced one pass, 4356970
silesia.tar, level 19, advanced one pass, 4264314
silesia.tar, no source size, advanced one pass, 4861423
silesia.tar, long distance mode, advanced one pass, 4847753
silesia.tar, multithreaded, advanced one pass, 4861507
@ -527,7 +527,7 @@ github.tar, level 16 with dict dms, advanced
github.tar, level 16 with dict dds, advanced one pass, 33544
github.tar, level 16 with dict copy, advanced one pass, 33639
github.tar, level 16 with dict load, advanced one pass, 39353
github.tar, level 19, advanced one pass, 32837
github.tar, level 19, advanced one pass, 32820
github.tar, level 19 with dict, advanced one pass, 32895
github.tar, level 19 with dict dms, advanced one pass, 32672
github.tar, level 19 with dict dds, advanced one pass, 32672
@ -543,7 +543,7 @@ github.tar, small hash log, advanced
github.tar, small chain log, advanced one pass, 41669
github.tar, explicit params, advanced one pass, 41227
github.tar, uncompressed literals, advanced one pass, 41122
github.tar, uncompressed literals optimal, advanced one pass, 35388
github.tar, uncompressed literals optimal, advanced one pass, 35494
github.tar, huffman literals, advanced one pass, 38777
github.tar, multithreaded with advanced params, advanced one pass, 41122
silesia, level -5, advanced one pass small out, 6737607
@ -566,8 +566,8 @@ silesia, level 11 row 2, advanced
silesia, level 12 row 1, advanced one pass small out, 4503117
silesia, level 12 row 2, advanced one pass small out, 4505152
silesia, level 13, advanced one pass small out, 4493990
silesia, level 16, advanced one pass small out, 4360251
silesia, level 19, advanced one pass small out, 4283236
silesia, level 16, advanced one pass small out, 4360072
silesia, level 19, advanced one pass small out, 4282792
silesia, no source size, advanced one pass small out, 4849551
silesia, long distance mode, advanced one pass small out, 4840738
silesia, multithreaded, advanced one pass small out, 4849551
@ -577,7 +577,7 @@ silesia, small hash log, advanced
silesia, small chain log, advanced one pass small out, 4912199
silesia, explicit params, advanced one pass small out, 4795856
silesia, uncompressed literals, advanced one pass small out, 5127982
silesia, uncompressed literals optimal, advanced one pass small out, 4317896
silesia, uncompressed literals optimal, advanced one pass small out, 4318000
silesia, huffman literals, advanced one pass small out, 5326269
silesia, multithreaded with advanced params, advanced one pass small out, 5127982
silesia.tar, level -5, advanced one pass small out, 6738593
@ -600,8 +600,8 @@ silesia.tar, level 11 row 2, advanced
silesia.tar, level 12 row 1, advanced one pass small out, 4513603
silesia.tar, level 12 row 2, advanced one pass small out, 4514568
silesia.tar, level 13, advanced one pass small out, 4502955
silesia.tar, level 16, advanced one pass small out, 4356834
silesia.tar, level 19, advanced one pass small out, 4264388
silesia.tar, level 16, advanced one pass small out, 4356970
silesia.tar, level 19, advanced one pass small out, 4264314
silesia.tar, no source size, advanced one pass small out, 4861423
silesia.tar, long distance mode, advanced one pass small out, 4847753
silesia.tar, multithreaded, advanced one pass small out, 4861507
@ -845,7 +845,7 @@ github.tar, level 16 with dict dms, advanced
github.tar, level 16 with dict dds, advanced one pass small out, 33544
github.tar, level 16 with dict copy, advanced one pass small out, 33639
github.tar, level 16 with dict load, advanced one pass small out, 39353
github.tar, level 19, advanced one pass small out, 32837
github.tar, level 19, advanced one pass small out, 32820
github.tar, level 19 with dict, advanced one pass small out, 32895
github.tar, level 19 with dict dms, advanced one pass small out, 32672
github.tar, level 19 with dict dds, advanced one pass small out, 32672
@ -861,7 +861,7 @@ github.tar, small hash log, advanced
github.tar, small chain log, advanced one pass small out, 41669
github.tar, explicit params, advanced one pass small out, 41227
github.tar, uncompressed literals, advanced one pass small out, 41122
github.tar, uncompressed literals optimal, advanced one pass small out, 35388
github.tar, uncompressed literals optimal, advanced one pass small out, 35494
github.tar, huffman literals, advanced one pass small out, 38777
github.tar, multithreaded with advanced params, advanced one pass small out, 41122
silesia, level -5, advanced streaming, 6882505
@ -884,8 +884,8 @@ silesia, level 11 row 2, advanced
silesia, level 12 row 1, advanced streaming, 4503117
silesia, level 12 row 2, advanced streaming, 4505152
silesia, level 13, advanced streaming, 4493990
silesia, level 16, advanced streaming, 4360251
silesia, level 19, advanced streaming, 4283236
silesia, level 16, advanced streaming, 4360072
silesia, level 19, advanced streaming, 4282792
silesia, no source size, advanced streaming, 4849515
silesia, long distance mode, advanced streaming, 4840738
silesia, multithreaded, advanced streaming, 4849551
@ -895,7 +895,7 @@ silesia, small hash log, advanced
silesia, small chain log, advanced streaming, 4912199
silesia, explicit params, advanced streaming, 4795884
silesia, uncompressed literals, advanced streaming, 5127982
silesia, uncompressed literals optimal, advanced streaming, 4317896
silesia, uncompressed literals optimal, advanced streaming, 4318000
silesia, huffman literals, advanced streaming, 5331171
silesia, multithreaded with advanced params, advanced streaming, 5127982
silesia.tar, level -5, advanced streaming, 6982759
@ -918,8 +918,8 @@ silesia.tar, level 11 row 2, advanced
silesia.tar, level 12 row 1, advanced streaming, 4513603
silesia.tar, level 12 row 2, advanced streaming, 4514569
silesia.tar, level 13, advanced streaming, 4502955
silesia.tar, level 16, advanced streaming, 4356834
silesia.tar, level 19, advanced streaming, 4264388
silesia.tar, level 16, advanced streaming, 4356970
silesia.tar, level 19, advanced streaming, 4264314
silesia.tar, no source size, advanced streaming, 4861421
silesia.tar, long distance mode, advanced streaming, 4847753
silesia.tar, multithreaded, advanced streaming, 4861507
@ -1163,7 +1163,7 @@ github.tar, level 16 with dict dms, advanced
github.tar, level 16 with dict dds, advanced streaming, 33544
github.tar, level 16 with dict copy, advanced streaming, 33639
github.tar, level 16 with dict load, advanced streaming, 39353
github.tar, level 19, advanced streaming, 32837
github.tar, level 19, advanced streaming, 32820
github.tar, level 19 with dict, advanced streaming, 32895
github.tar, level 19 with dict dms, advanced streaming, 32672
github.tar, level 19 with dict dds, advanced streaming, 32672
@ -1179,7 +1179,7 @@ github.tar, small hash log, advanced
github.tar, small chain log, advanced streaming, 41669
github.tar, explicit params, advanced streaming, 41227
github.tar, uncompressed literals, advanced streaming, 41122
github.tar, uncompressed literals optimal, advanced streaming, 35388
github.tar, uncompressed literals optimal, advanced streaming, 35494
github.tar, huffman literals, advanced streaming, 38800
github.tar, multithreaded with advanced params, advanced streaming, 41122
silesia, level -5, old streaming, 6882505
@ -1194,11 +1194,11 @@ silesia, level 6, old stre
silesia, level 7, old streaming, 4567203
silesia, level 9, old streaming, 4543311
silesia, level 13, old streaming, 4493990
silesia, level 16, old streaming, 4360251
silesia, level 19, old streaming, 4283236
silesia, level 16, old streaming, 4360072
silesia, level 19, old streaming, 4282792
silesia, no source size, old streaming, 4849515
silesia, uncompressed literals, old streaming, 4849551
silesia, uncompressed literals optimal, old streaming, 4283236
silesia, uncompressed literals optimal, old streaming, 4282792
silesia, huffman literals, old streaming, 6183403
silesia.tar, level -5, old streaming, 6982759
silesia.tar, level -3, old streaming, 6641283
@ -1212,11 +1212,11 @@ silesia.tar, level 6, old stre
silesia.tar, level 7, old streaming, 4576830
silesia.tar, level 9, old streaming, 4552590
silesia.tar, level 13, old streaming, 4502955
silesia.tar, level 16, old streaming, 4356834
silesia.tar, level 19, old streaming, 4264388
silesia.tar, level 16, old streaming, 4356970
silesia.tar, level 19, old streaming, 4264314
silesia.tar, no source size, old streaming, 4861421
silesia.tar, uncompressed literals, old streaming, 4861425
silesia.tar, uncompressed literals optimal, old streaming, 4264388
silesia.tar, uncompressed literals optimal, old streaming, 4264314
silesia.tar, huffman literals, old streaming, 6190795
github, level -5, old streaming, 205285
github, level -5 with dict, old streaming, 46718
@ -1277,12 +1277,12 @@ github.tar, level 13, old stre
github.tar, level 13 with dict, old streaming, 38726
github.tar, level 16, old streaming, 40255
github.tar, level 16 with dict, old streaming, 33639
github.tar, level 19, old streaming, 32837
github.tar, level 19, old streaming, 32820
github.tar, level 19 with dict, old streaming, 32895
github.tar, no source size, old streaming, 38438
github.tar, no source size with dict, old streaming, 38000
github.tar, uncompressed literals, old streaming, 38441
github.tar, uncompressed literals optimal, old streaming, 32837
github.tar, uncompressed literals optimal, old streaming, 32820
github.tar, huffman literals, old streaming, 42465
silesia, level -5, old streaming advanced, 6882505
silesia, level -3, old streaming advanced, 6568376
@ -1296,8 +1296,8 @@ silesia, level 6, old stre
silesia, level 7, old streaming advanced, 4567203
silesia, level 9, old streaming advanced, 4543311
silesia, level 13, old streaming advanced, 4493990
silesia, level 16, old streaming advanced, 4360251
silesia, level 19, old streaming advanced, 4283236
silesia, level 16, old streaming advanced, 4360072
silesia, level 19, old streaming advanced, 4282792
silesia, no source size, old streaming advanced, 4849515
silesia, long distance mode, old streaming advanced, 4849551
silesia, multithreaded, old streaming advanced, 4849551
@ -1307,7 +1307,7 @@ silesia, small hash log, old stre
silesia, small chain log, old streaming advanced, 4912199
silesia, explicit params, old streaming advanced, 4795884
silesia, uncompressed literals, old streaming advanced, 4849551
silesia, uncompressed literals optimal, old streaming advanced, 4283236
silesia, uncompressed literals optimal, old streaming advanced, 4282792
silesia, huffman literals, old streaming advanced, 6183403
silesia, multithreaded with advanced params, old streaming advanced, 4849551
silesia.tar, level -5, old streaming advanced, 6982759
@ -1322,8 +1322,8 @@ silesia.tar, level 6, old stre
silesia.tar, level 7, old streaming advanced, 4576830
silesia.tar, level 9, old streaming advanced, 4552590
silesia.tar, level 13, old streaming advanced, 4502955
silesia.tar, level 16, old streaming advanced, 4356834
silesia.tar, level 19, old streaming advanced, 4264388
silesia.tar, level 16, old streaming advanced, 4356970
silesia.tar, level 19, old streaming advanced, 4264314
silesia.tar, no source size, old streaming advanced, 4861421
silesia.tar, long distance mode, old streaming advanced, 4861425
silesia.tar, multithreaded, old streaming advanced, 4861425
@ -1333,7 +1333,7 @@ silesia.tar, small hash log, old stre
silesia.tar, small chain log, old streaming advanced, 4917019
silesia.tar, explicit params, old streaming advanced, 4807403
silesia.tar, uncompressed literals, old streaming advanced, 4861425
silesia.tar, uncompressed literals optimal, old streaming advanced, 4264388
silesia.tar, uncompressed literals optimal, old streaming advanced, 4264314
silesia.tar, huffman literals, old streaming advanced, 6190795
silesia.tar, multithreaded with advanced params, old streaming advanced, 4861425
github, level -5, old streaming advanced, 216734
@ -1403,7 +1403,7 @@ github.tar, level 13, old stre
github.tar, level 13 with dict, old streaming advanced, 35807
github.tar, level 16, old streaming advanced, 40255
github.tar, level 16 with dict, old streaming advanced, 38736
github.tar, level 19, old streaming advanced, 32837
github.tar, level 19, old streaming advanced, 32820
github.tar, level 19 with dict, old streaming advanced, 32876
github.tar, no source size, old streaming advanced, 38438
github.tar, no source size with dict, old streaming advanced, 38015
@ -1415,7 +1415,7 @@ github.tar, small hash log, old stre
github.tar, small chain log, old streaming advanced, 41669
github.tar, explicit params, old streaming advanced, 41227
github.tar, uncompressed literals, old streaming advanced, 38441
github.tar, uncompressed literals optimal, old streaming advanced, 32837
github.tar, uncompressed literals optimal, old streaming advanced, 32820
github.tar, huffman literals, old streaming advanced, 42465
github.tar, multithreaded with advanced params, old streaming advanced, 38441
github, level -5 with dict, old streaming cdict, 46718

1 Data Config Method Total compressed size
11 silesia.tar level 7 compress simple 4576828
12 silesia.tar level 9 compress simple 4552584
13 silesia.tar level 13 compress simple 4502955
14 silesia.tar level 16 compress simple 4356834 4356970
15 silesia.tar level 19 compress simple 4264388 4264314
16 silesia.tar uncompressed literals compress simple 4861423
17 silesia.tar uncompressed literals optimal compress simple 4264388 4264314
18 silesia.tar huffman literals compress simple 6186042
19 github.tar level -5 compress simple 46856
20 github.tar level -3 compress simple 43754
29 github.tar level 9 compress simple 36767
30 github.tar level 13 compress simple 35501
31 github.tar level 16 compress simple 40255
32 github.tar level 19 compress simple 32837 32820
33 github.tar uncompressed literals compress simple 38441
34 github.tar uncompressed literals optimal compress simple 32837 32820
35 github.tar huffman literals compress simple 42490
36 silesia level -5 compress cctx 6737607
37 silesia level -3 compress cctx 6444677
45 silesia level 7 compress cctx 4567203
46 silesia level 9 compress cctx 4543311
47 silesia level 13 compress cctx 4493990
48 silesia level 16 compress cctx 4360251 4360072
49 silesia level 19 compress cctx 4283236 4282792
50 silesia long distance mode compress cctx 4849551
51 silesia multithreaded compress cctx 4849551
52 silesia multithreaded long distance mode compress cctx 4849551
55 silesia small chain log compress cctx 4912199
56 silesia explicit params compress cctx 4794480
57 silesia uncompressed literals compress cctx 4849551
58 silesia uncompressed literals optimal compress cctx 4283236 4282792
59 silesia huffman literals compress cctx 6178460
60 silesia multithreaded with advanced params compress cctx 4849551
61 github level -5 compress cctx 205285
109 silesia level 7 zstdcli 4567251
110 silesia level 9 zstdcli 4543359
111 silesia level 13 zstdcli 4494038
112 silesia level 16 zstdcli 4360299 4360120
113 silesia level 19 zstdcli 4283284 4282840
114 silesia long distance mode zstdcli 4840807
115 silesia multithreaded zstdcli 4849599
116 silesia multithreaded long distance mode zstdcli 4840807
119 silesia small chain log zstdcli 4912247
120 silesia explicit params zstdcli 4795856
121 silesia uncompressed literals zstdcli 5128030
122 silesia uncompressed literals optimal zstdcli 4317944 4318048
123 silesia huffman literals zstdcli 5326317
124 silesia multithreaded with advanced params zstdcli 5128030
125 silesia.tar level -5 zstdcli 6738934
134 silesia.tar level 7 zstdcli 4578883
135 silesia.tar level 9 zstdcli 4553498
136 silesia.tar level 13 zstdcli 4502959
137 silesia.tar level 16 zstdcli 4356838 4356974
138 silesia.tar level 19 zstdcli 4264392 4264318
139 silesia.tar no source size zstdcli 4861507
140 silesia.tar long distance mode zstdcli 4853225
141 silesia.tar multithreaded zstdcli 4861511
213 github.tar level 13 with dict zstdcli 38730
214 github.tar level 16 zstdcli 40259
215 github.tar level 16 with dict zstdcli 33643
216 github.tar level 19 zstdcli 32841 32824
217 github.tar level 19 with dict zstdcli 32899
218 github.tar no source size zstdcli 38442
219 github.tar no source size with dict zstdcli 38004
225 github.tar small chain log zstdcli 41673
226 github.tar explicit params zstdcli 41227
227 github.tar uncompressed literals zstdcli 41126
228 github.tar uncompressed literals optimal zstdcli 35392 35498
229 github.tar huffman literals zstdcli 38781
230 github.tar multithreaded with advanced params zstdcli 41126
231 silesia level -5 advanced one pass 6737607
248 silesia level 12 row 1 advanced one pass 4503117
249 silesia level 12 row 2 advanced one pass 4505152
250 silesia level 13 advanced one pass 4493990
251 silesia level 16 advanced one pass 4360251 4360072
252 silesia level 19 advanced one pass 4283236 4282792
253 silesia no source size advanced one pass 4849551
254 silesia long distance mode advanced one pass 4840738
255 silesia multithreaded advanced one pass 4849551
259 silesia small chain log advanced one pass 4912199
260 silesia explicit params advanced one pass 4795856
261 silesia uncompressed literals advanced one pass 5127982
262 silesia uncompressed literals optimal advanced one pass 4317896 4318000
263 silesia huffman literals advanced one pass 5326269
264 silesia multithreaded with advanced params advanced one pass 5127982
265 silesia.tar level -5 advanced one pass 6738593
282 silesia.tar level 12 row 1 advanced one pass 4513603
283 silesia.tar level 12 row 2 advanced one pass 4514568
284 silesia.tar level 13 advanced one pass 4502955
285 silesia.tar level 16 advanced one pass 4356834 4356970
286 silesia.tar level 19 advanced one pass 4264388 4264314
287 silesia.tar no source size advanced one pass 4861423
288 silesia.tar long distance mode advanced one pass 4847753
289 silesia.tar multithreaded advanced one pass 4861507
527 github.tar level 16 with dict dds advanced one pass 33544
528 github.tar level 16 with dict copy advanced one pass 33639
529 github.tar level 16 with dict load advanced one pass 39353
530 github.tar level 19 advanced one pass 32837 32820
531 github.tar level 19 with dict advanced one pass 32895
532 github.tar level 19 with dict dms advanced one pass 32672
533 github.tar level 19 with dict dds advanced one pass 32672
543 github.tar small chain log advanced one pass 41669
544 github.tar explicit params advanced one pass 41227
545 github.tar uncompressed literals advanced one pass 41122
546 github.tar uncompressed literals optimal advanced one pass 35388 35494
547 github.tar huffman literals advanced one pass 38777
548 github.tar multithreaded with advanced params advanced one pass 41122
549 silesia level -5 advanced one pass small out 6737607
566 silesia level 12 row 1 advanced one pass small out 4503117
567 silesia level 12 row 2 advanced one pass small out 4505152
568 silesia level 13 advanced one pass small out 4493990
569 silesia level 16 advanced one pass small out 4360251 4360072
570 silesia level 19 advanced one pass small out 4283236 4282792
571 silesia no source size advanced one pass small out 4849551
572 silesia long distance mode advanced one pass small out 4840738
573 silesia multithreaded advanced one pass small out 4849551
577 silesia small chain log advanced one pass small out 4912199
578 silesia explicit params advanced one pass small out 4795856
579 silesia uncompressed literals advanced one pass small out 5127982
580 silesia uncompressed literals optimal advanced one pass small out 4317896 4318000
581 silesia huffman literals advanced one pass small out 5326269
582 silesia multithreaded with advanced params advanced one pass small out 5127982
583 silesia.tar level -5 advanced one pass small out 6738593
600 silesia.tar level 12 row 1 advanced one pass small out 4513603
601 silesia.tar level 12 row 2 advanced one pass small out 4514568
602 silesia.tar level 13 advanced one pass small out 4502955
603 silesia.tar level 16 advanced one pass small out 4356834 4356970
604 silesia.tar level 19 advanced one pass small out 4264388 4264314
605 silesia.tar no source size advanced one pass small out 4861423
606 silesia.tar long distance mode advanced one pass small out 4847753
607 silesia.tar multithreaded advanced one pass small out 4861507
845 github.tar level 16 with dict dds advanced one pass small out 33544
846 github.tar level 16 with dict copy advanced one pass small out 33639
847 github.tar level 16 with dict load advanced one pass small out 39353
848 github.tar level 19 advanced one pass small out 32837 32820
849 github.tar level 19 with dict advanced one pass small out 32895
850 github.tar level 19 with dict dms advanced one pass small out 32672
851 github.tar level 19 with dict dds advanced one pass small out 32672
861 github.tar small chain log advanced one pass small out 41669
862 github.tar explicit params advanced one pass small out 41227
863 github.tar uncompressed literals advanced one pass small out 41122
864 github.tar uncompressed literals optimal advanced one pass small out 35388 35494
865 github.tar huffman literals advanced one pass small out 38777
866 github.tar multithreaded with advanced params advanced one pass small out 41122
867 silesia level -5 advanced streaming 6882505
884 silesia level 12 row 1 advanced streaming 4503117
885 silesia level 12 row 2 advanced streaming 4505152
886 silesia level 13 advanced streaming 4493990
887 silesia level 16 advanced streaming 4360251 4360072
888 silesia level 19 advanced streaming 4283236 4282792
889 silesia no source size advanced streaming 4849515
890 silesia long distance mode advanced streaming 4840738
891 silesia multithreaded advanced streaming 4849551
895 silesia small chain log advanced streaming 4912199
896 silesia explicit params advanced streaming 4795884
897 silesia uncompressed literals advanced streaming 5127982
898 silesia uncompressed literals optimal advanced streaming 4317896 4318000
899 silesia huffman literals advanced streaming 5331171
900 silesia multithreaded with advanced params advanced streaming 5127982
901 silesia.tar level -5 advanced streaming 6982759
918 silesia.tar level 12 row 1 advanced streaming 4513603
919 silesia.tar level 12 row 2 advanced streaming 4514569
920 silesia.tar level 13 advanced streaming 4502955
921 silesia.tar level 16 advanced streaming 4356834 4356970
922 silesia.tar level 19 advanced streaming 4264388 4264314
923 silesia.tar no source size advanced streaming 4861421
924 silesia.tar long distance mode advanced streaming 4847753
925 silesia.tar multithreaded advanced streaming 4861507
1163 github.tar level 16 with dict dds advanced streaming 33544
1164 github.tar level 16 with dict copy advanced streaming 33639
1165 github.tar level 16 with dict load advanced streaming 39353
1166 github.tar level 19 advanced streaming 32837 32820
1167 github.tar level 19 with dict advanced streaming 32895
1168 github.tar level 19 with dict dms advanced streaming 32672
1169 github.tar level 19 with dict dds advanced streaming 32672
1179 github.tar small chain log advanced streaming 41669
1180 github.tar explicit params advanced streaming 41227
1181 github.tar uncompressed literals advanced streaming 41122
1182 github.tar uncompressed literals optimal advanced streaming 35388 35494
1183 github.tar huffman literals advanced streaming 38800
1184 github.tar multithreaded with advanced params advanced streaming 41122
1185 silesia level -5 old streaming 6882505
1194 silesia level 7 old streaming 4567203
1195 silesia level 9 old streaming 4543311
1196 silesia level 13 old streaming 4493990
1197 silesia level 16 old streaming 4360251 4360072
1198 silesia level 19 old streaming 4283236 4282792
1199 silesia no source size old streaming 4849515
1200 silesia uncompressed literals old streaming 4849551
1201 silesia uncompressed literals optimal old streaming 4283236 4282792
1202 silesia huffman literals old streaming 6183403
1203 silesia.tar level -5 old streaming 6982759
1204 silesia.tar level -3 old streaming 6641283
1212 silesia.tar level 7 old streaming 4576830
1213 silesia.tar level 9 old streaming 4552590
1214 silesia.tar level 13 old streaming 4502955
1215 silesia.tar level 16 old streaming 4356834 4356970
1216 silesia.tar level 19 old streaming 4264388 4264314
1217 silesia.tar no source size old streaming 4861421
1218 silesia.tar uncompressed literals old streaming 4861425
1219 silesia.tar uncompressed literals optimal old streaming 4264388 4264314
1220 silesia.tar huffman literals old streaming 6190795
1221 github level -5 old streaming 205285
1222 github level -5 with dict old streaming 46718
1277 github.tar level 13 with dict old streaming 38726
1278 github.tar level 16 old streaming 40255
1279 github.tar level 16 with dict old streaming 33639
1280 github.tar level 19 old streaming 32837 32820
1281 github.tar level 19 with dict old streaming 32895
1282 github.tar no source size old streaming 38438
1283 github.tar no source size with dict old streaming 38000
1284 github.tar uncompressed literals old streaming 38441
1285 github.tar uncompressed literals optimal old streaming 32837 32820
1286 github.tar huffman literals old streaming 42465
1287 silesia level -5 old streaming advanced 6882505
1288 silesia level -3 old streaming advanced 6568376
1296 silesia level 7 old streaming advanced 4567203
1297 silesia level 9 old streaming advanced 4543311
1298 silesia level 13 old streaming advanced 4493990
1299 silesia level 16 old streaming advanced 4360251 4360072
1300 silesia level 19 old streaming advanced 4283236 4282792
1301 silesia no source size old streaming advanced 4849515
1302 silesia long distance mode old streaming advanced 4849551
1303 silesia multithreaded old streaming advanced 4849551
1307 silesia small chain log old streaming advanced 4912199
1308 silesia explicit params old streaming advanced 4795884
1309 silesia uncompressed literals old streaming advanced 4849551
1310 silesia uncompressed literals optimal old streaming advanced 4283236 4282792
1311 silesia huffman literals old streaming advanced 6183403
1312 silesia multithreaded with advanced params old streaming advanced 4849551
1313 silesia.tar level -5 old streaming advanced 6982759
1322 silesia.tar level 7 old streaming advanced 4576830
1323 silesia.tar level 9 old streaming advanced 4552590
1324 silesia.tar level 13 old streaming advanced 4502955
1325 silesia.tar level 16 old streaming advanced 4356834 4356970
1326 silesia.tar level 19 old streaming advanced 4264388 4264314
1327 silesia.tar no source size old streaming advanced 4861421
1328 silesia.tar long distance mode old streaming advanced 4861425
1329 silesia.tar multithreaded old streaming advanced 4861425
1333 silesia.tar small chain log old streaming advanced 4917019
1334 silesia.tar explicit params old streaming advanced 4807403
1335 silesia.tar uncompressed literals old streaming advanced 4861425
1336 silesia.tar uncompressed literals optimal old streaming advanced 4264388 4264314
1337 silesia.tar huffman literals old streaming advanced 6190795
1338 silesia.tar multithreaded with advanced params old streaming advanced 4861425
1339 github level -5 old streaming advanced 216734
1403 github.tar level 13 with dict old streaming advanced 35807
1404 github.tar level 16 old streaming advanced 40255
1405 github.tar level 16 with dict old streaming advanced 38736
1406 github.tar level 19 old streaming advanced 32837 32820
1407 github.tar level 19 with dict old streaming advanced 32876
1408 github.tar no source size old streaming advanced 38438
1409 github.tar no source size with dict old streaming advanced 38015
1415 github.tar small chain log old streaming advanced 41669
1416 github.tar explicit params old streaming advanced 41227
1417 github.tar uncompressed literals old streaming advanced 38441
1418 github.tar uncompressed literals optimal old streaming advanced 32837 32820
1419 github.tar huffman literals old streaming advanced 42465
1420 github.tar multithreaded with advanced params old streaming advanced 38441
1421 github level -5 with dict old streaming cdict 46718