facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

removed new huffman depth heuristic

Browse Source 
results are now identical to before this PR
This commit is contained in:
Yann Collet 2022-01-26 15:22:06 -08:00
parent a66e8bb437
commit 8b46895588
2 changed files with 15 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
lib/compress/huf_compress.c
View File

@ -143,7 +143,10 @@ typedef struct {
S16 norm[HUF_TABLELOG_MAX+1]; S16 norm[HUF_TABLELOG_MAX+1];
} HUF_CompressWeightsWksp; } HUF_CompressWeightsWksp;
static size_t HUF_compressWeights(void* dst, size_t dstSize, const void* weightTable, size_t wtSize, void* workspace, size_t workspaceSize) static size_t
HUF_compressWeights(void* dst, size_t dstSize,
const void* weightTable, size_t wtSize,
void* workspace, size_t workspaceSize)
{ {
BYTE* const ostart = (BYTE*) dst; BYTE* const ostart = (BYTE*) dst;
BYTE* op = ostart; BYTE* op = ostart;
@ -731,18 +734,6 @@ static void HUF_buildCTableFromTree(HUF_CElt* CTable, nodeElt const* huffNode, i
CTable[0] = maxNbBits; CTable[0] = maxNbBits;
} }
static size_t
HUF_nbSymbolsTooLarge(const nodeElt* hnodes, U32 maxSymbolValue, U32 maxNbBits)
{
size_t nbSTL = 0;
int s = (int)maxSymbolValue;
for ( ; s > 0; s-- ) {
if (hnodes[s].nbBits > maxNbBits) nbSTL++;
else break;
}
return nbSTL;
}
size_t size_t
HUF_buildCTable_wksp(HUF_CElt* CTable, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, HUF_buildCTable_wksp(HUF_CElt* CTable, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
void* workSpace, size_t wkspSize) void* workSpace, size_t wkspSize)
@ -771,16 +762,6 @@ HUF_buildCTable_wksp(HUF_CElt* CTable, const unsigned* count, U32 maxSymbolValue
nonNullRank = HUF_buildTree(huffNode, maxSymbolValue); nonNullRank = HUF_buildTree(huffNode, maxSymbolValue);
/* determine and enforce maxTableLog */ /* determine and enforce maxTableLog */
/* Loosen target when maxNbBits is already within limits.
* A harsh rebalancing can be bad for compression ratio
* while a mild one tends to be better */
while (maxNbBits < HUF_TABLELOG_DEFAULT) {
size_t const nbSTL = HUF_nbSymbolsTooLarge(huffNode, maxSymbolValue, maxNbBits);
#define HUF_NB_NODES_TO_FIX_MAX 32
if (nbSTL < HUF_NB_NODES_TO_FIX_MAX) /* heuristic */
break;
maxNbBits++;
}
maxNbBits = HUF_setMaxHeight(huffNode, (U32)nonNullRank, maxNbBits); maxNbBits = HUF_setMaxHeight(huffNode, (U32)nonNullRank, maxNbBits);
if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */

22
tests/regression/results.csv
View File

@ -51,7 +51,7 @@ silesia, long distance mode, compress
silesia, multithreaded, compress cctx, 4842075 silesia, multithreaded, compress cctx, 4842075
silesia, multithreaded long distance mode, compress cctx, 4842075 silesia, multithreaded long distance mode, compress cctx, 4842075
silesia, small window log, compress cctx, 7082951 silesia, small window log, compress cctx, 7082951
silesia, small hash log, compress cctx, 6525895 silesia, small hash log, compress cctx, 6526141
silesia, small chain log, compress cctx, 4912197 silesia, small chain log, compress cctx, 4912197
silesia, explicit params, compress cctx, 4794052 silesia, explicit params, compress cctx, 4794052
silesia, uncompressed literals, compress cctx, 4842075 silesia, uncompressed literals, compress cctx, 4842075
@ -115,7 +115,7 @@ silesia, long distance mode, zstdcli,
silesia, multithreaded, zstdcli, 4842123 silesia, multithreaded, zstdcli, 4842123
silesia, multithreaded long distance mode, zstdcli, 4833785 silesia, multithreaded long distance mode, zstdcli, 4833785
silesia, small window log, zstdcli, 7095048 silesia, small window log, zstdcli, 7095048
silesia, small hash log, zstdcli, 6525943 silesia, small hash log, zstdcli, 6526189
silesia, small chain log, zstdcli, 4912245 silesia, small chain log, zstdcli, 4912245
silesia, explicit params, zstdcli, 4795432 silesia, explicit params, zstdcli, 4795432
silesia, uncompressed literals, zstdcli, 5120614 silesia, uncompressed literals, zstdcli, 5120614
@ -141,7 +141,7 @@ silesia.tar, long distance mode, zstdcli,
silesia.tar, multithreaded, zstdcli, 4854164 silesia.tar, multithreaded, zstdcli, 4854164
silesia.tar, multithreaded long distance mode, zstdcli, 4845745 silesia.tar, multithreaded long distance mode, zstdcli, 4845745
silesia.tar, small window log, zstdcli, 7100701 silesia.tar, small window log, zstdcli, 7100701
silesia.tar, small hash log, zstdcli, 6529041 silesia.tar, small hash log, zstdcli, 6529289
silesia.tar, small chain log, zstdcli, 4917022 silesia.tar, small chain log, zstdcli, 4917022
silesia.tar, explicit params, zstdcli, 4820713 silesia.tar, explicit params, zstdcli, 4820713
silesia.tar, uncompressed literals, zstdcli, 5122571 silesia.tar, uncompressed literals, zstdcli, 5122571
@ -255,7 +255,7 @@ silesia, long distance mode, advanced
silesia, multithreaded, advanced one pass, 4842075 silesia, multithreaded, advanced one pass, 4842075
silesia, multithreaded long distance mode, advanced one pass, 4833737 silesia, multithreaded long distance mode, advanced one pass, 4833737
silesia, small window log, advanced one pass, 7095000 silesia, small window log, advanced one pass, 7095000
silesia, small hash log, advanced one pass, 6525895 silesia, small hash log, advanced one pass, 6526141
silesia, small chain log, advanced one pass, 4912197 silesia, small chain log, advanced one pass, 4912197
silesia, explicit params, advanced one pass, 4795432 silesia, explicit params, advanced one pass, 4795432
silesia, uncompressed literals, advanced one pass, 5120566 silesia, uncompressed literals, advanced one pass, 5120566
@ -289,7 +289,7 @@ silesia.tar, long distance mode, advanced
silesia.tar, multithreaded, advanced one pass, 4854160 silesia.tar, multithreaded, advanced one pass, 4854160
silesia.tar, multithreaded long distance mode, advanced one pass, 4845741 silesia.tar, multithreaded long distance mode, advanced one pass, 4845741
silesia.tar, small window log, advanced one pass, 7100655 silesia.tar, small window log, advanced one pass, 7100655
silesia.tar, small hash log, advanced one pass, 6528983 silesia.tar, small hash log, advanced one pass, 6529231
silesia.tar, small chain log, advanced one pass, 4917041 silesia.tar, small chain log, advanced one pass, 4917041
silesia.tar, explicit params, advanced one pass, 4806855 silesia.tar, explicit params, advanced one pass, 4806855
silesia.tar, uncompressed literals, advanced one pass, 5122473 silesia.tar, uncompressed literals, advanced one pass, 5122473
@ -573,7 +573,7 @@ silesia, long distance mode, advanced
silesia, multithreaded, advanced one pass small out, 4842075 silesia, multithreaded, advanced one pass small out, 4842075
silesia, multithreaded long distance mode, advanced one pass small out, 4833737 silesia, multithreaded long distance mode, advanced one pass small out, 4833737
silesia, small window log, advanced one pass small out, 7095000 silesia, small window log, advanced one pass small out, 7095000
silesia, small hash log, advanced one pass small out, 6525895 silesia, small hash log, advanced one pass small out, 6526141
silesia, small chain log, advanced one pass small out, 4912197 silesia, small chain log, advanced one pass small out, 4912197
silesia, explicit params, advanced one pass small out, 4795432 silesia, explicit params, advanced one pass small out, 4795432
silesia, uncompressed literals, advanced one pass small out, 5120566 silesia, uncompressed literals, advanced one pass small out, 5120566
@ -607,7 +607,7 @@ silesia.tar, long distance mode, advanced
silesia.tar, multithreaded, advanced one pass small out, 4854160 silesia.tar, multithreaded, advanced one pass small out, 4854160
silesia.tar, multithreaded long distance mode, advanced one pass small out, 4845741 silesia.tar, multithreaded long distance mode, advanced one pass small out, 4845741
silesia.tar, small window log, advanced one pass small out, 7100655 silesia.tar, small window log, advanced one pass small out, 7100655
silesia.tar, small hash log, advanced one pass small out, 6528983 silesia.tar, small hash log, advanced one pass small out, 6529231
silesia.tar, small chain log, advanced one pass small out, 4917041 silesia.tar, small chain log, advanced one pass small out, 4917041
silesia.tar, explicit params, advanced one pass small out, 4806855 silesia.tar, explicit params, advanced one pass small out, 4806855
silesia.tar, uncompressed literals, advanced one pass small out, 5122473 silesia.tar, uncompressed literals, advanced one pass small out, 5122473
@ -891,7 +891,7 @@ silesia, long distance mode, advanced
silesia, multithreaded, advanced streaming, 4842075 silesia, multithreaded, advanced streaming, 4842075
silesia, multithreaded long distance mode, advanced streaming, 4833737 silesia, multithreaded long distance mode, advanced streaming, 4833737
silesia, small window log, advanced streaming, 7111103 silesia, small window log, advanced streaming, 7111103
silesia, small hash log, advanced streaming, 6525895 silesia, small hash log, advanced streaming, 6526141
silesia, small chain log, advanced streaming, 4912197 silesia, small chain log, advanced streaming, 4912197
silesia, explicit params, advanced streaming, 4795452 silesia, explicit params, advanced streaming, 4795452
silesia, uncompressed literals, advanced streaming, 5120566 silesia, uncompressed literals, advanced streaming, 5120566
@ -925,7 +925,7 @@ silesia.tar, long distance mode, advanced
silesia.tar, multithreaded, advanced streaming, 4854160 silesia.tar, multithreaded, advanced streaming, 4854160
silesia.tar, multithreaded long distance mode, advanced streaming, 4845741 silesia.tar, multithreaded long distance mode, advanced streaming, 4845741
silesia.tar, small window log, advanced streaming, 7117559 silesia.tar, small window log, advanced streaming, 7117559
silesia.tar, small hash log, advanced streaming, 6528986 silesia.tar, small hash log, advanced streaming, 6529234
silesia.tar, small chain log, advanced streaming, 4917021 silesia.tar, small chain log, advanced streaming, 4917021
silesia.tar, explicit params, advanced streaming, 4806873 silesia.tar, explicit params, advanced streaming, 4806873
silesia.tar, uncompressed literals, advanced streaming, 5127423 silesia.tar, uncompressed literals, advanced streaming, 5127423
@ -1303,7 +1303,7 @@ silesia, long distance mode, old stre
silesia, multithreaded, old streaming advanced, 4842075 silesia, multithreaded, old streaming advanced, 4842075
silesia, multithreaded long distance mode, old streaming advanced, 4842075 silesia, multithreaded long distance mode, old streaming advanced, 4842075
silesia, small window log, old streaming advanced, 7111103 silesia, small window log, old streaming advanced, 7111103
silesia, small hash log, old streaming advanced, 6525895 silesia, small hash log, old streaming advanced, 6526141
silesia, small chain log, old streaming advanced, 4912197 silesia, small chain log, old streaming advanced, 4912197
silesia, explicit params, old streaming advanced, 4795452 silesia, explicit params, old streaming advanced, 4795452
silesia, uncompressed literals, old streaming advanced, 4842075 silesia, uncompressed literals, old streaming advanced, 4842075
@ -1329,7 +1329,7 @@ silesia.tar, long distance mode, old stre
silesia.tar, multithreaded, old streaming advanced, 4859271 silesia.tar, multithreaded, old streaming advanced, 4859271
silesia.tar, multithreaded long distance mode, old streaming advanced, 4859271 silesia.tar, multithreaded long distance mode, old streaming advanced, 4859271
silesia.tar, small window log, old streaming advanced, 7117562 silesia.tar, small window log, old streaming advanced, 7117562
silesia.tar, small hash log, old streaming advanced, 6528986 silesia.tar, small hash log, old streaming advanced, 6529234
silesia.tar, small chain log, old streaming advanced, 4917021 silesia.tar, small chain log, old streaming advanced, 4917021
silesia.tar, explicit params, old streaming advanced, 4806873 silesia.tar, explicit params, old streaming advanced, 4806873
silesia.tar, uncompressed literals, old streaming advanced, 4859271 silesia.tar, uncompressed literals, old streaming advanced, 4859271

1 Data Config Method Total compressed size
51 silesia multithreaded compress cctx 4842075
52 silesia multithreaded long distance mode compress cctx 4842075
53 silesia small window log compress cctx 7082951
54 silesia small hash log compress cctx 6525895 6526141
55 silesia small chain log compress cctx 4912197
56 silesia explicit params compress cctx 4794052
57 silesia uncompressed literals compress cctx 4842075
115 silesia multithreaded zstdcli 4842123
116 silesia multithreaded long distance mode zstdcli 4833785
117 silesia small window log zstdcli 7095048
118 silesia small hash log zstdcli 6525943 6526189
119 silesia small chain log zstdcli 4912245
120 silesia explicit params zstdcli 4795432
121 silesia uncompressed literals zstdcli 5120614
141 silesia.tar multithreaded zstdcli 4854164
142 silesia.tar multithreaded long distance mode zstdcli 4845745
143 silesia.tar small window log zstdcli 7100701
144 silesia.tar small hash log zstdcli 6529041 6529289
145 silesia.tar small chain log zstdcli 4917022
146 silesia.tar explicit params zstdcli 4820713
147 silesia.tar uncompressed literals zstdcli 5122571
255 silesia multithreaded advanced one pass 4842075
256 silesia multithreaded long distance mode advanced one pass 4833737
257 silesia small window log advanced one pass 7095000
258 silesia small hash log advanced one pass 6525895 6526141
259 silesia small chain log advanced one pass 4912197
260 silesia explicit params advanced one pass 4795432
261 silesia uncompressed literals advanced one pass 5120566
289 silesia.tar multithreaded advanced one pass 4854160
290 silesia.tar multithreaded long distance mode advanced one pass 4845741
291 silesia.tar small window log advanced one pass 7100655
292 silesia.tar small hash log advanced one pass 6528983 6529231
293 silesia.tar small chain log advanced one pass 4917041
294 silesia.tar explicit params advanced one pass 4806855
295 silesia.tar uncompressed literals advanced one pass 5122473
573 silesia multithreaded advanced one pass small out 4842075
574 silesia multithreaded long distance mode advanced one pass small out 4833737
575 silesia small window log advanced one pass small out 7095000
576 silesia small hash log advanced one pass small out 6525895 6526141
577 silesia small chain log advanced one pass small out 4912197
578 silesia explicit params advanced one pass small out 4795432
579 silesia uncompressed literals advanced one pass small out 5120566
607 silesia.tar multithreaded advanced one pass small out 4854160
608 silesia.tar multithreaded long distance mode advanced one pass small out 4845741
609 silesia.tar small window log advanced one pass small out 7100655
610 silesia.tar small hash log advanced one pass small out 6528983 6529231
611 silesia.tar small chain log advanced one pass small out 4917041
612 silesia.tar explicit params advanced one pass small out 4806855
613 silesia.tar uncompressed literals advanced one pass small out 5122473
891 silesia multithreaded advanced streaming 4842075
892 silesia multithreaded long distance mode advanced streaming 4833737
893 silesia small window log advanced streaming 7111103
894 silesia small hash log advanced streaming 6525895 6526141
895 silesia small chain log advanced streaming 4912197
896 silesia explicit params advanced streaming 4795452
897 silesia uncompressed literals advanced streaming 5120566
925 silesia.tar multithreaded advanced streaming 4854160
926 silesia.tar multithreaded long distance mode advanced streaming 4845741
927 silesia.tar small window log advanced streaming 7117559
928 silesia.tar small hash log advanced streaming 6528986 6529234
929 silesia.tar small chain log advanced streaming 4917021
930 silesia.tar explicit params advanced streaming 4806873
931 silesia.tar uncompressed literals advanced streaming 5127423
1303 silesia multithreaded old streaming advanced 4842075
1304 silesia multithreaded long distance mode old streaming advanced 4842075
1305 silesia small window log old streaming advanced 7111103
1306 silesia small hash log old streaming advanced 6525895 6526141
1307 silesia small chain log old streaming advanced 4912197
1308 silesia explicit params old streaming advanced 4795452
1309 silesia uncompressed literals old streaming advanced 4842075
1329 silesia.tar multithreaded old streaming advanced 4859271
1330 silesia.tar multithreaded long distance mode old streaming advanced 4859271
1331 silesia.tar small window log old streaming advanced 7117562
1332 silesia.tar small hash log old streaming advanced 6528986 6529234
1333 silesia.tar small chain log old streaming advanced 4917021
1334 silesia.tar explicit params old streaming advanced 4806873
1335 silesia.tar uncompressed literals old streaming advanced 4859271