facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/Cyan4973/dev' into Other

Browse Source
This commit is contained in:
inikep 2016-08-16 16:45:50 +02:00
commit e4aef6c342
6 changed files with 39 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -120,7 +120,7 @@ asan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address" $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address"
msan: clean msan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory" # datagen.c used to fail this test for no obvious reason $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer" # datagen.c fails this test for no obvious reason
asan32: clean asan32: clean
$(MAKE) -C $(PRGDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address" $(MAKE) -C $(PRGDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"

2
NEWS
View File

@ -1,6 +1,8 @@
v0.8.1 v0.8.1
Changed : --ultra now enables levels beyond 19
Changed : -i# now selects benchmark time in second Changed : -i# now selects benchmark time in second
Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell
Fixed : speed regression on specific patterns (#272)
v0.8.0 v0.8.0
Improved : better speed on clang and gcc -O2, thanks to Eric Biggers Improved : better speed on clang and gcc -O2, thanks to Eric Biggers

BIN
images/Dspeed4.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 9.7 KiB

After

Width:  |  Height:  |  Size: 8.8 KiB

24
lib/dictBuilder/zdict.c
View File

@ -611,18 +611,17 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
{ const BYTE* codePtr = seqStorePtr->llCode; { const BYTE* codePtr = seqStorePtr->llCode;
U32 u; U32 u;
for (u=0; u<nbSeq; u++) litlengthCount[codePtr[u]]++; for (u=0; u<nbSeq; u++) litlengthCount[codePtr[u]]++;
} } }
/* rep offsets */ if (nbSeq >= 2) { /* rep offsets */
{ const seqDef* const seq = seqStorePtr->sequences; const seqDef* const seq = seqStorePtr->sequencesStart;
U32 offset1 = seq[0].offset - 3; U32 offset1 = seq[0].offset - 3;
U32 offset2 = seq[1].offset - 3; U32 offset2 = seq[1].offset - 3;
if (offset1 >= MAXREPOFFSET) offset1 = 0; if (offset1 >= MAXREPOFFSET) offset1 = 0;
if (offset2 >= MAXREPOFFSET) offset2 = 0; if (offset2 >= MAXREPOFFSET) offset2 = 0;
repOffsets[offset1] += 3; repOffsets[offset1] += 3;
repOffsets[offset2] += 1; repOffsets[offset2] += 1;
} } } }
}
} }
/* /*
@ -676,7 +675,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
short matchLengthNCount[MaxML+1]; short matchLengthNCount[MaxML+1];
U32 litLengthCount[MaxLL+1]; U32 litLengthCount[MaxLL+1];
short litLengthNCount[MaxLL+1]; short litLengthNCount[MaxLL+1];
U32 repOffset[MAXREPOFFSET] = { 0 }; U32 repOffset[MAXREPOFFSET];
offsetCount_t bestRepOffset[ZSTD_REP_NUM+1]; offsetCount_t bestRepOffset[ZSTD_REP_NUM+1];
EStats_ress_t esr; EStats_ress_t esr;
ZSTD_parameters params; ZSTD_parameters params;
@ -701,6 +700,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
for (u=0; u<=offcodeMax; u++) offcodeCount[u]=1; for (u=0; u<=offcodeMax; u++) offcodeCount[u]=1;
for (u=0; u<=MaxML; u++) matchLengthCount[u]=1; for (u=0; u<=MaxML; u++) matchLengthCount[u]=1;
for (u=0; u<=MaxLL; u++) litLengthCount[u]=1; for (u=0; u<=MaxLL; u++) litLengthCount[u]=1;
memset(repOffset, 0, sizeof(repOffset));
repOffset[1] = repOffset[4] = repOffset[8] = 1; repOffset[1] = repOffset[4] = repOffset[8] = 1;
memset(bestRepOffset, 0, sizeof(bestRepOffset)); memset(bestRepOffset, 0, sizeof(bestRepOffset));
if (compressionLevel==0) compressionLevel=g_compressionLevel_default; if (compressionLevel==0) compressionLevel=g_compressionLevel_default;

2
programs/playTests.sh
View File

@ -47,7 +47,7 @@ $ECHO "\n**** simple tests **** "
$ZSTD -f tmp # trivial compression case, creates tmp.zst $ZSTD -f tmp # trivial compression case, creates tmp.zst
$ZSTD -df tmp.zst # trivial decompression case (overwrites tmp) $ZSTD -df tmp.zst # trivial decompression case (overwrites tmp)
$ECHO "test : too large compression level (must fail)" $ECHO "test : too large compression level (must fail)"
$ZSTD -99 tmp && die "too large compression level undetected" $ZSTD -99 -f tmp # too large compression level, automatic sized down
$ECHO "test : compress to stdout" $ECHO "test : compress to stdout"
$ZSTD tmp -c > tmpCompressed $ZSTD tmp -c > tmpCompressed
$ZSTD tmp --stdout > tmpCompressed # long command format $ZSTD tmp --stdout > tmpCompressed # long command format

35
programs/zstdcli.c
View File

@ -35,6 +35,10 @@
# define ZSTDCLI_CLEVEL_DEFAULT 3 # define ZSTDCLI_CLEVEL_DEFAULT 3
#endif #endif
#ifndef ZSTDCLI_CLEVEL_MAX
# define ZSTDCLI_CLEVEL_MAX 19
#endif
/*-************************************ /*-************************************
* Includes * Includes
@ -88,6 +92,8 @@
#define MB *(1 <<20) #define MB *(1 <<20)
#define GB *(1U<<30) #define GB *(1U<<30)
#define DEFAULT_DISPLAY_LEVEL 2
static const char* g_defaultDictName = "dictionary"; static const char* g_defaultDictName = "dictionary";
static const unsigned g_defaultMaxDictSize = 110 KB; static const unsigned g_defaultMaxDictSize = 110 KB;
static const int g_defaultDictCLevel = 5; static const int g_defaultDictCLevel = 5;
@ -100,7 +106,7 @@ static const unsigned g_defaultSelectivityLevel = 9;
#define DISPLAY(...) fprintf(displayOut, __VA_ARGS__) #define DISPLAY(...) fprintf(displayOut, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static FILE* displayOut; static FILE* displayOut;
static unsigned displayLevel = 2; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */ static unsigned displayLevel = DEFAULT_DISPLAY_LEVEL; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */
/*-************************************ /*-************************************
@ -115,7 +121,7 @@ static int usage(const char* programName)
DISPLAY( " with no FILE, or when FILE is - , read standard input\n"); DISPLAY( " with no FILE, or when FILE is - , read standard input\n");
DISPLAY( "Arguments :\n"); DISPLAY( "Arguments :\n");
#ifndef ZSTD_NOCOMPRESS #ifndef ZSTD_NOCOMPRESS
DISPLAY( " -# : # compression level (1-%u, default:%u) \n", ZSTD_maxCLevel(), ZSTDCLI_CLEVEL_DEFAULT); DISPLAY( " -# : # compression level (1-%u, default:%u) \n", ZSTDCLI_CLEVEL_MAX, ZSTDCLI_CLEVEL_DEFAULT);
#endif #endif
#ifndef ZSTD_NODECOMPRESS #ifndef ZSTD_NODECOMPRESS
DISPLAY( " -d : decompression \n"); DISPLAY( " -d : decompression \n");
@ -136,14 +142,14 @@ static int usage_advanced(const char* programName)
DISPLAY( "\n"); DISPLAY( "\n");
DISPLAY( "Advanced arguments :\n"); DISPLAY( "Advanced arguments :\n");
DISPLAY( " -V : display Version number and exit\n"); DISPLAY( " -V : display Version number and exit\n");
DISPLAY( " -v : verbose mode\n"); DISPLAY( " -v : verbose mode; specify multiple times to increase log level (default:%d)\n", DEFAULT_DISPLAY_LEVEL);
DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
DISPLAY( " -c : force write to standard output, even if it is the console\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n");
#ifdef UTIL_HAS_CREATEFILELIST #ifdef UTIL_HAS_CREATEFILELIST
DISPLAY( " -r : operate recursively on directories\n"); DISPLAY( " -r : operate recursively on directories\n");
#endif #endif
#ifndef ZSTD_NOCOMPRESS #ifndef ZSTD_NOCOMPRESS
DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n"); DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n"); DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n");
DISPLAY( "--[no-]check : integrity check (default:enabled)\n"); DISPLAY( "--[no-]check : integrity check (default:enabled)\n");
#endif #endif
@ -215,7 +221,8 @@ int main(int argCount, const char** argv)
nextArgumentIsOutFileName=0, nextArgumentIsOutFileName=0,
nextArgumentIsMaxDict=0, nextArgumentIsMaxDict=0,
nextArgumentIsDictID=0, nextArgumentIsDictID=0,
nextArgumentIsFile=0; nextArgumentIsFile=0,
ultra=0;
int cLevel = ZSTDCLI_CLEVEL_DEFAULT; int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
int cLevelLast = 1; int cLevelLast = 1;
unsigned recursive = 0; unsigned recursive = 0;
@ -268,7 +275,7 @@ int main(int argCount, const char** argv)
if (!strcmp(argument, "--verbose")) { displayLevel++; continue; } if (!strcmp(argument, "--verbose")) { displayLevel++; continue; }
if (!strcmp(argument, "--quiet")) { displayLevel--; continue; } if (!strcmp(argument, "--quiet")) { displayLevel--; continue; }
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); continue; } if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); continue; }
if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; } if (!strcmp(argument, "--ultra")) { ultra=1; FIO_setMaxWLog(0); continue; }
if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; } if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; } if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; } if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
@ -298,10 +305,7 @@ int main(int argCount, const char** argv)
#ifndef ZSTD_NOCOMPRESS #ifndef ZSTD_NOCOMPRESS
/* compression Level */ /* compression Level */
if ((*argument>='0') && (*argument<='9')) { if ((*argument>='0') && (*argument<='9')) {
cLevel = readU32FromChar(&argument); dictCLevel = cLevel = readU32FromChar(&argument);
dictCLevel = cLevel;
if (dictCLevel > ZSTD_maxCLevel())
CLEAN_RETURN(badusage(programName));
continue; continue;
} }
#endif #endif
@ -317,7 +321,7 @@ int main(int argCount, const char** argv)
case 'd': decode=1; argument++; break; case 'd': decode=1; argument++; break;
/* Force stdout, even if stdout==console */ /* Force stdout, even if stdout==console */
case 'c': forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); argument++; break; case 'c': forceStdout=1; outFileName=stdoutmark; argument++; break;
/* Use file content as dictionary */ /* Use file content as dictionary */
case 'D': nextEntryIsDictionary = 1; argument++; break; case 'D': nextEntryIsDictionary = 1; argument++; break;
@ -488,7 +492,14 @@ int main(int argCount, const char** argv)
CLEAN_RETURN(filenameIdx); CLEAN_RETURN(filenameIdx);
} }
/* No warning message in pipe mode (stdin + stdout) or multiple mode */ /* check compression level limits */
{ int const maxCLevel = ultra ? ZSTD_maxCLevel() : ZSTDCLI_CLEVEL_MAX;
if (cLevel > maxCLevel) {
DISPLAYLEVEL(2, "Warning : compression level higher than max, reduced to %i \n", maxCLevel);
cLevel = maxCLevel;
} }
/* No warning message in pipe mode (stdin + stdout) or multi-files mode */
if (!strcmp(filenameTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1; if (!strcmp(filenameTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1;
if ((filenameIdx>1) & (displayLevel==2)) displayLevel=1; if ((filenameIdx>1) & (displayLevel==2)) displayLevel=1;