facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11

dev
Przemyslaw Skibinski 2016-12-13 12:19:19 +01:00
parent 671f28d1e5 086985316f
commit 334b8962b0
21 changed files with 202 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
NEWS
View File

@ -1,16 +1,19 @@
v1.1.2
Improved : faster decompression speed at ultra compression settings and in 32-bits mode
API : streaming : decompression : changed : implicit reset on starting new frames without init
API : experimental : added : dictID retrieval functions
API : zbuff : changed : prototypes now generate deprecation warnings
lib : improved : faster decompression speed at ultra compression settings and 32-bits mode
lib : changed : only public ZSTD_ symbols are now exposed
lib : changed : reduced usage of stack memory
lib : fixed : several corner case bugs, by Nick Terrell
cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski
cli : new : preserve file attributes
cli : new : added zstdless and zstdgrep tools
cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd)
cli : fixed : zstdcat
API : changed : zbuff prototypes now generate deprecation warnings
API : changed : streaming decompression implicit reset on starting new frame
API : added experimental : dictID retrieval functions
zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski
Changed : zbuff source files moved to lib/deprecated
Changed : reduced stack memory use
install : better compatibility with FreeBSD, by Dimitry Andric
source tree : changed : zbuff source files moved to lib/deprecated
v1.1.1
New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption

10
contrib/pzstd/Options.cpp
View File

@ -22,19 +22,13 @@
defined(__CYGWIN__)
#include <io.h> /* _isatty */
#define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
#else
#if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || \
defined(_POSIX_SOURCE) || \
(defined(__APPLE__) && \
defined( \
__MACH__)) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ \
*/
#elif defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) || \
defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */
#include <unistd.h> /* isatty */
#define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
#else
#define IS_CONSOLE(stdStream) 0
#endif
#endif
namespace pzstd {

21
examples/simple_compression.c
View File

@ -8,9 +8,9 @@
#include <stdlib.h> // malloc, exit
#include <stdio.h> // fprintf, perror
#include <string.h> // strerror
#include <stdlib.h> // malloc, free, exit
#include <stdio.h> // fprintf, perror, fopen, etc.
#include <string.h> // strlen, strcat, memset, strerror
#include <errno.h> // errno
#include <sys/stat.h> // stat
#include <zstd.h> // presumes zstd library is installed
@ -45,13 +45,18 @@ static void* malloc_orDie(size_t size)
static void* loadFile_orDie(const char* fileName, size_t* size)
{
off_t const buffSize = fsize_orDie(fileName);
off_t const fileSize = fsize_orDie(fileName);
size_t const buffSize = (size_t)fileSize;
if ((off_t)buffSize < fileSize) { /* narrowcast overflow */
fprintf(stderr, "%s : filesize too large \n", fileName);
exit(4);
}
FILE* const inFile = fopen_orDie(fileName, "rb");
void* const buffer = malloc_orDie(buffSize);
size_t const readSize = fread(buffer, 1, buffSize, inFile);
if (readSize != (size_t)buffSize) {
fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
exit(4);
exit(5);
}
fclose(inFile); /* can't fail, read only */
*size = buffSize;
@ -65,11 +70,11 @@ static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSi
size_t const wSize = fwrite(buff, 1, buffSize, oFile);
if (wSize != (size_t)buffSize) {
fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno));
exit(5);
exit(6);
}
if (fclose(oFile)) {
perror(fileName);
exit(6);
exit(7);
}
}
@ -84,7 +89,7 @@ static void compress_orDie(const char* fname, const char* oname)
size_t const cSize = ZSTD_compress(cBuff, cBuffSize, fBuff, fSize, 1);
if (ZSTD_isError(cSize)) {
fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize));
exit(7);
exit(8);
}
saveFile_orDie(oname, cBuff, cSize);

10
examples/streaming_compression.c
View File

@ -7,11 +7,9 @@
*/
#include <stdlib.h> // malloc, exit
#include <stdio.h> // fprintf, perror, feof
#include <string.h> // strerror
#include <errno.h> // errno
#define ZSTD_STATIC_LINKING_ONLY // streaming API defined as "experimental" for the time being
#include <stdlib.h> // malloc, free, exit
#include <stdio.h> // fprintf, perror, feof, fopen, etc.
#include <string.h> // strlen, memset, strcat
#include <zstd.h> // presumes zstd library is installed
@ -82,7 +80,7 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
toRead = ZSTD_compressStream(cstream, &output , &input); /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */
if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); }
if (toRead > buffInSize) toRead = buffInSize; /* Safely handle when `buffInSize` is manually changed to a smaller value */
if (toRead > buffInSize) toRead = buffInSize; /* Safely handle case when `buffInSize` is manually changed to a value < ZSTD_CStreamInSize()*/
fwrite_orDie(buffOut, output.pos, fout);
}
}

16
lib/Makefile
View File

@ -47,9 +47,9 @@ ifeq ($(shell uname), Darwin)
SHARED_EXT = dylib
SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
SONAME_FLAGS = -install_name $(PREFIX)/lib/$@.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
SONAME_FLAGS = -install_name $(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
else
SONAME_FLAGS = -Wl,-soname=$@.$(SHARED_EXT).$(LIBVER_MAJOR)
SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR)
SHARED_EXT = so
SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR)
SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
@ -107,15 +107,15 @@ libzstd.pc: libzstd.pc.in
install: libzstd.a libzstd libzstd.pc
@install -d -m 755 $(DESTDIR)$(LIBDIR)/pkgconfig/ $(DESTDIR)$(INCLUDEDIR)/
@install -m 755 libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER)
@install -m 755 libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
@cp -a libzstd.$(SHARED_EXT_MAJOR) $(DESTDIR)$(LIBDIR)
@cp -a libzstd.$(SHARED_EXT) $(DESTDIR)$(LIBDIR)
@cp -a libzstd.pc $(DESTDIR)$(LIBDIR)/pkgconfig/
@install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a
@install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h
@install -m 644 common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h
@install -m 644 deprecated/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h # prototypes generate deprecation warnings
@install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h
@install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)
@install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)
@install -m 644 common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)
@install -m 644 deprecated/zbuff.h $(DESTDIR)$(INCLUDEDIR) # prototypes generate deprecation warnings
@install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)
@echo zstd static and shared library installed
uninstall:

2
lib/common/zstd_internal.h
View File

@ -147,7 +147,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
/*! ZSTD_wildcopy() :
* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
#define WILDCOPY_OVERLENGTH 8
MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, size_t length)
MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;

29
lib/compress/zstd_compress.c
View File

@ -33,6 +33,7 @@ typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZS
/*-*************************************
* Helper functions
***************************************/
#define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; }
size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; }
@ -148,6 +149,14 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
}
/** ZSTD_cycleLog() :
* condition for correct operation : hashLog > 1 */
static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
{
U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2);
return hashLog - btScale;
}
/** ZSTD_adjustCParams() :
optimize `cPar` for a given input (`srcSize` and `dictSize`).
mostly downsizing to reduce memory consumption and initialization.
@ -166,9 +175,9 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u
if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
} }
if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog;
{ U32 const btPlus = (cPar.strategy == ZSTD_btlazy2) | (cPar.strategy == ZSTD_btopt) | (cPar.strategy == ZSTD_btopt2);
U32 const maxChainLog = cPar.windowLog+btPlus;
if (cPar.chainLog > maxChainLog) cPar.chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */
{ U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy);
if (cycleLog > cPar.windowLog) cPar.chainLog -= (cycleLog - cPar.windowLog);
}
if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
@ -2274,16 +2283,16 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
if (remaining < blockSize) blockSize = remaining;
/* preemptive overflow correction */
if (cctx->lowLimit > (1<<30)) {
U32 const btplus = (cctx->params.cParams.strategy == ZSTD_btlazy2) | (cctx->params.cParams.strategy == ZSTD_btopt) | (cctx->params.cParams.strategy == ZSTD_btopt2);
U32 const chainMask = (1 << (cctx->params.cParams.chainLog - btplus)) - 1;
U32 const supLog = MAX(cctx->params.cParams.chainLog, 17 /* blockSize */);
U32 const newLowLimit = (cctx->lowLimit & chainMask) + (1 << supLog); /* preserve position % chainSize, ensure current-repcode doesn't underflow */
U32 const correction = cctx->lowLimit - newLowLimit;
if (cctx->lowLimit > (2U<<30)) {
U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->params.cParams.hashLog, cctx->params.cParams.strategy)) - 1;
U32 const current = (U32)(ip - cctx->base);
U32 const newCurrent = (current & cycleMask) + (1 << cctx->params.cParams.windowLog);
U32 const correction = current - newCurrent;
ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_64 <= 30);
ZSTD_reduceIndex(cctx, correction);
cctx->base += correction;
cctx->dictBase += correction;
cctx->lowLimit = newLowLimit;
cctx->lowLimit -= correction;
cctx->dictLimit -= correction;
if (cctx->nextToUpdate < correction) cctx->nextToUpdate = 0;
else cctx->nextToUpdate -= correction;

12
lib/decompress/zstd_decompress.c
View File

@ -963,13 +963,13 @@ size_t ZSTD_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
if (op > oend_w) {
if (op > oend_w || sequence.matchLength < MINMATCH) {
U32 i;
for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i];
return sequenceLength;
}
} }
/* Requirement: op <= oend_w */
/* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */
/* match within prefix */
if (sequence.offset < 8) {
@ -997,7 +997,7 @@ size_t ZSTD_execSequence(BYTE* op,
}
while (op < oMatchEnd) *op++ = *match++;
} else {
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
}
return sequenceLength;
}
@ -1183,13 +1183,13 @@ size_t ZSTD_execSequenceLong(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
if (op > oend_w) {
if (op > oend_w || sequence.matchLength < MINMATCH) {
U32 i;
for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i];
return sequenceLength;
}
} }
/* Requirement: op <= oend_w */
/* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */
#endif
/* match within prefix */
@ -1218,7 +1218,7 @@ size_t ZSTD_execSequenceLong(BYTE* op,
}
while (op < oMatchEnd) *op++ = *match++;
} else {
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
}
return sequenceLength;
}

6
lib/legacy/zstd_v01.c
View File

@ -1354,7 +1354,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
@ -1803,7 +1803,7 @@ static size_t ZSTD_execSequence(BYTE* op,
} else { ZSTD_copy8(op, match); }
op += 8; match += 8;
if (endMatch > oend-12)
if (endMatch > oend-(16-MINMATCH))
{
if (op < oend-8)
{
@ -1814,7 +1814,7 @@ static size_t ZSTD_execSequence(BYTE* op,
while (op<endMatch) *op++ = *match++;
}
else
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
/* restore, in case of overlap */
if (overlapRisk) memcpy(endMatch, saved, qutt);

6
lib/legacy/zstd_v02.c
View File

@ -2808,7 +2808,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
@ -3206,7 +3206,7 @@ static size_t ZSTD_execSequence(BYTE* op,
}
op += 8; match += 8;
if (oMatchEnd > oend-12)
if (oMatchEnd > oend-(16-MINMATCH))
{
if (op < oend_8)
{
@ -3218,7 +3218,7 @@ static size_t ZSTD_execSequence(BYTE* op,
}
else
{
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
}
}

6
lib/legacy/zstd_v03.c
View File

@ -2449,7 +2449,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
@ -2847,7 +2847,7 @@ static size_t ZSTD_execSequence(BYTE* op,
}
op += 8; match += 8;
if (oMatchEnd > oend-12)
if (oMatchEnd > oend-(16-MINMATCH))
{
if (op < oend_8)
{
@ -2859,7 +2859,7 @@ static size_t ZSTD_execSequence(BYTE* op,
}
else
{
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
}
}

8
lib/legacy/zstd_v04.c
View File

@ -487,7 +487,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
@ -3107,7 +3107,7 @@ static size_t ZSTD_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
if (op > oend_8) {
if (op > oend_8 || sequence.matchLength < MINMATCH) {
while (op < oMatchEnd) *op++ = *match++;
return sequenceLength;
}
@ -3134,7 +3134,7 @@ static size_t ZSTD_execSequence(BYTE* op,
}
op += 8; match += 8;
if (oMatchEnd > oend-12)
if (oMatchEnd > oend-(16-MINMATCH))
{
if (op < oend_8)
{
@ -3146,7 +3146,7 @@ static size_t ZSTD_execSequence(BYTE* op,
}
else
{
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
}
return sequenceLength;
}

8
lib/legacy/zstd_v05.c
View File

@ -509,7 +509,7 @@ static void ZSTDv05_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
/*! ZSTDv05_wildcopy() :
* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
MEM_STATIC void ZSTDv05_wildcopy(void* dst, const void* src, size_t length)
MEM_STATIC void ZSTDv05_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
@ -3325,7 +3325,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
if (op > oend_8) {
if (op > oend_8 || sequence.matchLength < MINMATCH) {
while (op < oMatchEnd) *op++ = *match++;
return sequenceLength;
}
@ -3348,7 +3348,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
}
op += 8; match += 8;
if (oMatchEnd > oend-12) {
if (oMatchEnd > oend-(16-MINMATCH)) {
if (op < oend_8) {
ZSTDv05_wildcopy(op, match, oend_8 - op);
match += oend_8 - op;
@ -3357,7 +3357,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
while (op < oMatchEnd)
*op++ = *match++;
} else {
ZSTDv05_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
ZSTDv05_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
}
return sequenceLength;
}

6
lib/legacy/zstd_v06.c
View File

@ -537,7 +537,7 @@ static void ZSTDv06_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
/*! ZSTDv06_wildcopy() :
* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
#define WILDCOPY_OVERLENGTH 8
MEM_STATIC void ZSTDv06_wildcopy(void* dst, const void* src, size_t length)
MEM_STATIC void ZSTDv06_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
@ -3470,7 +3470,7 @@ size_t ZSTDv06_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
if (op > oend_8) {
if (op > oend_8 || sequence.matchLength < MINMATCH) {
while (op < oMatchEnd) *op++ = *match++;
return sequenceLength;
}
@ -3503,7 +3503,7 @@ size_t ZSTDv06_execSequence(BYTE* op,
}
while (op < oMatchEnd) *op++ = *match++;
} else {
ZSTDv06_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
ZSTDv06_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
}
return sequenceLength;
}

6
lib/legacy/zstd_v07.c
View File

@ -2845,7 +2845,7 @@ static void ZSTDv07_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
/*! ZSTDv07_wildcopy() :
* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
#define WILDCOPY_OVERLENGTH 8
MEM_STATIC void ZSTDv07_wildcopy(void* dst, const void* src, size_t length)
MEM_STATIC void ZSTDv07_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
@ -3693,7 +3693,7 @@ size_t ZSTDv07_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
if (op > oend_w) {
if (op > oend_w || sequence.matchLength < MINMATCH) {
while (op < oMatchEnd) *op++ = *match++;
return sequenceLength;
}
@ -3726,7 +3726,7 @@ size_t ZSTDv07_execSequence(BYTE* op,
}
while (op < oMatchEnd) *op++ = *match++;
} else {
ZSTDv07_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
ZSTDv07_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
}
return sequenceLength;
}

22
programs/util.h
View File

@ -25,7 +25,7 @@ extern "C" {
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#if _MSC_VER <= 1800 /* (1800 = Visual Studio 2013) */
#define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */
# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */
#endif
#endif
@ -49,11 +49,11 @@ extern "C" {
#include <sys/types.h> /* stat, utime */
#include <sys/stat.h> /* stat */
#if defined(_MSC_VER)
#include <sys/utime.h> /* utime */
#include <io.h> /* _chmod */
# include <sys/utime.h> /* utime */
# include <io.h> /* _chmod */
#else
#include <unistd.h> /* chown, stat */
#include <utime.h> /* utime */
# include <unistd.h> /* chown, stat */
# include <utime.h> /* utime */
#endif
#include <time.h> /* time */
#include <errno.h>
@ -98,7 +98,7 @@ extern "C" {
# define SET_HIGH_PRIORITY /* disabled */
# endif
# define UTIL_sleep(s) sleep(s)
# if defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 199309L)
# if defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 199309L))
# define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); }
# else
# define UTIL_sleepMilli(milli) /* disabled */
@ -408,16 +408,15 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i
{
size_t pos;
unsigned i, nbFiles;
char *bufend, *buf;
char* buf = (char*)malloc(LIST_SIZE_INCREASE);
char* bufend = buf + LIST_SIZE_INCREASE;
const char** fileTable;
buf = (char*)malloc(LIST_SIZE_INCREASE);
if (!buf) return NULL;
bufend = buf + LIST_SIZE_INCREASE;
for (i=0, pos=0, nbFiles=0; i<inputNamesNb; i++) {
if (!UTIL_isDirectory(inputNames[i])) {
size_t len = strlen(inputNames[i]);
size_t const len = strlen(inputNames[i]);
if (buf + pos + len >= bufend) {
ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
buf = (char*)UTIL_realloc(buf, newListSize);
@ -439,8 +438,7 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i
fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*));
if (!fileTable) { free(buf); return NULL; }
for (i=0, pos=0; i<nbFiles; i++)
{
for (i=0, pos=0; i<nbFiles; i++) {
fileTable[i] = buf + pos;
pos += strlen(fileTable[i]) + 1;
}

3
programs/zstdcli.c
View File

@ -43,7 +43,8 @@
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
# include <io.h> /* _isatty */
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
#elif defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */
#elif defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) || \
defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */
# include <unistd.h> /* isatty */
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
#else

1
tests/.gitignore vendored
View File

@ -11,6 +11,7 @@ datagen
paramgrill
paramgrill32
roundTripCrash
longmatch
# Tmp test directory
zstdtest

10
tests/Makefile
View File

@ -124,6 +124,9 @@ datagen : $(PRGDIR)/datagen.c datagencli.c
roundTripCrash : $(ZSTD_FILES) roundTripCrash.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
longmatch : $(ZSTD_FILES) longmatch.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
namespaceTest:
if $(CC) namespaceTest.c ../lib/common/xxhash.c -o $@ ; then echo compilation should fail; exit 1 ; fi
$(RM) $@
@ -140,7 +143,7 @@ clean:
fullbench-lib$(EXT) fullbench-dll$(EXT) \
fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
zstreamtest$(EXT) zstreamtest32$(EXT) \
datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT)
datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT)
@echo Cleaning completed
@ -180,7 +183,7 @@ zstd-playTests: datagen
file $(ZSTD)
ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
test: test-zstd test-fullbench test-fuzzer test-zstream
test: test-zstd test-fullbench test-fuzzer test-zstream test-longmatch
test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32
@ -237,4 +240,7 @@ test-zstream: zstreamtest
test-zstream32: zstreamtest32
$(QEMU_SYS) ./zstreamtest32 $(ZSTREAM_TESTTIME)
test-longmatch: longmatch
$(QEMU_SYS) ./longmatch
endif

90
tests/longmatch.c Normal file
View File

@ -0,0 +1,90 @@
#include <stdio.h>
#include <stddef.h>
#include <stdlib.h>
#include <stdint.h>
#include "mem.h"
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
int compress(ZSTD_CStream *ctx, ZSTD_outBuffer out, const void *data, size_t size) {
ZSTD_inBuffer in = { data, size, 0 };
while (in.pos < in.size) {
ZSTD_outBuffer tmp = out;
const size_t rc = ZSTD_compressStream(ctx, &tmp, &in);
if (ZSTD_isError(rc)) {
return 1;
}
}
{
ZSTD_outBuffer tmp = out;
const size_t rc = ZSTD_flushStream(ctx, &tmp);
if (rc != 0) { return 1; }
}
return 0;
}
int main(int argc, const char** argv) {
ZSTD_CStream *ctx;
ZSTD_parameters params;
size_t rc;
unsigned windowLog;
(void)argc;
(void)argv;
/* Create stream */
ctx = ZSTD_createCStream();
if (!ctx) { return 1; }
/* Set parameters */
memset(&params, 0, sizeof(params));
params.cParams.windowLog = 18;
params.cParams.chainLog = 13;
params.cParams.hashLog = 14;
params.cParams.searchLog = 1;
params.cParams.searchLength = 7;
params.cParams.targetLength = 16;
params.cParams.strategy = ZSTD_fast;
windowLog = params.cParams.windowLog;
/* Initialize stream */
rc = ZSTD_initCStream_advanced(ctx, NULL, 0, params, 0);
if (ZSTD_isError(rc)) { return 2; }
{
U64 compressed = 0;
const U64 toCompress = ((U64)1) << 33;
const size_t size = 1 << windowLog;
size_t pos = 0;
char *srcBuffer = (char*) malloc(1 << windowLog);
char *dstBuffer = (char*) malloc(ZSTD_compressBound(1 << windowLog));
ZSTD_outBuffer out = { dstBuffer, ZSTD_compressBound(1 << windowLog), 0 };
const char match[] = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
const size_t randomData = (1 << windowLog) - 2*sizeof(match);
size_t i;
printf("\n === Long Match Test === \n");
printf("Creating random data to produce long matches \n");
for (i = 0; i < sizeof(match); ++i) {
srcBuffer[i] = match[i];
}
for (i = 0; i < randomData; ++i) {
srcBuffer[sizeof(match) + i] = (char)(rand() & 0xFF);
}
for (i = 0; i < sizeof(match); ++i) {
srcBuffer[sizeof(match) + randomData + i] = match[i];
}
printf("Compressing, trying to generate a segfault \n");
if (compress(ctx, out, srcBuffer, size)) {
return 1;
}
compressed += size;
while (compressed < toCompress) {
const size_t block = rand() % (size - pos + 1);
if (pos == size) { pos = 0; }
if (compress(ctx, out, srcBuffer + pos, block)) {
return 1;
}
pos += block;
compressed += block;
}
printf("Compression completed successfully (no error triggered)\n");
free(srcBuffer);
free(dstBuffer);
}
return 0;
}

10
tests/playTests.sh
View File

@ -31,8 +31,10 @@ case "$OS" in
;;
esac
case "$OSTYPE" in
darwin*) MD5SUM="md5 -r" ;;
UNAME=$(uname)
case "$UNAME" in
Darwin) MD5SUM="md5 -r" ;;
FreeBSD) MD5SUM="gmd5sum" ;;
*) MD5SUM="md5sum" ;;
esac
@ -228,8 +230,8 @@ cp ../programs/*.h dirTestDict
$MD5SUM dirTestDict/* > tmph1
$ZSTD -f --rm dirTestDict/* -D tmpDictC
$ZSTD -d --rm dirTestDict/*.zst -D tmpDictC # note : use internal checksum by default
case "$OSTYPE" in
darwin*) $ECHO "md5sum -c not supported on OS-X : test skipped" ;; # not compatible with OS-X's md5
case "$UNAME" in
Darwin) $ECHO "md5sum -c not supported on OS-X : test skipped" ;; # not compatible with OS-X's md5
*) $MD5SUM -c tmph1 ;;
esac
rm -rf dirTestDict