diff --git a/.gitignore b/.gitignore index 2d3b6917..f0e2c9f1 100644 --- a/.gitignore +++ b/.gitignore @@ -34,4 +34,4 @@ _zstdbench/ zlib_wrapper/ # CMake -contrib/cmake/ +projects/cmake/ diff --git a/.travis.yml b/.travis.yml index 573a77e5..e6a32c25 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,34 +1,97 @@ language: c - -before_install: - - sudo apt-get update -qq - - sudo apt-get install -qq clang - - sudo apt-get install -qq g++-multilib - - sudo apt-get install -qq gcc-multilib - - sudo apt-get install -qq valgrind - -env: - - ZSTD_TRAVIS_CI_ENV=travis-install - - ZSTD_TRAVIS_CI_ENV=cmaketest - - ZSTD_TRAVIS_CI_ENV=clangtest - - ZSTD_TRAVIS_CI_ENV=gpptest - - ZSTD_TRAVIS_CI_ENV=gnu90test - - ZSTD_TRAVIS_CI_ENV=c99test - - ZSTD_TRAVIS_CI_ENV=gnu99test - - ZSTD_TRAVIS_CI_ENV=armtest-w-install - - ZSTD_TRAVIS_CI_ENV=test - - ZSTD_TRAVIS_CI_ENV="-C programs test32" - - ZSTD_TRAVIS_CI_ENV="-C programs test-zstd_nolegacy" - #- ZSTD_TRAVIS_CI_ENV="-C versionsTest" - - ZSTD_TRAVIS_CI_ENV=usan - - ZSTD_TRAVIS_CI_ENV=asan - - ZSTD_TRAVIS_CI_ENV=asan32 - - ZSTD_TRAVIS_CI_ENV="-C programs valgrindTest" - compiler: gcc - -script: - - make $ZSTD_TRAVIS_CI_ENV - matrix: fast_finish: true + include: + # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=travis-install + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=cmaketest + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=test + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM="-C programs test-zstd_nolegacy" + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=usan + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=asan + # Standard Ubuntu 12.04 LTS Server Edition 64 bit + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=clangtest + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gpptest + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu90test + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=c99test + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu99test + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=armtest-w-install + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs test32" + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C versionsTest" + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=asan32 + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs valgrindTest" + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu90test + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=c99test + #- os: linux + # sudo: required + # env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=zlibwrapper + # Ubuntu 14.04 LTS Server Edition 64 bit + - os: linux + dist: trusty + sudo: required + env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=ppctest-w-install + - os: linux + dist: trusty + sudo: required + env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=zlibwrapper + # OS X Mavericks + - os: osx + env: PLATFORM="OS X Mavericks" MAKE_PARAM=travis-install + - os: osx + env: PLATFORM="OS X Mavericks" MAKE_PARAM=gnu90test + - os: osx + env: PLATFORM="OS X Mavericks" MAKE_PARAM=test + exclude: + - compiler: gcc + +before_install: + - set -e + - | + if [ "$TRAVIS_OS_NAME" == "linux" ]; then + CAN_I_RUN_SUDO=$(sudo -n uptime 2>&1|grep "load"|wc -l) + echo "CAN_I_RUN_SUDO=$CAN_I_RUN_SUDO\n" + if [ ${CAN_I_RUN_SUDO} -gt 0 ]; then + sudo apt-get install -y -qq clang g++-multilib gcc-multilib valgrind + fi + fi + +script: + - make $MAKE_PARAM + diff --git a/Makefile b/Makefile index 8d7ee5a5..3c600f54 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,7 @@ PRGDIR = programs ZSTDDIR = lib +ZWRAPDIR = zlibWrapper # Define nul output ifneq (,$(filter Windows%,$(OS))) @@ -40,7 +41,7 @@ else VOID = /dev/null endif -.PHONY: default all zstdprogram clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan +.PHONY: default all zlibwrapper zstdprogram clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan default: zstdprogram @@ -51,12 +52,17 @@ all: zstdprogram: $(MAKE) -C $(PRGDIR) +zlibwrapper: + $(MAKE) -C $(ZSTDDIR) all + $(MAKE) -C $(ZWRAPDIR) all + test: $(MAKE) -C $(PRGDIR) $@ clean: @$(MAKE) -C $(ZSTDDIR) $@ > $(VOID) @$(MAKE) -C $(PRGDIR) $@ > $(VOID) + @$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID) @echo Cleaning completed @@ -118,10 +124,7 @@ armtest: clean # for Travis CI arminstall: clean - sudo apt-get install -q qemu - sudo apt-get install -q binfmt-support - sudo apt-get install -q qemu-user-static - sudo apt-get install -q gcc-arm-linux-gnueabi + sudo apt-get install -y -q qemu binfmt-support qemu-user-static gcc-arm-linux-gnueabi # for Travis CI armtest-w-install: clean arminstall armtest @@ -132,15 +135,23 @@ ppctest: clean # for Travis CI ppcinstall: clean - sudo apt-get install -q qemu - sudo apt-get install -q binfmt-support - sudo apt-get install -q qemu-user-static - sudo apt-get update -q - sudo apt-get install -q gcc-powerpc-linux-gnu # unfortunately, doesn't work on Travis CI (package not available) + sudo apt-get update -y -q + sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu # doesn't work with Ubuntu 12.04 # for Travis CI ppctest-w-install: clean ppcinstall ppctest +ppc64test: clean + $(MAKE) -C $(PRGDIR) datagen # use native, faster + $(MAKE) -C $(PRGDIR) test CC=powerpc64le-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static" + +ppc64install: clean + sudo apt-get update -y -q + sudo apt-get install -y -q qemu-ppc64le binfmt-support qemu-user-static gcc-powerpc64le-linux-gnu + update-binfmts --displ + +ppc64test-w-install: clean ppc64install ppc64test + usan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined" diff --git a/appveyor.yml b/appveyor.yml index 4721fa95..3d46ddc1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -80,7 +80,7 @@ build_script: ECHO *** && msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && - COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe projects\ + COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe programs\ ) test_script: @@ -91,12 +91,11 @@ test_script: ) - if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] ( CD programs && - SET ZSTD=..\projects\zstd.exe && - SET DATAGEN=..\projects\datagen.exe && - CALL tests\playTests.bat --test-large-data && + SET ZSTD=./zstd.exe && + sh -e playTests.sh --test-large-data && + fullbench.exe -i1 && + fullbench.exe -i1 -P0 && CD .. && - projects\fullbench.exe -i1 && - projects\fullbench.exe -i1 -P0 && projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% && projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% && diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index e0a8a82d..63e56aaf 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -253,6 +253,6 @@ typedef struct { const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq); - +int ZSTD_isSkipFrame(ZSTD_DCtx* dctx); #endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h index d9f619f2..596ca03e 100644 --- a/lib/common/zstd_static.h +++ b/lib/common/zstd_static.h @@ -51,7 +51,8 @@ extern "C" { /*-************************************* * Constants ***************************************/ -#define ZSTD_MAGICNUMBER 0xFD2FB527 /* v0.7 */ +#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */ +#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U /*-************************************* @@ -201,6 +202,7 @@ typedef struct { #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ static const size_t ZSTD_frameHeaderSize_min = 6; static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; +static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index d4ca550a..72456fd5 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -218,12 +218,13 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, break; } if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ + const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd); size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, - zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, + zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart), ip, neededInSize); if (ZSTD_isError(decodedSize)) return decodedSize; ip += neededInSize; - if (!decodedSize) break; /* this was just a header */ + if (!decodedSize && !isSkipFrame) break; /* this was just a header */ zbd->outEnd = zbd->outStart + decodedSize; zbd->stage = ZBUFFds_flush; break; @@ -243,12 +244,13 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */ /* decode loaded input */ - { size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, + { const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd); + size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, zbd->inBuff, neededInSize); if (ZSTD_isError(decodedSize)) return decodedSize; zbd->inPos = 0; /* input is consumed */ - if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */ + if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */ zbd->outEnd = zbd->outStart + decodedSize; zbd->stage = ZBUFFds_flush; // break; /* ZBUFFds_flush follows */ @@ -276,7 +278,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, *srcSizePtr = ip-istart; *dstCapacityPtr = op-ostart; { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbd->zd); - if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */ +// if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */ nextSrcSizeHint -= zbd->inPos; /* already loaded*/ return nextSrcSizeHint; } diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index b0831617..da00799c 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -101,7 +101,8 @@ static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } * Context management ***************************************************************/ typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, - ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock } ZSTD_dStage; + ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, + ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; struct ZSTD_DCtx_s { @@ -326,7 +327,15 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t const BYTE* ip = (const BYTE*)src; if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_min; - if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown); + if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) { + if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { + if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */ + fparamsPtr->frameContentSize = 0; + fparamsPtr->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; + return 0; + } + return ERROR(prefix_unknown); + } /* ensure there is enough `srcSize` to fully read/decode frame header */ { size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize); @@ -1021,6 +1030,11 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) return dctx->expected; } +int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) +{ + return dctx->stage == ZSTDds_skipFrame; +} + size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { /* Sanity check */ @@ -1032,6 +1046,12 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c { case ZSTDds_getFrameHeaderSize : if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ + if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { + memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); + dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_min; /* magic number + skippable frame length */ + dctx->stage = ZSTDds_decodeSkippableHeader; + return 0; + } dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min); if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); @@ -1089,6 +1109,17 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c dctx->previousDstEnd = (char*)dst + rSize; return rSize; } + case ZSTDds_decodeSkippableHeader: + { memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected); + dctx->expected = MEM_readLE32(dctx->headerBuffer + 4); + dctx->stage = ZSTDds_skipFrame; + return 0; + } + case ZSTDds_skipFrame: + { dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + } default: return ERROR(GENERIC); /* impossible */ } diff --git a/programs/Makefile b/programs/Makefile index 377e4947..1e8a7f26 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -38,7 +38,7 @@ MANDIR = $(PREFIX)/share/man/man1 ZSTDDIR = ../lib -CPPFLAGS= -I$(ZSTDDIR)/common -DXXH_NAMESPACE=ZSTD_ +CPPFLAGS= -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -DXXH_NAMESPACE=ZSTD_ CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) @@ -208,7 +208,7 @@ endif #------------------------------------------------------------------------ ifneq (,$(filter $(HOST_OS),MSYS POSIX)) zstd-playTests: datagen - ZSTD=$(ZSTD) ./tests/playTests.sh $(ZSTDRTTEST) + ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST) test: test-zstd test-fullbench test-fuzzer test-zbuff diff --git a/programs/dibio.h b/programs/dibio.h index fd1f21df..0ccec413 100644 --- a/programs/dibio.h +++ b/programs/dibio.h @@ -32,7 +32,7 @@ /*-************************************* * Dependencies ***************************************/ -#include "../lib/dictBuilder/zdict_static.h" /* ZDICT_params_t */ +#include "zdict_static.h" /* ZDICT_params_t */ /*-************************************* diff --git a/programs/fileio.c b/programs/fileio.c index 767d4c05..55e36cc5 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -689,7 +689,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) continue; } #endif - if (magic != ZSTD_MAGICNUMBER) { + if (((magic & 0xFFFFFFF0U) != ZSTD_MAGIC_SKIPPABLE_START) && (magic != ZSTD_MAGICNUMBER)) { if (g_overwrite) /* -df : pass-through mode */ return FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); else { diff --git a/programs/tests/playTests.sh b/programs/playTests.sh old mode 100755 new mode 100644 similarity index 93% rename from programs/tests/playTests.sh rename to programs/playTests.sh index 42abc3a8..46e2fb6a --- a/programs/tests/playTests.sh +++ b/programs/playTests.sh @@ -15,22 +15,30 @@ roundTripTest() { rm -f tmp1 tmp2 $ECHO "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d" - ./datagen $1 $p | md5sum > tmp1 - ./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d | md5sum > tmp2 + ./datagen $1 $p | $MD5SUM > tmp1 + ./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d | $MD5SUM > tmp2 diff -q tmp1 tmp2 } isWindows=false ECHO="echo" -if [[ "$OS" == "Windows"* ]]; then +case "$OS" in + Windows*) isWindows=true ECHO="echo -e" + ;; +esac + +MD5SUM="md5sum" +if [ "$TRAVIS_OS_NAME" = "osx" ]; then + MD5SUM="md5 -r" fi -$ECHO "\nStarting playTests.sh isWindows=$isWindows" +$ECHO "\nStarting playTests.sh isWindows=$isWindows TRAVIS_OS_NAME=$TRAVIS_OS_NAME" [ -n "$ZSTD" ] || die "ZSTD variable must be defined!" +file $ZSTD $ECHO "\n**** simple tests **** " ./datagen > tmp @@ -120,8 +128,8 @@ rm tmpSparse* $ECHO "\n**** dictionary tests **** " ./datagen > tmpDict -./datagen -g1M | md5sum > tmp1 -./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2 +./datagen -g1M | $MD5SUM > tmp1 +./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | $MD5SUM > tmp2 diff -q tmp1 tmp2 $ECHO "Create first dictionary" $ZSTD --train *.c -o tmpDict diff --git a/programs/tests/playTests.bat b/programs/tests/playTests.bat deleted file mode 100644 index 5f7ebd68..00000000 --- a/programs/tests/playTests.bat +++ /dev/null @@ -1,157 +0,0 @@ -@echo off -ECHO Start playTests.bat -ECHO ZSTD^: %ZSTD% -ECHO DATAGEN^: %DATAGEN% -if "%ZSTD%"=="" echo ZSTD variable must be defined! && exit /b 1 -if "%DATAGEN%"=="" echo DATAGEN variable must be defined! && exit /b 1 - -SET ROUNDTRIPTEST=tests\roundTripTest.bat - -echo. && echo **** simple tests **** -%DATAGEN% > tmp -%ZSTD% -f tmp && REM trivial compression case, creates tmp.zst -%ZSTD% -df tmp.zst && REM trivial decompression case (overwrites tmp) -echo test : too large compression level (must fail) -%ZSTD% -99 tmp && (echo too large compression level undetected && exit /b 1) -echo test : compress to stdout -%ZSTD% tmp -c > tmpCompressed -%ZSTD% tmp --stdout > tmpCompressed && REM long command format -echo test : null-length file roundtrip -echo. 2>tmpEmpty | cat tmpEmpty | %ZSTD% - --stdout | %ZSTD% -d --stdout || (echo wrong null-length file roundtrip && exit /b 1) -echo test : decompress file with wrong suffix (must fail) -%ZSTD% -d tmpCompressed && (echo wrong suffix error not detected! && exit /b 1) -%ZSTD% -d tmpCompressed -c > tmpResult && REM decompression using stdout -%ZSTD% --decompress tmpCompressed -c > tmpResult -%ZSTD% --decompress tmpCompressed --stdout > tmpResult -REM %ZSTD% -d < tmp.zst > NUL && REM combine decompression, stdin & stdout -REM %ZSTD% -d - < tmp.zst > NUL -%ZSTD% -dc < tmp.zst > NUL -%ZSTD% -dc - < tmp.zst > NUL -%ZSTD% -q tmp && (echo overwrite check failed! && exit /b 1) -%ZSTD% -q -f tmp -%ZSTD% -q --force tmp -%ZSTD% -df tmp && (echo should have refused : wrong extension && exit /b 1) - -echo. && echo **** Pass-Through mode **** -echo "Hello world !" | %ZSTD% -df -echo "Hello world !" | %ZSTD% -dcf - - -echo. && echo **** frame concatenation **** - -echo hello > hello.tmp -echo world! > world.tmp -cat hello.tmp world.tmp > helloworld.tmp -%ZSTD% -c hello.tmp > hello.zstd -%ZSTD% -c world.tmp > world.zstd -cat hello.zstd world.zstd > helloworld.zstd -%ZSTD% -dc helloworld.zstd > result.tmp -cat result.tmp -fc /b helloworld.tmp result.tmp -rm *.tmp *.zstd - -echo frame concatenation tests completed - - -REM echo. && echo **** flush write error test **** - -REM echo echo foo ^| %ZSTD% ^> v:\full -REM echo foo | %ZSTD% > v:\full && (echo write error not detected! && exit /b 1) -REM echo "echo foo | %ZSTD% | %ZSTD% -d > /dev/full" -REM echo foo | %ZSTD% | %ZSTD% -d > /dev/full && (echo write error not detected! && exit /b 1) - - -echo. && echo **** dictionary tests **** - -%DATAGEN% > tmpDict -%DATAGEN% -g1M | md5sum > tmp1 -%DATAGEN% -g1M | %ZSTD% -D tmpDict | %ZSTD% -D tmpDict -dvq | md5sum > tmp2 -fc tmp1 tmp2 -%ZSTD% --train *.c *.h -o tmpDict -%ZSTD% xxhash.c -D tmpDict -of tmp -%ZSTD% -d tmp -D tmpDict -of result -fc xxhash.c result - - -echo. && echo **** multiple files tests **** - -%DATAGEN% -s1 > tmp1 2> NUL -%DATAGEN% -s2 -g100K > tmp2 2> NUL -%DATAGEN% -s3 -g1M > tmp3 2> NUL -%ZSTD% -f tmp* -echo compress tmp* : -ls -ls tmp* -rm tmp1 tmp2 tmp3 -echo decompress tmp* : -%ZSTD% -df *.zst -ls -ls tmp* - -echo compress tmp* into stdout ^> tmpall : -%ZSTD% -c tmp1 tmp2 tmp3 > tmpall -ls -ls tmp* -echo decompress tmpall* into stdout ^> tmpdec : -cp tmpall tmpall2 -%ZSTD% -dc tmpall* > tmpdec -ls -ls tmp* -echo compress multiple files including a missing one (notHere) : -%ZSTD% -f tmp1 notHere tmp2 && (echo missing file not detected! && exit /b 1) - - -echo. && echo **** integrity tests **** -echo test one file (tmp1.zst) -%ZSTD% -t tmp1.zst -%ZSTD% --test tmp1.zst -echo test multiple files (*.zst) -%ZSTD% -t *.zst -echo test good and bad files (*) -%ZSTD% -t * && (echo bad files not detected! && exit /b 1) - - -echo. && echo **** zstd round-trip tests **** - -CALL %ROUNDTRIPTEST% -CALL %ROUNDTRIPTEST% -g15K && REM TableID==3 -CALL %ROUNDTRIPTEST% -g127K && REM TableID==2 -CALL %ROUNDTRIPTEST% -g255K && REM TableID==1 -CALL %ROUNDTRIPTEST% -g513K && REM TableID==0 -CALL %ROUNDTRIPTEST% -g512K 6 && REM greedy, hash chain -CALL %ROUNDTRIPTEST% -g512K 16 && REM btlazy2 -CALL %ROUNDTRIPTEST% -g512K 19 && REM btopt - -rm tmp* -echo Param = %1 -if NOT "%1"=="--test-large-data" ( - echo skipping large data tests - exit /b 0 -) - -CALL %ROUNDTRIPTEST% -g270000000 1 -CALL %ROUNDTRIPTEST% -g270000000 2 -CALL %ROUNDTRIPTEST% -g270000000 3 - -CALL %ROUNDTRIPTEST% -g140000000 -P60 4 -CALL %ROUNDTRIPTEST% -g140000000 -P60 5 -CALL %ROUNDTRIPTEST% -g140000000 -P60 6 - -CALL %ROUNDTRIPTEST% -g70000000 -P70 7 -CALL %ROUNDTRIPTEST% -g70000000 -P70 8 -CALL %ROUNDTRIPTEST% -g70000000 -P70 9 - -CALL %ROUNDTRIPTEST% -g35000000 -P75 10 -CALL %ROUNDTRIPTEST% -g35000000 -P75 11 -CALL %ROUNDTRIPTEST% -g35000000 -P75 12 - -CALL %ROUNDTRIPTEST% -g18000000 -P80 13 -CALL %ROUNDTRIPTEST% -g18000000 -P80 14 -CALL %ROUNDTRIPTEST% -g18000000 -P80 15 -CALL %ROUNDTRIPTEST% -g18000000 -P80 16 -CALL %ROUNDTRIPTEST% -g18000000 -P80 17 - -CALL %ROUNDTRIPTEST% -g50000000 -P94 18 -CALL %ROUNDTRIPTEST% -g50000000 -P94 19 - -CALL %ROUNDTRIPTEST% -g99000000 -P99 20 -CALL %ROUNDTRIPTEST% -g6000000000 -P99 1 - -rm tmp* -exit /b 0 diff --git a/programs/tests/roundTripTest.bat b/programs/tests/roundTripTest.bat deleted file mode 100644 index 759891f0..00000000 --- a/programs/tests/roundTripTest.bat +++ /dev/null @@ -1,8 +0,0 @@ -@echo off -if [%3]==[] (SET C=%2 && SET P=) ELSE (SET C=%3 && SET P=%2) -rm -f tmp1 tmp2 -echo roundTripTest: datagen %1 %P% ^| %ZSTD% -v%C% ^| %ZSTD% -d -%DATAGEN% %1 %P% | md5sum > tmp1 -%DATAGEN% %1 %P% | %ZSTD% -vq%C% | %ZSTD% -d | md5sum > tmp2 -fc tmp1 tmp2 -EXIT /B %ERRORLEVEL% diff --git a/programs/zbufftest.c b/programs/zbufftest.c index d0c0503a..278339dd 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -147,11 +147,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo int testResult = 0; size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH; void* CNBuffer = malloc(CNBufferSize); - size_t const compressedBufferSize = ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH); + size_t const skippableFrameSize = 11; + size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH); void* compressedBuffer = malloc(compressedBufferSize); size_t const decodedBufferSize = CNBufferSize; void* decodedBuffer = malloc(decodedBufferSize); - size_t result, cSize, readSize, genSize; + size_t result, cSize, readSize, readSkipSize, genSize; U32 testNb=0; ZBUFF_CCtx* zc = ZBUFF_createCCtx_advanced(customMem); ZBUFF_DCtx* zd = ZBUFF_createDCtx_advanced(customMem); @@ -163,15 +164,19 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo } RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed); + /* generate skippable frame */ + MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START); + MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize); + cSize = skippableFrameSize + 8; /* Basic compression test */ DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1); readSize = CNBufferSize; genSize = compressedBufferSize; - result = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize); + result = ZBUFF_compressContinue(zc, ((char*)compressedBuffer)+cSize, &genSize, CNBuffer, &readSize); if (ZBUFF_isError(result)) goto _output_error; if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */ - cSize = genSize; + cSize += genSize; genSize = compressedBufferSize - cSize; result = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize); if (result != 0) goto _output_error; /* error, or some data not flushed */ @@ -181,12 +186,17 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* Basic decompression test */ DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); - readSize = cSize; + readSkipSize = cSize; genSize = CNBufferSize; - result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSize); + result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSkipSize); + if (genSize != 0) goto _output_error; /* skippable frame */ + ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); + readSize = cSize - readSkipSize; + genSize = CNBufferSize; + result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, ((char*)compressedBuffer)+readSkipSize, &readSize); if (result != 0) goto _output_error; /* should reach end of frame == 0; otherwise, some data left, or an error */ if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */ - if (readSize != cSize) goto _output_error; /* should have read the entire frame */ + if (readSize+readSkipSize != cSize) goto _output_error; /* should have read the entire frame */ DISPLAYLEVEL(4, "OK \n"); /* check regenerated data is byte exact */ @@ -200,17 +210,20 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* Byte-by-byte decompression test */ DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++); - ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); - { size_t r = 1, pIn=0, pOut=0; - while (r) { - size_t inS = 1; - size_t outS = 1; - r = ZBUFF_decompressContinue(zd, ((BYTE*)decodedBuffer)+pOut, &outS, ((BYTE*)compressedBuffer)+pIn, &inS); - pIn += inS; - pOut += outS; - } - readSize = pIn; - genSize = pOut; + { size_t r, pIn=0, pOut=0; + do + { ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); + r = 1; + while (r) { + size_t inS = 1; + size_t outS = 1; + r = ZBUFF_decompressContinue(zd, ((BYTE*)decodedBuffer)+pOut, &outS, ((BYTE*)compressedBuffer)+pIn, &inS); + pIn += inS; + pOut += outS; + } + readSize = pIn; + genSize = pOut; + } while (genSize==0); } if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */ if (readSize != cSize) goto _output_error; /* should have read the entire frame */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index e9fa0526..24bba89d 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -52,7 +52,7 @@ # include /* _isatty */ # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) #else -#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE +#if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) # include /* isatty */ # define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) #else diff --git a/projects/VS2008/zstd/zstd.vcproj b/projects/VS2008/zstd/zstd.vcproj index 64949899..38b76934 100644 --- a/projects/VS2008/zstd/zstd.vcproj +++ b/projects/VS2008/zstd/zstd.vcproj @@ -44,7 +44,7 @@ + @@ -119,27 +119,27 @@ true - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false $(LibraryPath) $(Platform)\$(Configuration)\ true - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false $(LibraryPath); false - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false $(LibraryPath) $(Platform)\$(Configuration)\ false - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false $(LibraryPath); diff --git a/projects/VS2010/zstdlib/zstdlib.vcxproj b/projects/VS2010/zstdlib/zstdlib.vcxproj index 25d3ca1d..79900501 100644 --- a/projects/VS2010/zstdlib/zstdlib.vcxproj +++ b/projects/VS2010/zstdlib/zstdlib.vcxproj @@ -1,4 +1,4 @@ - + @@ -99,28 +99,28 @@ true zstdlib_x86 $(Platform)\$(Configuration)\ - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false true zstdlib_x64 $(Platform)\$(Configuration)\ - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false false zstdlib_x86 $(Platform)\$(Configuration)\ - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false false zstdlib_x64 $(Platform)\$(Configuration)\ - $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath); + $(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath); false diff --git a/projects/cmake/CMakeLists.txt b/projects/cmake/CMakeLists.txt index 41883373..ae8f7be6 100644 --- a/projects/cmake/CMakeLists.txt +++ b/projects/cmake/CMakeLists.txt @@ -44,13 +44,6 @@ ELSE (ZSTD_LEGACY_SUPPORT) ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=0) ENDIF (ZSTD_LEGACY_SUPPORT) -# TARGET_INCLUDE_DIRECTORIES can use in version 2.8.11 and greate -IF ((${CMAKE_MAJOR_VERSION} EQUAL 2) AND (${CMAKE_MINOR_VERSION} EQUAL 8) AND (${CMAKE_PATCH_VERSION} LESS 11)) - SET(WORKAROUND_OUTDATED_CODE_STYLE TRUE) -ELSE () - SET(WORKAROUND_OUTDATED_CODE_STYLE FALSE) -ENDIF ((${CMAKE_MAJOR_VERSION} EQUAL 2) AND (${CMAKE_MINOR_VERSION} EQUAL 8) AND (${CMAKE_PATCH_VERSION} LESS 11)) - ADD_SUBDIRECTORY(lib) ADD_SUBDIRECTORY(programs) diff --git a/projects/cmake/lib/CMakeLists.txt b/projects/cmake/lib/CMakeLists.txt index 540f2074..716ccbf3 100644 --- a/projects/cmake/lib/CMakeLists.txt +++ b/projects/cmake/lib/CMakeLists.txt @@ -96,16 +96,23 @@ IF (ZSTD_LEGACY_SUPPORT) ${LIBRARY_LEGACY_DIR}/zstd_v01.c ${LIBRARY_LEGACY_DIR}/zstd_v02.c ${LIBRARY_LEGACY_DIR}/zstd_v03.c - ${LIBRARY_LEGACY_DIR}/zstd_v04.c) + ${LIBRARY_LEGACY_DIR}/zstd_v04.c + ${LIBRARY_LEGACY_DIR}/zstd_v05.c) SET(Headers ${Headers} ${LIBRARY_LEGACY_DIR}/zstd_legacy.h ${LIBRARY_LEGACY_DIR}/zstd_v01.h ${LIBRARY_LEGACY_DIR}/zstd_v02.h ${LIBRARY_LEGACY_DIR}/zstd_v03.h - ${LIBRARY_LEGACY_DIR}/zstd_v04.h) + ${LIBRARY_LEGACY_DIR}/zstd_v04.h + ${LIBRARY_LEGACY_DIR}/zstd_v05.h) ENDIF (ZSTD_LEGACY_SUPPORT) +IF (MSVC) + SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/projects/VS2010/zstdlib) + SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstdlib.rc) +ENDIF (MSVC) + # Split project to static and shared libraries build ADD_LIBRARY(libzstd_static STATIC ${Sources} ${Headers} ${PlatformDependResources}) ADD_LIBRARY(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources}) @@ -116,16 +123,6 @@ IF (MSVC) SET_TARGET_PROPERTIES(libzstd_shared PROPERTIES COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS") ENDIF (MSVC) -# Define include directories -IF (NOT WORKAROUND_OUTDATED_CODE_STYLE) - TARGET_INCLUDE_DIRECTORIES(libzstd_static PUBLIC ${LIBRARY_DIR}/common) - TARGET_INCLUDE_DIRECTORIES(libzstd_shared PUBLIC ${LIBRARY_DIR}/common) - IF (ZSTD_LEGACY_SUPPORT) - TARGET_INCLUDE_DIRECTORIES(libzstd_static PUBLIC ${LIBRARY_LEGACY_DIR}) - TARGET_INCLUDE_DIRECTORIES(libzstd_shared PUBLIC ${LIBRARY_LEGACY_DIR}) - ENDIF (ZSTD_LEGACY_SUPPORT) -ENDIF (NOT WORKAROUND_OUTDATED_CODE_STYLE) - # Define library base name IF (MSVC) SET(LIBRARY_BASE_NAME zstdlib) diff --git a/projects/cmake/programs/CMakeLists.txt b/projects/cmake/programs/CMakeLists.txt index ef5f0eaf..c8fe5d2a 100644 --- a/projects/cmake/programs/CMakeLists.txt +++ b/projects/cmake/programs/CMakeLists.txt @@ -44,12 +44,7 @@ INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/dictBui IF (ZSTD_LEGACY_SUPPORT) SET(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy) - INCLUDE_DIRECTORIES(${PROGRAMS_LEGACY_DIR}) - - IF (WORKAROUND_OUTDATED_CODE_STYLE) - INCLUDE_DIRECTORIES(${LIBRARY_DIR}/legacy) - ENDIF (WORKAROUND_OUTDATED_CODE_STYLE) - + INCLUDE_DIRECTORIES(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy) SET(ZSTD_FILEIO_LEGACY ${PROGRAMS_LEGACY_DIR}/fileio_legacy.c) ENDIF (ZSTD_LEGACY_SUPPORT) diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index fd45ee6d..c74107d4 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -15,14 +15,14 @@ IMPLIB = -lz ../lib/libzstd.a endif ZLIBWRAPPER_PATH = . -EXAMPLE_PATH = examples/ +EXAMPLE_PATH = examples CC = gcc CFLAGS = $(LOC) -I../lib/common -I$(ZLIBWRAPPER_PATH) -O3 -Wall -std=gnu89 LDFLAGS = $(LOC) RM = rm -f -all: example example_d +all: clean test testdll test: example ./example diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h index c9039ddb..c2c908dd 100644 --- a/zlibWrapper/zstd_zlibwrapper.h +++ b/zlibWrapper/zstd_zlibwrapper.h @@ -40,6 +40,9 @@ extern "C" { #define Z_PREFIX #include +#if !defined(z_const) +# define z_const const +#endif void useZSTD(int turn_on); int isUsingZSTD();