facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #384 from inikep/zlibWrapper 

Zlibwrapper
dev
Yann Collet 2016-09-27 00:32:05 +02:00 committed by GitHub
parent 47094ea66b 60dddc2109
commit 8675394d0d
12 changed files with 2140 additions and 333 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.travis.yml
View File

@ -22,7 +22,7 @@ matrix:
packages: packages:
- gcc-4.8 - gcc-4.8
- g++-4.8 - g++-4.8
env: PLATFORM="Ubuntu 12.04 container" CMD="make -C tests test-zstd_nolegacy && make clean && make zlibwrapper && make clean && make cmaketest && make clean && make -C contrib/pzstd pzstd && make -C contrib/pzstd googletest && make -C contrib/pzstd test && make -C contrib/pzstd clean" env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper && make clean && make -C tests test-zstd_nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd pzstd && make -C contrib/pzstd googletest && make -C contrib/pzstd test && make -C contrib/pzstd clean"
- os: linux - os: linux
sudo: false sudo: false
env: PLATFORM="Ubuntu 12.04 container" CMD="make usan" env: PLATFORM="Ubuntu 12.04 container" CMD="make usan"
@ -55,14 +55,15 @@ matrix:
packages: packages:
- libc6-dev-i386 - libc6-dev-i386
- gcc-multilib - gcc-multilib
# Ubuntu 14.04 LTS Server Edition 64 bit
- os: linux - os: linux
dist: trusty
sudo: required sudo: required
env: PLATFORM="Ubuntu 12.04" CMD="make -C tests valgrindTest" env: PLATFORM="Ubuntu 14.04" CMD='make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest'
addons: addons:
apt: apt:
packages: packages:
- valgrind - valgrind
# Ubuntu 14.04 LTS Server Edition 64 bit
- os: linux - os: linux
dist: trusty dist: trusty
sudo: required sudo: required
@ -90,7 +91,7 @@ matrix:
- os: linux - os: linux
dist: trusty dist: trusty
sudo: required sudo: required
env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper && make clean && make gcc5test && make clean && make gcc6test && sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu && make clean && make ppctest" env: PLATFORM="Ubuntu 14.04" CMD="make gcc5test && make clean && make gcc6test && sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu && make clean && make ppctest"
addons: addons:
apt: apt:
sources: sources:

3
Makefile
View File

@ -34,8 +34,7 @@ zstd:
cp $(PRGDIR)/zstd . cp $(PRGDIR)/zstd .
zlibwrapper: zlibwrapper:
$(MAKE) -C $(ZSTDDIR) all $(MAKE) -C $(ZWRAPDIR) test
$(MAKE) -C $(ZWRAPDIR) all
test: test:
$(MAKE) -C $(TESTDIR) $@ $(MAKE) -C $(TESTDIR) $@

2
contrib/pzstd/Options.cpp
View File

@ -104,7 +104,7 @@ void usage() {
std::fprintf(stderr, " -V, --version : display version number and exit\n"); std::fprintf(stderr, " -V, --version : display version number and exit\n");
std::fprintf(stderr, " -v, --verbose : verbose mode; specify multiple times to increase log level (default:2)\n"); std::fprintf(stderr, " -v, --verbose : verbose mode; specify multiple times to increase log level (default:2)\n");
std::fprintf(stderr, " -q, --quiet : suppress warnings; specify twice to suppress errors too\n"); std::fprintf(stderr, " -q, --quiet : suppress warnings; specify twice to suppress errors too\n");
std::fprintf(stderr, " -c, --stdout : force wrtie to standard output, even if it is the console\n"); std::fprintf(stderr, " -c, --stdout : force write to standard output, even if it is the console\n");
#ifdef UTIL_HAS_CREATEFILELIST #ifdef UTIL_HAS_CREATEFILELIST
std::fprintf(stderr, " -r : operate recursively on directories\n"); std::fprintf(stderr, " -r : operate recursively on directories\n");
#endif #endif

2
zlibWrapper/.gitignore vendored
View File

@ -26,4 +26,4 @@ foo.gz
# Misc files # Misc files
*.bat *.bat
*.zip *.zip
examples/example2.c *.txt

79
zlibWrapper/Makefile
View File

@ -1,54 +1,71 @@
# Makefile for example of using zstd wrapper for zlib # Makefile for example of using zstd wrapper for zlib
# #
# make - compiles statically and dynamically linked examples/example.c # make - compiles examples
# make test testdll - compiles and runs statically and dynamically linked examples/example.c # make LOC=-DZWRAP_USE_ZSTD=1 - compiles examples with zstd compression turned on
# make LOC=-DZWRAP_USE_ZSTD=1 - compiles statically and dynamically linked examples/example.c with zstd compression turned on # make test - runs examples
# Paths to static and dynamic zlib and zstd libraries # Paths to static and dynamic zlib and zstd libraries
# Use "make ZLIBDIR=path/to/zlib" to select a path to library # Use "make ZLIB_LIBRARY=path/to/zlib" to select a path to library
ifdef ZLIBDIR ZLIB_LIBRARY ?= -lz
STATICLIB = $(ZLIBDIR)/libz.a ../lib/libzstd.a
IMPLIB = $(ZLIBDIR)/libz.dll.a ../lib/libzstd.a
else
STATICLIB = -static -lz ../lib/libzstd.a
IMPLIB = -lz ../lib/libzstd.a
endif
ZSTDLIBDIR = ../lib
ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.a
ZLIBWRAPPER_PATH = . ZLIBWRAPPER_PATH = .
EXAMPLE_PATH = examples EXAMPLE_PATH = examples
PROGRAMS_PATH = ../programs
CC ?= gcc CC ?= gcc
CFLAGS = $(LOC) -I../lib -I../lib/common -I$(ZLIBDIR) -I$(ZLIBWRAPPER_PATH) -O3 -std=gnu90 CFLAGS ?= -O3
CFLAGS += $(LOC) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -std=gnu99
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
LDFLAGS = $(LOC) LDFLAGS = $(LOC)
RM = rm -f RM = rm -f
all: clean test testzstd all: clean fitblk example zwrapbench
test: example test: example fitblk example_zstd fitblk_zstd zwrapbench
./example ./example
testdll: example_d
./example_d
testzstd: example_zstd
./example_zstd ./example_zstd
./fitblk 10240 <../zstd_compression_format.md
./fitblk 40960 <../zstd_compression_format.md
./fitblk_zstd 10240 <../zstd_compression_format.md
./fitblk_zstd 40960 <../zstd_compression_format.md
./zwrapbench -qb3B1K ../zstd_compression_format.md
./zwrapbench -rqb1e5 ../lib ../programs ../tests
#valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so
valgrindTest: VALGRIND = LD_LIBRARY_PATH=$(ZSTDLIBDIR) valgrind --track-origins=yes --leak-check=full --error-exitcode=1
valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench
@echo "\n ---- valgrind tests ----"
$(VALGRIND) ./example
$(VALGRIND) ./example_zstd
$(VALGRIND) ./fitblk 10240 <../zstd_compression_format.md
$(VALGRIND) ./fitblk 40960 <../zstd_compression_format.md
$(VALGRIND) ./fitblk_zstd 10240 <../zstd_compression_format.md
$(VALGRIND) ./fitblk_zstd 40960 <../zstd_compression_format.md
$(VALGRIND) ./zwrapbench -qb3B1K ../zstd_compression_format.md
$(VALGRIND) ./zwrapbench -rqb1e5 ../lib ../programs ../tests
.c.o: .c.o:
$(CC) $(CFLAGS) -c -o $@ $< $(CC) $(CFLAGS) -c -o $@ $<
example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY)
$(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY) $(ZLIB_LIBRARY)
example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o example_zstd: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY)
$(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(STATICLIB) $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY) $(ZLIB_LIBRARY)
example_d: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o fitblk: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY)
$(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(IMPLIB) $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY) $(ZLIB_LIBRARY)
example_zstd: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY)
$(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(STATICLIB) $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY) $(ZLIB_LIBRARY)
$(EXAMPLE_PATH)/example.o: $(EXAMPLE_PATH)/example.c zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY)
$(CC) $(CFLAGS) -I. -c -o $@ $(EXAMPLE_PATH)/example.c $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY) $(ZLIB_LIBRARY)
$(EXAMPLE_PATH)/zwrapbench.o: $(EXAMPLE_PATH)/zwrapbench.c
$(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
$(CC) $(CFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(CC) $(CFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
@ -56,6 +73,12 @@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $
$(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
$(CC) $(CFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(CC) $(CFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
$(ZSTDLIBDIR)/libzstd.a:
$(MAKE) -C $(ZSTDLIBDIR) all
$(ZSTDLIBDIR)/libzstd.so:
$(MAKE) -C $(ZSTDLIBDIR) all
clean: clean:
-$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_d example_zstd -$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_zstd fitblk fitblk_zstd zwrapbench
@echo Cleaning completed @echo Cleaning completed

74
zlibWrapper/README.md
View File

@ -23,10 +23,10 @@ Let's assume that your project that uses zlib is compiled with:
To compile the zstd wrapper with your project you have to do the following: To compile the zstd wrapper with your project you have to do the following:
- change all references with ```#include "zlib.h"``` to ```#include "zstd_zlibwrapper.h"``` - change all references with ```#include "zlib.h"``` to ```#include "zstd_zlibwrapper.h"```
- compile your project with zlib_wrapper.c and a static or dynamic zstd library - compile your project with `zstd_zlibwrapper.c` and a static or dynamic zstd library
The linking should be changed to: The linking should be changed to:
```gcc project.o zlib_wrapper.o -lz -lzstd``` ```gcc project.o zstd_zlibwrapper.o -lz -lzstd```
#### Enabling zstd compression within your project #### Enabling zstd compression within your project
@ -34,8 +34,10 @@ The linking should be changed to:
After embedding the zstd wrapper within your project the zstd library is turned off by default. After embedding the zstd wrapper within your project the zstd library is turned off by default.
Your project should work as before with zlib. There are two options to enable zstd compression: Your project should work as before with zlib. There are two options to enable zstd compression:
- compilation with ```-DZWRAP_USE_ZSTD=1``` (or using ```#define ZWRAP_USE_ZSTD 1``` before ```#include "zstd_zlibwrapper.h"```) - compilation with ```-DZWRAP_USE_ZSTD=1``` (or using ```#define ZWRAP_USE_ZSTD 1``` before ```#include "zstd_zlibwrapper.h"```)
- using the ```void useZSTD(int turn_on)``` function (declared in ```#include "zstd_zlibwrapper.h"```) - using the ```void ZWRAP_useZSTDcompression(int turn_on)``` function (declared in ```#include "zstd_zlibwrapper.h"```)
There is no switch for zstd decompression because zlib and zstd streams are automatically detected and decompressed using a proper library.
During decompression zlib and zstd streams are automatically detected and decompressed using a proper library.
This behavior can be changed using `ZWRAP_setDecompressionType(ZWRAP_FORCE_ZLIB)` what will make zlib decompression slightly faster.
#### Example #### Example
@ -52,7 +54,7 @@ after inflateSync(): hello, hello!
inflate with dictionary: hello, hello! inflate with dictionary: hello, hello!
``` ```
Then we have changed ```#include "zlib.h"``` to ```#include "zstd_zlibwrapper.h"```, compiled the [example.c](examples/example.c) file Then we have changed ```#include "zlib.h"``` to ```#include "zstd_zlibwrapper.h"```, compiled the [example.c](examples/example.c) file
with ```-DZWRAP_USE_ZSTD=1``` and linked with additional ```zlib_wrapper.o -lzstd```. with ```-DZWRAP_USE_ZSTD=1``` and linked with additional ```zstd_zlibwrapper.o -lzstd```.
We were forced to turn off the following functions: ```test_gzio```, ```test_flush```, ```test_sync``` which use currently unsupported features. We were forced to turn off the following functions: ```test_gzio```, ```test_flush```, ```test_sync``` which use currently unsupported features.
After running it shows the following results: After running it shows the following results:
``` ```
@ -65,18 +67,72 @@ inflate with dictionary: hello, hello!
The script used for compilation can be found at [zlibWrapper/Makefile](Makefile). The script used for compilation can be found at [zlibWrapper/Makefile](Makefile).
#### The measurement of performace of Zstandard wrapper for zlib
The zstd distribution contains a tool called `zwrapbench` which can measure speed and ratio of zlib, zstd, and the wrapper.
The benchmark is conducted using given filenames or synthetic data if filenames are not provided.
The files are read into memory and joined together.
It makes benchmark more precise as it eliminates I/O overhead.
Many filenames can be supplied as multiple parameters, parameters with wildcards or names of directories can be used as parameters with the -r option.
One can select compression levels starting from `-b` and ending with `-e`. The `-i` parameter selects minimal time used for each of tested levels.
With `-B` option bigger files can be divided into smaller, independently compressed blocks.
The benchmark tool can be compiled with `make zwrapbench` using [zlibWrapper/Makefile](Makefile).
#### Improving speed of streaming compression
During streaming compression the compressor never knows how big is data to compress.
Zstandard compression can be improved by providing size of source data to the compressor. By default streaming compressor assumes that data is bigger than 256 KB but it can hurt compression speed on smaller data.
The zstd wrapper provides the `ZWRAP_setPledgedSrcSize()` function that allows to change a pledged source size for a given compression stream.
The function will change zstd compression parameters what may improve compression speed and/or ratio.
It should be called just after `deflateInit()`. The function is only helpful when data is compressed in blocks. There will be no change in case of `deflateInit()` immediately followed by `deflate(strm, Z_FINISH)`
as this case is automatically detected.
#### Reusing contexts
The ordinary zlib compression of two files/streams allocates two contexts:
- for the 1st file calls `deflateInit`, `deflate`, `...`, `deflate`, `defalateEnd`
- for the 2nd file calls `deflateInit`, `deflate`, `...`, `deflate`, `defalateEnd`
The speed of compression can be improved with reusing a single context with following steps:
- initialize the context with `deflateInit`
- for the 1st file call `deflate`, `...`, `deflate`
- for the 2nd file call `deflateReset`, `deflate`, `...`, `deflate`
- free the context with `deflateEnd`
To check the difference we made experiments using `zwrapbench -ri6b6` with zstd and zlib compression (both at level 6).
The input data was decompressed git repository downloaded from https://github.com/git/git/archive/master.zip which contains 2979 files.
The table below shows that reusing contexts has a minor influence on zlib but it gives improvement for zstd.
In our example (the last 2 lines) it gives 4% better compression speed and 5% better decompression speed.
| Compression type | Compression | Decompress.| Compr. size | Ratio |
| ------------------------------------------------- | ------------| -----------| ----------- | ----- |
| zlib 1.2.8 | 30.51 MB/s | 219.3 MB/s | 6819783 | 3.459 |
| zlib 1.2.8 not reusing a context | 30.22 MB/s | 218.1 MB/s | 6819783 | 3.459 |
| zlib 1.2.8 with zlibWrapper and reusing a context | 30.40 MB/s | 218.9 MB/s | 6819783 | 3.459 |
| zlib 1.2.8 with zlibWrapper not reusing a context | 30.28 MB/s | 218.1 MB/s | 6819783 | 3.459 |
| zstd 1.1.0 using ZSTD_CCtx | 68.35 MB/s | 430.9 MB/s | 6868521 | 3.435 |
| zstd 1.1.0 using ZSTD_CStream | 66.63 MB/s | 422.3 MB/s | 6868521 | 3.435 |
| zstd 1.1.0 with zlibWrapper and reusing a context | 54.01 MB/s | 403.2 MB/s | 6763482 | 3.488 |
| zstd 1.1.0 with zlibWrapper not reusing a context | 51.59 MB/s | 383.7 MB/s | 6763482 | 3.488 |
#### Compatibility issues #### Compatibility issues
After enabling zstd compression not all native zlib functions are supported. When calling unsupported methods they print error message and return an error value. After enabling zstd compression not all native zlib functions are supported. When calling unsupported methods they put error message into strm->msg and return Z_STREAM_ERROR.
Supported methods: Supported methods:
- deflateInit - deflateInit
- deflate (with exception of Z_FULL_FLUSH) - deflate (with exception of Z_FULL_FLUSH, Z_BLOCK, and Z_TREES)
- deflateSetDictionary - deflateSetDictionary
- deflateEnd - deflateEnd
- deflateReset
- deflateBound - deflateBound
- inflateInit - inflateInit
- inflate - inflate
- inflateSetDictionary - inflateSetDictionary
- inflateReset
- inflateReset2
- compress - compress
- compress2 - compress2
- compressBound - compressBound
@ -88,15 +144,13 @@ Ignored methods (they do nothing):
Unsupported methods: Unsupported methods:
- gzip file access functions - gzip file access functions
- deflateCopy - deflateCopy
- deflateReset
- deflateTune - deflateTune
- deflatePending - deflatePending
- deflatePrime - deflatePrime
- deflateSetHeader - deflateSetHeader
- inflateGetDictionary - inflateGetDictionary
- inflateCopy - inflateCopy
- inflateReset - inflateSync
- inflateReset2
- inflatePrime - inflatePrime
- inflateMark - inflateMark
- inflateGetHeader - inflateGetHeader

16
zlibWrapper/examples/example.c
View File

@ -45,12 +45,12 @@
} \ } \
} }
z_const char hello[] = "hello, hello!"; z_const char hello[] = "hello, hello! I said hello, hello!";
/* "hello world" would be more standard, but the repeated "hello" /* "hello world" would be more standard, but the repeated "hello"
* stresses the compression code better, sorry... * stresses the compression code better, sorry...
*/ */
const char dictionary[] = "hello"; const char dictionary[] = "hello, hello!";
uLong dictId; /* Adler32 value of the dictionary */ uLong dictId; /* Adler32 value of the dictionary */
void test_deflate OF((Byte *compr, uLong comprLen)); void test_deflate OF((Byte *compr, uLong comprLen));
@ -156,7 +156,7 @@ void test_gzio(fname, uncompr, uncomprLen)
fprintf(stderr, "gzputs err: %s\n", gzerror(file, &err)); fprintf(stderr, "gzputs err: %s\n", gzerror(file, &err));
exit(1); exit(1);
} }
if (gzprintf(file, ", %s!", "hello") != 8) { if (gzprintf(file, ", %s! I said hello, hello!", "hello") != 8+21) {
fprintf(stderr, "gzprintf err: %s\n", gzerror(file, &err)); fprintf(stderr, "gzprintf err: %s\n", gzerror(file, &err));
exit(1); exit(1);
} }
@ -182,7 +182,7 @@ void test_gzio(fname, uncompr, uncomprLen)
} }
pos = gzseek(file, -8L, SEEK_CUR); pos = gzseek(file, -8L, SEEK_CUR);
if (pos != 6 || gztell(file) != pos) { if (pos != 6+21 || gztell(file) != pos) {
fprintf(stderr, "gzseek error, pos=%ld, gztell=%ld\n", fprintf(stderr, "gzseek error, pos=%ld, gztell=%ld\n",
(long)pos, (long)gztell(file)); (long)pos, (long)gztell(file));
exit(1); exit(1);
@ -203,7 +203,7 @@ void test_gzio(fname, uncompr, uncomprLen)
fprintf(stderr, "gzgets err after gzseek: %s\n", gzerror(file, &err)); fprintf(stderr, "gzgets err after gzseek: %s\n", gzerror(file, &err));
exit(1); exit(1);
} }
if (strcmp((char*)uncompr, hello + 6)) { if (strcmp((char*)uncompr, hello + 6+21)) {
fprintf(stderr, "bad gzgets after gzseek\n"); fprintf(stderr, "bad gzgets after gzseek\n");
exit(1); exit(1);
} else { } else {
@ -583,7 +583,7 @@ int main(argc, argv)
printf("zlib version %s = 0x%04x, compile flags = 0x%lx\n", printf("zlib version %s = 0x%04x, compile flags = 0x%lx\n",
ZLIB_VERSION, ZLIB_VERNUM, zlibCompileFlags()); ZLIB_VERSION, ZLIB_VERNUM, zlibCompileFlags());
if (isUsingZSTD()) printf("zstd version %s\n", zstdVersion()); if (ZWRAP_isUsingZSTDcompression()) printf("zstd version %s\n", zstdVersion());
compr = (Byte*)calloc((uInt)comprLen, 1); compr = (Byte*)calloc((uInt)comprLen, 1);
uncompr = (Byte*)calloc((uInt)uncomprLen, 1); uncompr = (Byte*)calloc((uInt)uncomprLen, 1);
@ -600,7 +600,7 @@ int main(argc, argv)
#else #else
test_compress(compr, comprLen, uncompr, uncomprLen); test_compress(compr, comprLen, uncompr, uncomprLen);
if (!isUsingZSTD()) if (!ZWRAP_isUsingZSTDcompression())
test_gzio((argc > 1 ? argv[1] : TESTFILE), test_gzio((argc > 1 ? argv[1] : TESTFILE),
uncompr, uncomprLen); uncompr, uncomprLen);
#endif #endif
@ -611,7 +611,7 @@ int main(argc, argv)
test_large_deflate(compr, comprLen, uncompr, uncomprLen); test_large_deflate(compr, comprLen, uncompr, uncomprLen);
test_large_inflate(compr, comprLen, uncompr, uncomprLen); test_large_inflate(compr, comprLen, uncompr, uncomprLen);
if (!isUsingZSTD()) { if (!ZWRAP_isUsingZSTDcompression()) {
test_flush(compr, &comprLen); test_flush(compr, &comprLen);
test_sync(compr, comprLen, uncompr, uncomprLen); test_sync(compr, comprLen, uncompr, uncomprLen);
} }

253
zlibWrapper/examples/fitblk.c Normal file
View File

@ -0,0 +1,253 @@
/* fitblk.c: example of fitting compressed output to a specified size
Not copyrighted -- provided to the public domain
Version 1.1 25 November 2004 Mark Adler */
/* Version history:
1.0 24 Nov 2004 First version
1.1 25 Nov 2004 Change deflateInit2() to deflateInit()
Use fixed-size, stack-allocated raw buffers
Simplify code moving compression to subroutines
Use assert() for internal errors
Add detailed description of approach
*/
/* Approach to just fitting a requested compressed size:
fitblk performs three compression passes on a portion of the input
data in order to determine how much of that input will compress to
nearly the requested output block size. The first pass generates
enough deflate blocks to produce output to fill the requested
output size plus a specfied excess amount (see the EXCESS define
below). The last deflate block may go quite a bit past that, but
is discarded. The second pass decompresses and recompresses just
the compressed data that fit in the requested plus excess sized
buffer. The deflate process is terminated after that amount of
input, which is less than the amount consumed on the first pass.
The last deflate block of the result will be of a comparable size
to the final product, so that the header for that deflate block and
the compression ratio for that block will be about the same as in
the final product. The third compression pass decompresses the
result of the second step, but only the compressed data up to the
requested size minus an amount to allow the compressed stream to
complete (see the MARGIN define below). That will result in a
final compressed stream whose length is less than or equal to the
requested size. Assuming sufficient input and a requested size
greater than a few hundred bytes, the shortfall will typically be
less than ten bytes.
If the input is short enough that the first compression completes
before filling the requested output size, then that compressed
stream is return with no recompression.
EXCESS is chosen to be just greater than the shortfall seen in a
two pass approach similar to the above. That shortfall is due to
the last deflate block compressing more efficiently with a smaller
header on the second pass. EXCESS is set to be large enough so
that there is enough uncompressed data for the second pass to fill
out the requested size, and small enough so that the final deflate
block of the second pass will be close in size to the final deflate
block of the third and final pass. MARGIN is chosen to be just
large enough to assure that the final compression has enough room
to complete in all cases.
*/
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
//#include "zlib.h"
#include "zstd_zlibwrapper.h"
#define LOG_FITBLK(...) /*printf(__VA_ARGS__)*/
#define local static
/* print nastygram and leave */
local void quit(char *why)
{
fprintf(stderr, "fitblk abort: %s\n", why);
exit(1);
}
#define RAWLEN 4096 /* intermediate uncompressed buffer size */
/* compress from file to def until provided buffer is full or end of
input reached; return last deflate() return value, or Z_ERRNO if
there was read error on the file */
local int partcompress(FILE *in, z_streamp def)
{
int ret, flush;
unsigned char raw[RAWLEN];
flush = Z_SYNC_FLUSH;
do {
def->avail_in = fread(raw, 1, RAWLEN, in);
if (ferror(in))
return Z_ERRNO;
def->next_in = raw;
if (feof(in))
flush = Z_FINISH;
LOG_FITBLK("partcompress1 avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
ret = deflate(def, flush);
LOG_FITBLK("partcompress2 avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
assert(ret != Z_STREAM_ERROR);
} while (def->avail_out != 0 && flush == Z_SYNC_FLUSH);
return ret;
}
/* recompress from inf's input to def's output; the input for inf and
the output for def are set in those structures before calling;
return last deflate() return value, or Z_MEM_ERROR if inflate()
was not able to allocate enough memory when it needed to */
local int recompress(z_streamp inf, z_streamp def)
{
int ret, flush;
unsigned char raw[RAWLEN];
flush = Z_NO_FLUSH;
do {
/* decompress */
inf->avail_out = RAWLEN;
inf->next_out = raw;
LOG_FITBLK("recompress1inflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)inf->avail_in, (int)inf->total_in, (int)inf->avail_out, (int)inf->total_out);
ret = inflate(inf, Z_NO_FLUSH);
LOG_FITBLK("recompress2inflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)inf->avail_in, (int)inf->total_in, (int)inf->avail_out, (int)inf->total_out);
assert(ret != Z_STREAM_ERROR && ret != Z_DATA_ERROR &&
ret != Z_NEED_DICT);
if (ret == Z_MEM_ERROR)
return ret;
/* compress what was decompresed until done or no room */
def->avail_in = RAWLEN - inf->avail_out;
def->next_in = raw;
if (inf->avail_out != 0)
flush = Z_FINISH;
LOG_FITBLK("recompress1deflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
ret = deflate(def, flush);
LOG_FITBLK("recompress2deflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
assert(ret != Z_STREAM_ERROR);
} while (ret != Z_STREAM_END && def->avail_out != 0);
return ret;
}
#define EXCESS 256 /* empirically determined stream overage */
#define MARGIN 8 /* amount to back off for completion */
/* compress from stdin to fixed-size block on stdout */
int main(int argc, char **argv)
{
int ret; /* return code */
unsigned size; /* requested fixed output block size */
unsigned have; /* bytes written by deflate() call */
unsigned char *blk; /* intermediate and final stream */
unsigned char *tmp; /* close to desired size stream */
z_stream def, inf; /* zlib deflate and inflate states */
/* get requested output size */
if (argc != 2)
quit("need one argument: size of output block");
ret = strtol(argv[1], argv + 1, 10);
if (argv[1][0] != 0)
quit("argument must be a number");
if (ret < 8) /* 8 is minimum zlib stream size */
quit("need positive size of 8 or greater");
size = (unsigned)ret;
printf("zlib version %s\n", ZLIB_VERSION);
if (ZWRAP_isUsingZSTDcompression()) printf("zstd version %s\n", zstdVersion());
/* allocate memory for buffers and compression engine */
blk = malloc(size + EXCESS);
def.zalloc = Z_NULL;
def.zfree = Z_NULL;
def.opaque = Z_NULL;
ret = deflateInit(&def, Z_DEFAULT_COMPRESSION);
if (ret != Z_OK || blk == NULL)
quit("out of memory");
ret = ZWRAP_setPledgedSrcSize(&def, 1<<16);
if (ret != Z_OK)
quit("ZWRAP_setPledgedSrcSize");
/* compress from stdin until output full, or no more input */
def.avail_out = size + EXCESS;
def.next_out = blk;
LOG_FITBLK("partcompress1 total_in=%d total_out=%d\n", (int)def.total_in, (int)def.total_out);
ret = partcompress(stdin, &def);
printf("partcompress total_in=%d total_out=%d\n", (int)def.total_in, (int)def.total_out);
if (ret == Z_ERRNO)
quit("error reading input");
/* if it all fit, then size was undersubscribed -- done! */
if (ret == Z_STREAM_END && def.avail_out >= EXCESS) {
/* write block to stdout */
have = size + EXCESS - def.avail_out;
// if (fwrite(blk, 1, have, stdout) != have || ferror(stdout))
// quit("error writing output");
/* clean up and print results to stderr */
ret = deflateEnd(&def);
assert(ret != Z_STREAM_ERROR);
free(blk);
fprintf(stderr,
"%u bytes unused out of %u requested (all input)\n",
size - have, size);
return 0;
}
/* it didn't all fit -- set up for recompression */
inf.zalloc = Z_NULL;
inf.zfree = Z_NULL;
inf.opaque = Z_NULL;
inf.avail_in = 0;
inf.next_in = Z_NULL;
ret = inflateInit(&inf);
tmp = malloc(size + EXCESS);
if (ret != Z_OK || tmp == NULL)
quit("out of memory");
ret = deflateReset(&def);
assert(ret != Z_STREAM_ERROR);
/* do first recompression close to the right amount */
inf.avail_in = size + EXCESS;
inf.next_in = blk;
def.avail_out = size + EXCESS;
def.next_out = tmp;
LOG_FITBLK("recompress1 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out);
ret = recompress(&inf, &def);
LOG_FITBLK("recompress1 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out);
if (ret == Z_MEM_ERROR)
quit("out of memory");
/* set up for next reocmpression */
ret = inflateReset(&inf);
assert(ret != Z_STREAM_ERROR);
ret = deflateReset(&def);
assert(ret != Z_STREAM_ERROR);
/* do second and final recompression (third compression) */
inf.avail_in = size - MARGIN; /* assure stream will complete */
inf.next_in = tmp;
def.avail_out = size;
def.next_out = blk;
LOG_FITBLK("recompress2 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out);
ret = recompress(&inf, &def);
LOG_FITBLK("recompress2 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out);
if (ret == Z_MEM_ERROR)
quit("out of memory");
assert(ret == Z_STREAM_END); /* otherwise MARGIN too small */
/* done -- write block to stdout */
have = size - def.avail_out;
// if (fwrite(blk, 1, have, stdout) != have || ferror(stdout))
// quit("error writing output");
/* clean up and print results to stderr */
free(tmp);
ret = inflateEnd(&inf);
assert(ret != Z_STREAM_ERROR);
ret = deflateEnd(&def);
assert(ret != Z_STREAM_ERROR);
free(blk);
fprintf(stderr,
"%u bytes unused out of %u requested (%lu input)\n",
size - have, size, def.total_in);
return 0;
}

233
zlibWrapper/examples/fitblk_original.c Normal file
View File

@ -0,0 +1,233 @@
/* fitblk.c: example of fitting compressed output to a specified size
Not copyrighted -- provided to the public domain
Version 1.1 25 November 2004 Mark Adler */
/* Version history:
1.0 24 Nov 2004 First version
1.1 25 Nov 2004 Change deflateInit2() to deflateInit()
Use fixed-size, stack-allocated raw buffers
Simplify code moving compression to subroutines
Use assert() for internal errors
Add detailed description of approach
*/
/* Approach to just fitting a requested compressed size:
fitblk performs three compression passes on a portion of the input
data in order to determine how much of that input will compress to
nearly the requested output block size. The first pass generates
enough deflate blocks to produce output to fill the requested
output size plus a specfied excess amount (see the EXCESS define
below). The last deflate block may go quite a bit past that, but
is discarded. The second pass decompresses and recompresses just
the compressed data that fit in the requested plus excess sized
buffer. The deflate process is terminated after that amount of
input, which is less than the amount consumed on the first pass.
The last deflate block of the result will be of a comparable size
to the final product, so that the header for that deflate block and
the compression ratio for that block will be about the same as in
the final product. The third compression pass decompresses the
result of the second step, but only the compressed data up to the
requested size minus an amount to allow the compressed stream to
complete (see the MARGIN define below). That will result in a
final compressed stream whose length is less than or equal to the
requested size. Assuming sufficient input and a requested size
greater than a few hundred bytes, the shortfall will typically be
less than ten bytes.
If the input is short enough that the first compression completes
before filling the requested output size, then that compressed
stream is return with no recompression.
EXCESS is chosen to be just greater than the shortfall seen in a
two pass approach similar to the above. That shortfall is due to
the last deflate block compressing more efficiently with a smaller
header on the second pass. EXCESS is set to be large enough so
that there is enough uncompressed data for the second pass to fill
out the requested size, and small enough so that the final deflate
block of the second pass will be close in size to the final deflate
block of the third and final pass. MARGIN is chosen to be just
large enough to assure that the final compression has enough room
to complete in all cases.
*/
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include "zlib.h"
#define local static
/* print nastygram and leave */
local void quit(char *why)
{
fprintf(stderr, "fitblk abort: %s\n", why);
exit(1);
}
#define RAWLEN 4096 /* intermediate uncompressed buffer size */
/* compress from file to def until provided buffer is full or end of
input reached; return last deflate() return value, or Z_ERRNO if
there was read error on the file */
local int partcompress(FILE *in, z_streamp def)
{
int ret, flush;
unsigned char raw[RAWLEN];
flush = Z_NO_FLUSH;
do {
def->avail_in = fread(raw, 1, RAWLEN, in);
if (ferror(in))
return Z_ERRNO;
def->next_in = raw;
if (feof(in))
flush = Z_FINISH;
ret = deflate(def, flush);
assert(ret != Z_STREAM_ERROR);
} while (def->avail_out != 0 && flush == Z_NO_FLUSH);
return ret;
}
/* recompress from inf's input to def's output; the input for inf and
the output for def are set in those structures before calling;
return last deflate() return value, or Z_MEM_ERROR if inflate()
was not able to allocate enough memory when it needed to */
local int recompress(z_streamp inf, z_streamp def)
{
int ret, flush;
unsigned char raw[RAWLEN];
flush = Z_NO_FLUSH;
do {
/* decompress */
inf->avail_out = RAWLEN;
inf->next_out = raw;
ret = inflate(inf, Z_NO_FLUSH);
assert(ret != Z_STREAM_ERROR && ret != Z_DATA_ERROR &&
ret != Z_NEED_DICT);
if (ret == Z_MEM_ERROR)
return ret;
/* compress what was decompresed until done or no room */
def->avail_in = RAWLEN - inf->avail_out;
def->next_in = raw;
if (inf->avail_out != 0)
flush = Z_FINISH;
ret = deflate(def, flush);
assert(ret != Z_STREAM_ERROR);
} while (ret != Z_STREAM_END && def->avail_out != 0);
return ret;
}
#define EXCESS 256 /* empirically determined stream overage */
#define MARGIN 8 /* amount to back off for completion */
/* compress from stdin to fixed-size block on stdout */
int main(int argc, char **argv)
{
int ret; /* return code */
unsigned size; /* requested fixed output block size */
unsigned have; /* bytes written by deflate() call */
unsigned char *blk; /* intermediate and final stream */
unsigned char *tmp; /* close to desired size stream */
z_stream def, inf; /* zlib deflate and inflate states */
/* get requested output size */
if (argc != 2)
quit("need one argument: size of output block");
ret = strtol(argv[1], argv + 1, 10);
if (argv[1][0] != 0)
quit("argument must be a number");
if (ret < 8) /* 8 is minimum zlib stream size */
quit("need positive size of 8 or greater");
size = (unsigned)ret;
/* allocate memory for buffers and compression engine */
blk = malloc(size + EXCESS);
def.zalloc = Z_NULL;
def.zfree = Z_NULL;
def.opaque = Z_NULL;
ret = deflateInit(&def, Z_DEFAULT_COMPRESSION);
if (ret != Z_OK || blk == NULL)
quit("out of memory");
/* compress from stdin until output full, or no more input */
def.avail_out = size + EXCESS;
def.next_out = blk;
ret = partcompress(stdin, &def);
if (ret == Z_ERRNO)
quit("error reading input");
/* if it all fit, then size was undersubscribed -- done! */
if (ret == Z_STREAM_END && def.avail_out >= EXCESS) {
/* write block to stdout */
have = size + EXCESS - def.avail_out;
if (fwrite(blk, 1, have, stdout) != have || ferror(stdout))
quit("error writing output");
/* clean up and print results to stderr */
ret = deflateEnd(&def);
assert(ret != Z_STREAM_ERROR);
free(blk);
fprintf(stderr,
"%u bytes unused out of %u requested (all input)\n",
size - have, size);
return 0;
}
/* it didn't all fit -- set up for recompression */
inf.zalloc = Z_NULL;
inf.zfree = Z_NULL;
inf.opaque = Z_NULL;
inf.avail_in = 0;
inf.next_in = Z_NULL;
ret = inflateInit(&inf);
tmp = malloc(size + EXCESS);
if (ret != Z_OK || tmp == NULL)
quit("out of memory");
ret = deflateReset(&def);
assert(ret != Z_STREAM_ERROR);
/* do first recompression close to the right amount */
inf.avail_in = size + EXCESS;
inf.next_in = blk;
def.avail_out = size + EXCESS;
def.next_out = tmp;
ret = recompress(&inf, &def);
if (ret == Z_MEM_ERROR)
quit("out of memory");
/* set up for next reocmpression */
ret = inflateReset(&inf);
assert(ret != Z_STREAM_ERROR);
ret = deflateReset(&def);
assert(ret != Z_STREAM_ERROR);
/* do second and final recompression (third compression) */
inf.avail_in = size - MARGIN; /* assure stream will complete */
inf.next_in = tmp;
def.avail_out = size;
def.next_out = blk;
ret = recompress(&inf, &def);
if (ret == Z_MEM_ERROR)
quit("out of memory");
assert(ret == Z_STREAM_END); /* otherwise MARGIN too small */
/* done -- write block to stdout */
have = size - def.avail_out;
if (fwrite(blk, 1, have, stdout) != have || ferror(stdout))
quit("error writing output");
/* clean up and print results to stderr */
free(tmp);
ret = inflateEnd(&inf);
assert(ret != Z_STREAM_ERROR);
ret = deflateEnd(&def);
assert(ret != Z_STREAM_ERROR);
free(blk);
fprintf(stderr,
"%u bytes unused out of %u requested (%lu input)\n",
size - have, size, def.total_in);
return 0;
}

994
zlibWrapper/examples/zwrapbench.c Normal file
View File

@ -0,0 +1,994 @@
/**
* Copyright (c) 2016-present, Yann Collet, Przemyslaw Skibinski, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
/* *************************************
* Includes
***************************************/
#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_sleep */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* memset */
#include <stdio.h> /* fprintf, fopen, ftello64 */
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
#include <ctype.h> /* toupper */
#include "mem.h"
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
#include "datagen.h" /* RDG_genBuffer */
#include "xxhash.h"
#include "zstd_zlibwrapper.h"
/*-************************************
* Tuning parameters
**************************************/
#ifndef ZSTDCLI_CLEVEL_DEFAULT
# define ZSTDCLI_CLEVEL_DEFAULT 3
#endif
/*-************************************
* Constants
**************************************/
#define COMPRESSOR_NAME "Zstandard wrapper for zlib command line interface"
#ifndef ZSTD_VERSION
# define ZSTD_VERSION "v" ZSTD_VERSION_STRING
#endif
#define AUTHOR "Yann Collet"
#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR
#ifndef ZSTD_GIT_COMMIT
# define ZSTD_GIT_COMMIT_STRING ""
#else
# define ZSTD_GIT_COMMIT_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_GIT_COMMIT)
#endif
#define NBLOOPS 3
#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */
#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */
#define COOLPERIOD_SEC 10
#define KB *(1 <<10)
#define MB *(1 <<20)
#define GB *(1U<<30)
static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
static U32 g_compressibilityDefault = 50;
/* *************************************
* console display
***************************************/
#define DEFAULT_DISPLAY_LEVEL 2
#define DISPLAY(...) fprintf(displayOut, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static U32 g_displayLevel = DEFAULT_DISPLAY_LEVEL; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
static FILE* displayOut;
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \
{ g_time = clock(); DISPLAY(__VA_ARGS__); \
if (g_displayLevel>=4) fflush(stdout); } }
static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
static clock_t g_time = 0;
/* *************************************
* Exceptions
***************************************/
#ifndef DEBUG
# define DEBUG 0
#endif
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
#define EXM_THROW(error, ...) \
{ \
DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
DISPLAYLEVEL(1, "Error %i : ", error); \
DISPLAYLEVEL(1, __VA_ARGS__); \
DISPLAYLEVEL(1, "\n"); \
exit(error); \
}
/* *************************************
* Benchmark Parameters
***************************************/
static U32 g_nbIterations = NBLOOPS;
static size_t g_blockSize = 0;
int g_additionalParam = 0;
void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
void BMK_SetNbIterations(unsigned nbLoops)
{
g_nbIterations = nbLoops;
DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbIterations);
}
void BMK_SetBlockSize(size_t blockSize)
{
g_blockSize = blockSize;
DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10));
}
/* ********************************************************
* Bench functions
**********************************************************/
typedef struct
{
const char* srcPtr;
size_t srcSize;
char* cPtr;
size_t cRoom;
size_t cSize;
char* resPtr;
size_t resSize;
} blockParam_t;
typedef enum { BMK_ZSTD, BMK_ZSTD_STREAM, BMK_ZLIB, BMK_ZWRAP_ZLIB, BMK_ZWRAP_ZSTD, BMK_ZLIB_REUSE, BMK_ZWRAP_ZLIB_REUSE, BMK_ZWRAP_ZSTD_REUSE } BMK_compressor;
#define MIN(a,b) ((a)<(b) ? (a) : (b))
#define MAX(a,b) ((a)>(b) ? (a) : (b))
static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
const char* displayName, int cLevel,
const size_t* fileSizes, U32 nbFiles,
const void* dictBuffer, size_t dictBufferSize, BMK_compressor compressor)
{
size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
size_t const avgSize = MIN(g_blockSize, (srcSize / nbFiles));
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
void* const compressedBuffer = malloc(maxCompressedSize);
void* const resultBuffer = malloc(srcSize);
ZSTD_CCtx* const ctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
U32 nbBlocks;
UTIL_time_t ticksPerSecond;
/* checks */
if (!compressedBuffer || !resultBuffer || !blockTable || !ctx || !dctx)
EXM_THROW(31, "allocation error : not enough memory");
/* init */
if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */
UTIL_initTimer(&ticksPerSecond);
/* Init blockTable data */
{ const char* srcPtr = (const char*)srcBuffer;
char* cPtr = (char*)compressedBuffer;
char* resPtr = (char*)resultBuffer;
U32 fileNb;
for (nbBlocks=0, fileNb=0; fileNb<nbFiles; fileNb++) {
size_t remaining = fileSizes[fileNb];
U32 const nbBlocksforThisFile = (U32)((remaining + (blockSize-1)) / blockSize);
U32 const blockEnd = nbBlocks + nbBlocksforThisFile;
for ( ; nbBlocks<blockEnd; nbBlocks++) {
size_t const thisBlockSize = MIN(remaining, blockSize);
blockTable[nbBlocks].srcPtr = srcPtr;
blockTable[nbBlocks].cPtr = cPtr;
blockTable[nbBlocks].resPtr = resPtr;
blockTable[nbBlocks].srcSize = thisBlockSize;
blockTable[nbBlocks].cRoom = ZSTD_compressBound(thisBlockSize);
srcPtr += thisBlockSize;
cPtr += blockTable[nbBlocks].cRoom;
resPtr += thisBlockSize;
remaining -= thisBlockSize;
} } }
/* warmimg up memory */
RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
/* Bench */
{ U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL);
U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
UTIL_time_t coolTime;
U64 const maxTime = (g_nbIterations * TIMELOOP_MICROSEC) + 100;
U64 totalCTime=0, totalDTime=0;
U32 cCompleted=0, dCompleted=0;
# define NB_MARKS 4
const char* const marks[NB_MARKS] = { " |", " /", " =", "\\" };
U32 markNb = 0;
size_t cSize = 0;
double ratio = 0.;
UTIL_getTime(&coolTime);
DISPLAYLEVEL(2, "\r%79s\r", "");
while (!cCompleted | !dCompleted) {
UTIL_time_t clockStart;
U64 clockLoop = g_nbIterations ? TIMELOOP_MICROSEC : 1;
/* overheat protection */
if (UTIL_clockSpanMicro(coolTime, ticksPerSecond) > ACTIVEPERIOD_MICROSEC) {
DISPLAYLEVEL(2, "\rcooling down ... \r");
UTIL_sleep(COOLPERIOD_SEC);
UTIL_getTime(&coolTime);
}
/* Compression */
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */
UTIL_sleepMilli(1); /* give processor time to other processes */
UTIL_waitForNextTick(ticksPerSecond);
UTIL_getTime(&clockStart);
if (!cCompleted) { /* still some time to do compression tests */
U32 nbLoops = 0;
if (compressor == BMK_ZSTD) {
ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
ZSTD_customMem const cmem = { NULL, NULL, NULL };
ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem);
if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
do {
U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
size_t const rSize = ZSTD_compress_usingCDict(ctx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
cdict);
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s", ZSTD_getErrorName(rSize));
blockTable[blockNb].cSize = rSize;
}
nbLoops++;
} while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
ZSTD_freeCDict(cdict);
} else if (compressor == BMK_ZSTD_STREAM) {
ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
ZSTD_inBuffer inBuffer;
ZSTD_outBuffer outBuffer;
ZSTD_CStream* zbc = ZSTD_createCStream();
size_t rSize;
if (zbc == NULL) EXM_THROW(1, "ZSTD_createCStream() allocation failure");
rSize = ZSTD_initCStream_advanced(zbc, dictBuffer, dictBufferSize, zparams, avgSize);
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_initCStream_advanced() failed : %s", ZSTD_getErrorName(rSize));
do {
U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
rSize = ZSTD_resetCStream(zbc, blockTable[blockNb].srcSize);
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_resetCStream() failed : %s", ZSTD_getErrorName(rSize));
inBuffer.src = blockTable[blockNb].srcPtr;
inBuffer.size = blockTable[blockNb].srcSize;
inBuffer.pos = 0;
outBuffer.dst = blockTable[blockNb].cPtr;
outBuffer.size = blockTable[blockNb].cRoom;
outBuffer.pos = 0;
rSize = ZSTD_compressStream(zbc, &outBuffer, &inBuffer);
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compressStream() failed : %s", ZSTD_getErrorName(rSize));
rSize = ZSTD_endStream(zbc, &outBuffer);
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_endStream() failed : %s", ZSTD_getErrorName(rSize));
blockTable[blockNb].cSize = outBuffer.pos;
}
nbLoops++;
} while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
ZSTD_freeCStream(zbc);
} else if (compressor == BMK_ZWRAP_ZLIB_REUSE || compressor == BMK_ZWRAP_ZSTD_REUSE || compressor == BMK_ZLIB_REUSE) {
z_stream def;
int ret;
if (compressor == BMK_ZLIB_REUSE || compressor == BMK_ZWRAP_ZLIB_REUSE) ZWRAP_useZSTDcompression(0);
else ZWRAP_useZSTDcompression(1);
def.zalloc = Z_NULL;
def.zfree = Z_NULL;
def.opaque = Z_NULL;
ret = deflateInit(&def, cLevel);
if (ret != Z_OK) EXM_THROW(1, "deflateInit failure");
/* if (ZWRAP_isUsingZSTDcompression()) {
ret = ZWRAP_setPledgedSrcSize(&def, avgSize);
if (ret != Z_OK) EXM_THROW(1, "ZWRAP_setPledgedSrcSize failure");
} */
do {
U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
ret = deflateReset(&def);
if (ret != Z_OK) EXM_THROW(1, "deflateReset failure");
if (dictBuffer) {
ret = deflateSetDictionary(&def, dictBuffer, dictBufferSize);
if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure");
}
def.next_in = (const void*) blockTable[blockNb].srcPtr;
def.avail_in = blockTable[blockNb].srcSize;
def.total_in = 0;
def.next_out = (void*) blockTable[blockNb].cPtr;
def.avail_out = blockTable[blockNb].cRoom;
def.total_out = 0;
ret = deflate(&def, Z_FINISH);
if (ret != Z_STREAM_END) EXM_THROW(1, "deflate failure ret=%d srcSize=%d" , ret, (int)blockTable[blockNb].srcSize);
blockTable[blockNb].cSize = def.total_out;
}
nbLoops++;
} while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
ret = deflateEnd(&def);
if (ret != Z_OK) EXM_THROW(1, "deflateEnd failure");
} else {
z_stream def;
if (compressor == BMK_ZLIB || compressor == BMK_ZWRAP_ZLIB) ZWRAP_useZSTDcompression(0);
else ZWRAP_useZSTDcompression(1);
do {
U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
int ret;
def.zalloc = Z_NULL;
def.zfree = Z_NULL;
def.opaque = Z_NULL;
ret = deflateInit(&def, cLevel);
if (ret != Z_OK) EXM_THROW(1, "deflateInit failure");
if (dictBuffer) {
ret = deflateSetDictionary(&def, dictBuffer, dictBufferSize);
if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure");
}
def.next_in = (const void*) blockTable[blockNb].srcPtr;
def.avail_in = blockTable[blockNb].srcSize;
def.total_in = 0;
def.next_out = (void*) blockTable[blockNb].cPtr;
def.avail_out = blockTable[blockNb].cRoom;
def.total_out = 0;
ret = deflate(&def, Z_FINISH);
if (ret != Z_STREAM_END) EXM_THROW(1, "deflate failure");
ret = deflateEnd(&def);
if (ret != Z_OK) EXM_THROW(1, "deflateEnd failure");
blockTable[blockNb].cSize = def.total_out;
}
nbLoops++;
} while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
}
{ U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
if (clockSpan < fastestC*nbLoops) fastestC = clockSpan / nbLoops;
totalCTime += clockSpan;
cCompleted = totalCTime>maxTime;
} }
cSize = 0;
{ U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
ratio = (double)srcSize / (double)cSize;
markNb = (markNb+1) % NB_MARKS;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
(double)srcSize / fastestC );
(void)fastestD; (void)crcOrig; /* unused when decompression disabled */
#if 1
/* Decompression */
if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */
UTIL_sleepMilli(1); /* give processor time to other processes */
UTIL_waitForNextTick(ticksPerSecond);
UTIL_getTime(&clockStart);
if (!dCompleted) {
U32 nbLoops = 0;
if (compressor == BMK_ZSTD) {
ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictBufferSize);
if (!ddict) EXM_THROW(2, "ZSTD_createDDict() allocation failure");
do {
U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
size_t const regenSize = ZSTD_decompress_usingDDict(dctx,
blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
blockTable[blockNb].cPtr, blockTable[blockNb].cSize,
ddict);
if (ZSTD_isError(regenSize)) {
DISPLAY("ZSTD_decompress_usingDDict() failed on block %u : %s \n",
blockNb, ZSTD_getErrorName(regenSize));
clockLoop = 0; /* force immediate test end */
break;
}
blockTable[blockNb].resSize = regenSize;
}
nbLoops++;
} while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
ZSTD_freeDDict(ddict);
} else if (compressor == BMK_ZSTD_STREAM) {
ZSTD_inBuffer inBuffer;
ZSTD_outBuffer outBuffer;
ZSTD_DStream* zbd = ZSTD_createDStream();
size_t rSize;
if (zbd == NULL) EXM_THROW(1, "ZSTD_createDStream() allocation failure");
rSize = ZSTD_initDStream_usingDict(zbd, dictBuffer, dictBufferSize);
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_initDStream() failed : %s", ZSTD_getErrorName(rSize));
do {
U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
rSize = ZSTD_resetDStream(zbd);
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_resetDStream() failed : %s", ZSTD_getErrorName(rSize));
inBuffer.src = blockTable[blockNb].cPtr;
inBuffer.size = blockTable[blockNb].cSize;
inBuffer.pos = 0;
outBuffer.dst = blockTable[blockNb].resPtr;
outBuffer.size = blockTable[blockNb].srcSize;
outBuffer.pos = 0;
rSize = ZSTD_decompressStream(zbd, &outBuffer, &inBuffer);
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_decompressStream() failed : %s", ZSTD_getErrorName(rSize));
blockTable[blockNb].resSize = outBuffer.pos;
}
nbLoops++;
} while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
ZSTD_freeDStream(zbd);
} else if (compressor == BMK_ZWRAP_ZLIB_REUSE || compressor == BMK_ZWRAP_ZSTD_REUSE || compressor == BMK_ZLIB_REUSE) {
z_stream inf;
int ret;
if (compressor == BMK_ZLIB_REUSE) ZWRAP_setDecompressionType(ZWRAP_FORCE_ZLIB);
else ZWRAP_setDecompressionType(ZWRAP_AUTO);
inf.zalloc = Z_NULL;
inf.zfree = Z_NULL;
inf.opaque = Z_NULL;
ret = inflateInit(&inf);
if (ret != Z_OK) EXM_THROW(1, "inflateInit failure");
do {
U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
ret = inflateReset(&inf);
if (ret != Z_OK) EXM_THROW(1, "inflateReset failure");
inf.next_in = (const void*) blockTable[blockNb].cPtr;
inf.avail_in = blockTable[blockNb].cSize;
inf.total_in = 0;
inf.next_out = (void*) blockTable[blockNb].resPtr;
inf.avail_out = blockTable[blockNb].srcSize;
inf.total_out = 0;
ret = inflate(&inf, Z_FINISH);
if (ret == Z_NEED_DICT) {
ret = inflateSetDictionary(&inf, dictBuffer, dictBufferSize);
if (ret != Z_OK) EXM_THROW(1, "inflateSetDictionary failure");
ret = inflate(&inf, Z_FINISH);
}
if (ret != Z_STREAM_END) EXM_THROW(1, "inflate failure");
blockTable[blockNb].resSize = inf.total_out;
}
nbLoops++;
} while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
ret = inflateEnd(&inf);
if (ret != Z_OK) EXM_THROW(1, "inflateEnd failure");
} else {
z_stream inf;
if (compressor == BMK_ZLIB) ZWRAP_setDecompressionType(ZWRAP_FORCE_ZLIB);
else ZWRAP_setDecompressionType(ZWRAP_AUTO);
do {
U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
int ret;
inf.zalloc = Z_NULL;
inf.zfree = Z_NULL;
inf.opaque = Z_NULL;
ret = inflateInit(&inf);
if (ret != Z_OK) EXM_THROW(1, "inflateInit failure");
inf.next_in = (const void*) blockTable[blockNb].cPtr;
inf.avail_in = blockTable[blockNb].cSize;
inf.total_in = 0;
inf.next_out = (void*) blockTable[blockNb].resPtr;
inf.avail_out = blockTable[blockNb].srcSize;
inf.total_out = 0;
ret = inflate(&inf, Z_FINISH);
if (ret == Z_NEED_DICT) {
ret = inflateSetDictionary(&inf, dictBuffer, dictBufferSize);
if (ret != Z_OK) EXM_THROW(1, "inflateSetDictionary failure");
ret = inflate(&inf, Z_FINISH);
}
if (ret != Z_STREAM_END) EXM_THROW(1, "inflate failure");
ret = inflateEnd(&inf);
if (ret != Z_OK) EXM_THROW(1, "inflateEnd failure");
blockTable[blockNb].resSize = inf.total_out;
}
nbLoops++;
} while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
}
{ U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
if (clockSpan < fastestD*nbLoops) fastestD = clockSpan / nbLoops;
totalDTime += clockSpan;
dCompleted = totalDTime>maxTime;
} }
markNb = (markNb+1) % NB_MARKS;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
(double)srcSize / fastestC,
(double)srcSize / fastestD );
/* CRC Checking */
{ U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
if (crcOrig!=crcCheck) {
size_t u;
DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
for (u=0; u<srcSize; u++) {
if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u]) {
U32 segNb, bNb, pos;
size_t bacc = 0;
DISPLAY("Decoding error at pos %u ", (U32)u);
for (segNb = 0; segNb < nbBlocks; segNb++) {
if (bacc + blockTable[segNb].srcSize > u) break;
bacc += blockTable[segNb].srcSize;
}
pos = (U32)(u - bacc);
bNb = pos / (128 KB);
DISPLAY("(block %u, sub %u, pos %u) \n", segNb, bNb, pos);
break;
}
if (u==srcSize-1) { /* should never happen */
DISPLAY("no difference detected\n");
} }
break;
} } /* CRC Checking */
#endif
} /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */
if (g_displayLevel == 1) {
double cSpeed = (double)srcSize / fastestC;
double dSpeed = (double)srcSize / fastestD;
if (g_additionalParam)
DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam);
else
DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName);
}
DISPLAYLEVEL(2, "%2i#\n", cLevel);
} /* Bench */
/* clean up */
free(blockTable);
free(compressedBuffer);
free(resultBuffer);
ZSTD_freeCCtx(ctx);
ZSTD_freeDCtx(dctx);
return 0;
}
static size_t BMK_findMaxMem(U64 requiredMem)
{
size_t const step = 64 MB;
BYTE* testmem = NULL;
requiredMem = (((requiredMem >> 26) + 1) << 26);
requiredMem += step;
if (requiredMem > maxMemory) requiredMem = maxMemory;
do {
testmem = (BYTE*)malloc((size_t)requiredMem);
requiredMem -= step;
} while (!testmem);
free(testmem);
return (size_t)(requiredMem);
}
static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
const char* displayName, int cLevel, int cLevelLast,
const size_t* fileSizes, unsigned nbFiles,
const void* dictBuffer, size_t dictBufferSize)
{
int l;
const char* pch = strrchr(displayName, '\\'); /* Windows */
if (!pch) pch = strrchr(displayName, '/'); /* Linux */
if (pch) displayName = pch+1;
SET_HIGH_PRIORITY;
if (g_displayLevel == 1 && !g_additionalParam)
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
if (cLevelLast < cLevel) cLevelLast = cLevel;
DISPLAY("benchmarking zstd %s (using ZSTD_CCtx)\n", ZSTD_VERSION_STRING);
for (l=cLevel; l <= cLevelLast; l++) {
BMK_benchMem(srcBuffer, benchedSize,
displayName, l,
fileSizes, nbFiles,
dictBuffer, dictBufferSize, BMK_ZSTD);
}
DISPLAY("benchmarking zstd %s (using ZSTD_CStream)\n", ZSTD_VERSION_STRING);
for (l=cLevel; l <= cLevelLast; l++) {
BMK_benchMem(srcBuffer, benchedSize,
displayName, l,
fileSizes, nbFiles,
dictBuffer, dictBufferSize, BMK_ZSTD_STREAM);
}
DISPLAY("benchmarking zstd %s (using zlibWrapper)\n", ZSTD_VERSION_STRING);
for (l=cLevel; l <= cLevelLast; l++) {
BMK_benchMem(srcBuffer, benchedSize,
displayName, l,
fileSizes, nbFiles,
dictBuffer, dictBufferSize, BMK_ZWRAP_ZSTD_REUSE);
}
DISPLAY("benchmarking zstd %s (zlibWrapper not reusing a context)\n", ZSTD_VERSION_STRING);
for (l=cLevel; l <= cLevelLast; l++) {
BMK_benchMem(srcBuffer, benchedSize,
displayName, l,
fileSizes, nbFiles,
dictBuffer, dictBufferSize, BMK_ZWRAP_ZSTD);
}
if (cLevelLast > Z_BEST_COMPRESSION) cLevelLast = Z_BEST_COMPRESSION;
DISPLAY("\n");
DISPLAY("benchmarking zlib %s\n", ZLIB_VERSION);
for (l=cLevel; l <= cLevelLast; l++) {
BMK_benchMem(srcBuffer, benchedSize,
displayName, l,
fileSizes, nbFiles,
dictBuffer, dictBufferSize, BMK_ZLIB_REUSE);
}
DISPLAY("benchmarking zlib %s (zlib not reusing a context)\n", ZLIB_VERSION);
for (l=cLevel; l <= cLevelLast; l++) {
BMK_benchMem(srcBuffer, benchedSize,
displayName, l,
fileSizes, nbFiles,
dictBuffer, dictBufferSize, BMK_ZLIB);
}
DISPLAY("benchmarking zlib %s (using zlibWrapper)\n", ZLIB_VERSION);
for (l=cLevel; l <= cLevelLast; l++) {
BMK_benchMem(srcBuffer, benchedSize,
displayName, l,
fileSizes, nbFiles,
dictBuffer, dictBufferSize, BMK_ZWRAP_ZLIB_REUSE);
}
DISPLAY("benchmarking zlib %s (zlibWrapper not reusing a context)\n", ZLIB_VERSION);
for (l=cLevel; l <= cLevelLast; l++) {
BMK_benchMem(srcBuffer, benchedSize,
displayName, l,
fileSizes, nbFiles,
dictBuffer, dictBufferSize, BMK_ZWRAP_ZLIB);
}
}
/*! BMK_loadFiles() :
Loads `buffer` with content of files listed within `fileNamesTable`.
At most, fills `buffer` entirely */
static void BMK_loadFiles(void* buffer, size_t bufferSize,
size_t* fileSizes,
const char** fileNamesTable, unsigned nbFiles)
{
size_t pos = 0, totalSize = 0;
unsigned n;
for (n=0; n<nbFiles; n++) {
FILE* f;
U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
if (UTIL_isDirectory(fileNamesTable[n])) {
DISPLAYLEVEL(2, "Ignoring %s directory... \n", fileNamesTable[n]);
fileSizes[n] = 0;
continue;
}
f = fopen(fileNamesTable[n], "rb");
if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
pos += readSize; }
fileSizes[n] = (size_t)fileSize;
totalSize += (size_t)fileSize;
fclose(f);
}
if (totalSize == 0) EXM_THROW(12, "no data to bench");
}
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName, int cLevel, int cLevelLast)
{
void* srcBuffer;
size_t benchedSize;
void* dictBuffer = NULL;
size_t dictBufferSize = 0;
size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
char mfName[20] = {0};
if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");
/* Load dictionary */
if (dictFileName != NULL) {
U64 dictFileSize = UTIL_getFileSize(dictFileName);
if (dictFileSize > 64 MB) EXM_THROW(10, "dictionary file %s too large", dictFileName);
dictBufferSize = (size_t)dictFileSize;
dictBuffer = malloc(dictBufferSize);
if (dictBuffer==NULL) EXM_THROW(11, "not enough memory for dictionary (%u bytes)", (U32)dictBufferSize);
BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1);
}
/* Memory allocation & restrictions */
benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
if (benchedSize < totalSizeToLoad)
DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
srcBuffer = malloc(benchedSize);
if (!srcBuffer) EXM_THROW(12, "not enough memory");
/* Load input buffer */
BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
/* Bench */
snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
{ const char* displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
BMK_benchCLevel(srcBuffer, benchedSize,
displayName, cLevel, cLevelLast,
fileSizes, nbFiles,
dictBuffer, dictBufferSize);
}
/* clean up */
free(srcBuffer);
free(dictBuffer);
free(fileSizes);
}
static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility)
{
char name[20] = {0};
size_t benchedSize = 10000000;
void* const srcBuffer = malloc(benchedSize);
/* Memory allocation */
if (!srcBuffer) EXM_THROW(21, "not enough memory");
/* Fill input buffer */
RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
/* Bench */
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0);
/* clean up */
free(srcBuffer);
}
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName, int cLevel, int cLevelLast)
{
double const compressibility = (double)g_compressibilityDefault / 100;
if (nbFiles == 0)
BMK_syntheticTest(cLevel, cLevelLast, compressibility);
else
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast);
return 0;
}
/*-************************************
* Command Line
**************************************/
static int usage(const char* programName)
{
DISPLAY(WELCOME_MESSAGE);
DISPLAY( "Usage :\n");
DISPLAY( " %s [args] [FILE(s)] [-o file]\n", programName);
DISPLAY( "\n");
DISPLAY( "FILE : a filename\n");
DISPLAY( " with no FILE, or when FILE is - , read standard input\n");
DISPLAY( "Arguments :\n");
DISPLAY( " -D file: use `file` as Dictionary \n");
DISPLAY( " -h/-H : display help/long help and exit\n");
DISPLAY( " -V : display Version number and exit\n");
DISPLAY( " -v : verbose mode; specify multiple times to increase log level (default:%d)\n", DEFAULT_DISPLAY_LEVEL);
DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
#ifdef UTIL_HAS_CREATEFILELIST
DISPLAY( " -r : operate recursively on directories\n");
#endif
DISPLAY( "\n");
DISPLAY( "Benchmark arguments :\n");
DISPLAY( " -b# : benchmark file(s), using # compression level (default : %d) \n", ZSTDCLI_CLEVEL_DEFAULT);
DISPLAY( " -e# : test all compression levels from -bX to # (default: %d)\n", ZSTDCLI_CLEVEL_DEFAULT);
DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n");
DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n");
return 0;
}
static int badusage(const char* programName)
{
DISPLAYLEVEL(1, "Incorrect parameters\n");
if (g_displayLevel >= 1) usage(programName);
return 1;
}
static void waitEnter(void)
{
int unused;
DISPLAY("Press enter to continue...\n");
unused = getchar();
(void)unused;
}
/*! readU32FromChar() :
@return : unsigned integer value reach from input in `char` format
Will also modify `*stringPtr`, advancing it to position where it stopped reading.
Note : this function can overflow if digit string > MAX_UINT */
static unsigned readU32FromChar(const char** stringPtr)
{
unsigned result = 0;
while ((**stringPtr >='0') && (**stringPtr <='9'))
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
return result;
}
#define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
int main(int argCount, char** argv)
{
int argNb,
main_pause=0,
nextEntryIsDictionary=0,
operationResult=0,
nextArgumentIsFile=0;
int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
int cLevelLast = 1;
unsigned recursive = 0;
const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */
unsigned filenameIdx = 0;
const char* programName = argv[0];
const char* dictFileName = NULL;
char* dynNameSpace = NULL;
#ifdef UTIL_HAS_CREATEFILELIST
const char** fileNamesTable = NULL;
char* fileNamesBuf = NULL;
unsigned fileNamesNb;
#endif
/* init */
if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
displayOut = stderr;
/* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */
{ size_t pos;
for (pos = (int)strlen(programName); pos > 0; pos--) { if (programName[pos] == '/') { pos++; break; } }
programName += pos;
}
/* command switches */
for(argNb=1; argNb<argCount; argNb++) {
const char* argument = argv[argNb];
if(!argument) continue; /* Protection if argument empty */
if (nextArgumentIsFile==0) {
/* long commands (--long-word) */
if (!strcmp(argument, "--")) { nextArgumentIsFile=1; continue; }
if (!strcmp(argument, "--version")) { displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); }
if (!strcmp(argument, "--help")) { displayOut=stdout; CLEAN_RETURN(usage(programName)); }
if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; }
if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; }
/* Decode commands (note : aggregated commands are allowed) */
if (argument[0]=='-') {
argument++;
while (argument[0]!=0) {
switch(argument[0])
{
/* Display help */
case 'V': displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); /* Version Only */
case 'H':
case 'h': displayOut=stdout; CLEAN_RETURN(usage(programName));
/* Use file content as dictionary */
case 'D': nextEntryIsDictionary = 1; argument++; break;
/* Verbose mode */
case 'v': g_displayLevel++; argument++; break;
/* Quiet mode */
case 'q': g_displayLevel--; argument++; break;
#ifdef UTIL_HAS_CREATEFILELIST
/* recursive */
case 'r': recursive=1; argument++; break;
#endif
/* Benchmark */
case 'b':
/* first compression Level */
argument++;
cLevel = readU32FromChar(&argument);
break;
/* range bench (benchmark only) */
case 'e':
/* last compression Level */
argument++;
cLevelLast = readU32FromChar(&argument);
break;
/* Modify Nb Iterations (benchmark only) */
case 'i':
argument++;
{ U32 const iters = readU32FromChar(&argument);
BMK_setNotificationLevel(g_displayLevel);
BMK_SetNbIterations(iters);
}
break;
/* cut input into blocks (benchmark only) */
case 'B':
argument++;
{ size_t bSize = readU32FromChar(&argument);
if (toupper(*argument)=='K') bSize<<=10, argument++; /* allows using KB notation */
if (toupper(*argument)=='M') bSize<<=20, argument++;
if (toupper(*argument)=='B') argument++;
BMK_setNotificationLevel(g_displayLevel);
BMK_SetBlockSize(bSize);
}
break;
/* Pause at the end (-p) or set an additional param (-p#) (hidden option) */
case 'p': argument++;
if ((*argument>='0') && (*argument<='9')) {
BMK_setAdditionalParam(readU32FromChar(&argument));
} else
main_pause=1;
break;
/* unknown command */
default : CLEAN_RETURN(badusage(programName));
}
}
continue;
} /* if (argument[0]=='-') */
} /* if (nextArgumentIsAFile==0) */
if (nextEntryIsDictionary) {
nextEntryIsDictionary = 0;
dictFileName = argument;
continue;
}
/* add filename to list */
filenameTable[filenameIdx++] = argument;
}
/* Welcome message (if verbose) */
DISPLAYLEVEL(3, WELCOME_MESSAGE);
#ifdef UTIL_HAS_CREATEFILELIST
if (recursive) {
fileNamesTable = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb);
if (fileNamesTable) {
unsigned u;
for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, fileNamesTable[u]);
free((void*)filenameTable);
filenameTable = fileNamesTable;
filenameIdx = fileNamesNb;
}
}
#endif
BMK_setNotificationLevel(g_displayLevel);
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast);
_end:
if (main_pause) waitEnter();
free(dynNameSpace);
#ifdef UTIL_HAS_CREATEFILELIST
if (fileNamesTable)
UTIL_freeFileList(fileNamesTable, fileNamesBuf);
else
#endif
free((void*)filenameTable);
return operationResult;
}

777
zlibWrapper/zstd_zlibwrapper.c
View File

File diff suppressed because it is too large Load Diff

31
zlibWrapper/zstd_zlibwrapper.h
View File

@ -26,11 +26,38 @@ extern "C" {
#endif #endif
#endif #endif
void useZSTD(int turn_on); /* returns a string with version of zstd library */
int isUsingZSTD(void);
const char * zstdVersion(void); const char * zstdVersion(void);
/* COMPRESSION */
/* enables/disables zstd compression during runtime */
void ZWRAP_useZSTDcompression(int turn_on);
/* check if zstd compression is turned on */
int ZWRAP_isUsingZSTDcompression(void);
/* Changes a pledged source size for a given compression stream.
It will change ZSTD compression parameters what may improve compression speed and/or ratio.
The function should be called just after deflateInit(). It's only helpful when data is compressed in blocks.
There will be no change in case of deflateInit() immediately followed by deflate(strm, Z_FINISH)
as this case is automatically detected. */
int ZWRAP_setPledgedSrcSize(z_streamp strm, unsigned long long pledgedSrcSize);
/* DECOMPRESSION */
typedef enum { ZWRAP_FORCE_ZLIB, ZWRAP_AUTO } ZWRAP_decompress_type;
/* enables/disables automatic recognition of zstd/zlib compressed data during runtime */
void ZWRAP_setDecompressionType(ZWRAP_decompress_type type);
/* check zstd decompression type */
ZWRAP_decompress_type ZWRAP_getDecompressionType(void);
#if defined (__cplusplus) #if defined (__cplusplus)
} }
#endif #endif