From 13f42d9085832d5988551dcacac6d4f25dd0ba06 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 1 Jun 2016 14:44:31 +0200 Subject: [PATCH 01/22] VS2010 project: reverted zstdlib.rc --- Makefile | 79 ++++++++++++++----------- lib/decompress/zstd_decompress.c | 3 +- projects/VS2010/zstdlib/zstdlib.rc | 51 ++++++++++++++++ projects/VS2010/zstdlib/zstdlib.vcxproj | 4 +- 4 files changed, 102 insertions(+), 35 deletions(-) create mode 100644 projects/VS2010/zstdlib/zstdlib.rc diff --git a/Makefile b/Makefile index 3c600f54..db58d430 100644 --- a/Makefile +++ b/Makefile @@ -68,8 +68,9 @@ clean: #------------------------------------------------------------------------ #make install is validated only for Linux, OSX, kFreeBSD and Hurd targets +#------------------------------------------------------------------------ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU)) - +HOST_OS = POSIX install: $(MAKE) -C $(ZSTDDIR) $@ $(MAKE) -C $(PRGDIR) $@ @@ -81,43 +82,13 @@ uninstall: travis-install: $(MAKE) install PREFIX=~/install_test_dir -cmaketest: - cmake --version - rm -rf projects/cmake/build - mkdir projects/cmake/build - cd projects/cmake/build ; cmake .. ; $(MAKE) +gpptest: clean + $(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" clangtest: clean clang -v $(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion" -gpptest: clean - $(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" - -c90test: clean - CFLAGS="-std=c90" $(MAKE) all # will fail, due to // and long long - -gnu90test: clean - CFLAGS="-std=gnu90" $(MAKE) all - -c99test: clean - CFLAGS="-std=c99" $(MAKE) all - -gnu99test: clean - CFLAGS="-std=gnu99" $(MAKE) all - -c11test: clean - CFLAGS="-std=c11" $(MAKE) all - -bmix64test: clean - CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(PRGDIR) test - -bmix32test: clean - CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(PRGDIR) test - -bmi32test: clean - CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(PRGDIR) test - armtest: clean $(MAKE) -C $(PRGDIR) datagen # use native, faster $(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static" @@ -168,3 +139,45 @@ uasan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined" endif + + +ifneq (,$(filter MSYS%,$(shell uname))) +HOST_OS = MSYS +CMAKE_PARAMS = -G"MSYS Makefiles" +endif + + +#------------------------------------------------------------------------ +#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets +#------------------------------------------------------------------------ +ifneq (,$(filter $(HOST_OS),MSYS POSIX)) +cmaketest: + cmake --version + rm -rf projects/cmake/build + mkdir projects/cmake/build + cd projects/cmake/build ; cmake $(CMAKE_PARAMS) .. ; $(MAKE) + +c90test: clean + CFLAGS="-std=c90" $(MAKE) all # will fail, due to // and long long + +gnu90test: clean + CFLAGS="-std=gnu90" $(MAKE) all + +c99test: clean + CFLAGS="-std=c99" $(MAKE) all + +gnu99test: clean + CFLAGS="-std=gnu99" $(MAKE) all + +c11test: clean + CFLAGS="-std=c11" $(MAKE) all + +bmix64test: clean + CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(PRGDIR) test + +bmix32test: clean + CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(PRGDIR) test + +bmi32test: clean + CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(PRGDIR) test +endif diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 6d79e678..e88b873c 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -333,7 +333,8 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) { if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */ - fparamsPtr->frameContentSize = 0; + memset(fparamsPtr, 0, sizeof(*fparamsPtr)); + fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4); fparamsPtr->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; return 0; } diff --git a/projects/VS2010/zstdlib/zstdlib.rc b/projects/VS2010/zstdlib/zstdlib.rc new file mode 100644 index 00000000..6c4dde48 --- /dev/null +++ b/projects/VS2010/zstdlib/zstdlib.rc @@ -0,0 +1,51 @@ +// Microsoft Visual C++ generated resource script. +// + +#include "zstd.h" /* ZSTD_VERSION_STRING */ +#define APSTUDIO_READONLY_SYMBOLS +#include "verrsrc.h" +#undef APSTUDIO_READONLY_SYMBOLS + + +#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) +LANGUAGE 9, 1 + +///////////////////////////////////////////////////////////////////////////// +// +// Version +// + +VS_VERSION_INFO VERSIONINFO + FILEVERSION ZSTD_LIB_VERSION + PRODUCTVERSION ZSTD_LIB_VERSION + FILEFLAGSMASK VS_FFI_FILEFLAGSMASK +#ifdef _DEBUG + FILEFLAGS VS_FF_DEBUG +#else + FILEFLAGS 0x0L +#endif + FILEOS VOS_NT_WINDOWS32 + FILETYPE VFT_DLL + FILESUBTYPE VFT2_UNKNOWN +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904B0" + BEGIN + VALUE "CompanyName", "Yann Collet" + VALUE "FileDescription", "Fast and efficient compression algorithm" + VALUE "FileVersion", ZSTD_VERSION_STRING + VALUE "InternalName", "zstdlib.dll" + VALUE "LegalCopyright", "Copyright (C) 2013-2015, Yann Collet" + VALUE "OriginalFilename", "zstdlib.dll" + VALUE "ProductName", "Zstandard" + VALUE "ProductVersion", ZSTD_VERSION_STRING + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x0409, 1200 + END +END + +#endif diff --git a/projects/VS2010/zstdlib/zstdlib.vcxproj b/projects/VS2010/zstdlib/zstdlib.vcxproj index 79900501..e0e3dbb6 100644 --- a/projects/VS2010/zstdlib/zstdlib.vcxproj +++ b/projects/VS2010/zstdlib/zstdlib.vcxproj @@ -48,7 +48,9 @@ - + + + {8BFD8150-94D5-4BF9-8A50-7BD9929A0850} From 9242816b5640f99547dd731f0e947b67eafa113e Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 1 Jun 2016 18:47:04 +0200 Subject: [PATCH 02/22] fparamsPtr->windowLog==0 means that a frame is skippable --- lib/common/zstd_static.h | 9 +++++++++ lib/decompress/zbuff_decompress.c | 2 ++ lib/decompress/zstd_decompress.c | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h index e4c992be..19118f46 100644 --- a/lib/common/zstd_static.h +++ b/lib/common/zstd_static.h @@ -246,6 +246,15 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. Context can then be reset to start a new decompression. + + Skippable frames allow the integration of user-defined data into a flow of concatenated frames. + Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frame is following: + a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F + b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits + c) Frame Content - any content (User Data) of length equal to Frame Size + For skippable frames ZSTD_decompressContinue() always returns 0. + For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable. + It also returns Frame Size as fparamsPtr->frameContentSize. */ diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index 72456fd5..69898fdb 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -192,6 +192,8 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, if (ZSTD_isError(h2Result)) return h2Result; } } + if (zbd->fParams.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) zbd->fParams.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for buffer allocation */ + /* Frame header instruct buffer sizes */ { size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTD_BLOCKSIZE_MAX); zbd->blockSize = blockSize; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index e88b873c..27dd780a 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -335,7 +335,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */ memset(fparamsPtr, 0, sizeof(*fparamsPtr)); fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4); - fparamsPtr->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; + fparamsPtr->windowLog = 0; /* windowLog==0 means a frame is skippable */ return 0; } return ERROR(prefix_unknown); From bf25d7ac574bac5da3e1fbdc4dc6273392d4daf2 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 2 Jun 2016 10:19:35 +0200 Subject: [PATCH 03/22] zlibWrapper: support for zlib versions from 1.2.3 to 1.2.8 --- .travis.yml | 12 ++--- zlibWrapper/Makefile | 9 ++-- zlibWrapper/zstd_zlibwrapper.c | 84 +++++++++++++++++++++------------- zlibWrapper/zstd_zlibwrapper.h | 6 ++- 4 files changed, 64 insertions(+), 47 deletions(-) diff --git a/.travis.yml b/.travis.yml index e6a32c25..41f67c77 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,9 @@ matrix: - os: linux sudo: false env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=asan + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=zlibwrapper # Standard Ubuntu 12.04 LTS Server Edition 64 bit - os: linux sudo: required @@ -53,15 +56,6 @@ matrix: - os: linux sudo: required env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs valgrindTest" - - os: linux - sudo: required - env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu90test - - os: linux - sudo: required - env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=c99test - #- os: linux - # sudo: required - # env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=zlibwrapper # Ubuntu 14.04 LTS Server Edition 64 bit - os: linux dist: trusty diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index 3f293cd9..6c00622b 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -7,8 +7,9 @@ # Paths to static and dynamic zlib and zstd libraries ifneq (,$(filter Windows%,$(OS))) -STATICLIB = ../../zlib/libz.a ../lib/libzstd.a -IMPLIB = ../../zlib/libz.dll.a ../lib/libzstd.a +ZLIBDIR = ../../zlib1.2.8 +STATICLIB = $(ZLIBDIR)/libz.a ../lib/libzstd.a +IMPLIB = $(ZLIBDIR)/libz.dll.a ../lib/libzstd.a else STATICLIB = -static -lz ../lib/libzstd.a IMPLIB = -lz ../lib/libzstd.a @@ -17,12 +18,12 @@ endif ZLIBWRAPPER_PATH = . EXAMPLE_PATH = examples CC = gcc -CFLAGS = $(LOC) -I../lib/common -I$(ZLIBWRAPPER_PATH) -O3 -Wall -std=gnu89 +CFLAGS = $(LOC) -I../lib/common -I$(ZLIBDIR) -I$(ZLIBWRAPPER_PATH) -O3 -Wall -std=gnu89 LDFLAGS = $(LOC) RM = rm -f -all: clean test testdll testzstd +all: clean test testzstd test: example ./example diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 7ec68a4c..c9a8b1d1 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -31,6 +31,7 @@ #include /* fprintf */ #include /* malloc */ +#include /* va_list */ #include #include "zstd_zlibwrapper.h" #include "zstd.h" @@ -492,6 +493,7 @@ ZEXTERN int ZEXPORT z_deflateTune OF((z_streamp strm, } +#if ZLIB_VERNUM >= 0x1260 ZEXTERN int ZEXPORT z_deflatePending OF((z_streamp strm, unsigned *pending, int *bits)) @@ -500,6 +502,7 @@ ZEXTERN int ZEXPORT z_deflatePending OF((z_streamp strm, return deflatePending(strm, pending, bits); FINISH_WITH_ERR("deflatePending is not supported!"); } +#endif ZEXTERN int ZEXPORT z_deflatePrime OF((z_streamp strm, @@ -524,6 +527,7 @@ ZEXTERN int ZEXPORT z_deflateSetHeader OF((z_streamp strm, /* Advanced compression functions */ +#if ZLIB_VERNUM >= 0x1280 ZEXTERN int ZEXPORT z_inflateGetDictionary OF((z_streamp strm, Bytef *dictionary, uInt *dictLength)) @@ -532,7 +536,7 @@ ZEXTERN int ZEXPORT z_inflateGetDictionary OF((z_streamp strm, return inflateGetDictionary(strm, dictionary, dictLength); FINISH_WITH_ERR("inflateGetDictionary is not supported!"); } - +#endif ZEXTERN int ZEXPORT z_inflateCopy OF((z_streamp dest, @@ -552,6 +556,7 @@ ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm)) } +#if ZLIB_VERNUM >= 0x1240 ZEXTERN int ZEXPORT z_inflateReset2 OF((z_streamp strm, int windowBits)) { @@ -559,6 +564,17 @@ ZEXTERN int ZEXPORT z_inflateReset2 OF((z_streamp strm, return inflateReset2(strm, windowBits); FINISH_WITH_ERR("inflateReset2 is not supported!"); } +#endif + + +#if ZLIB_VERNUM >= 0x1240 +ZEXTERN long ZEXPORT z_inflateMark OF((z_streamp strm)) +{ + if (!strm->reserved) + return inflateMark(strm); + FINISH_WITH_ERR("inflateMark is not supported!"); +} +#endif ZEXTERN int ZEXPORT z_inflatePrime OF((z_streamp strm, @@ -571,14 +587,6 @@ ZEXTERN int ZEXPORT z_inflatePrime OF((z_streamp strm, } -ZEXTERN long ZEXPORT z_inflateMark OF((z_streamp strm)) -{ - if (!strm->reserved) - return inflateMark(strm); - FINISH_WITH_ERR("inflateMark is not supported!"); -} - - ZEXTERN int ZEXPORT z_inflateGetHeader OF((z_streamp strm, gz_headerp head)) { @@ -699,6 +707,7 @@ ZEXTERN gzFile ZEXPORT z_gzdopen OF((int fd, const char *mode)) } +#if ZLIB_VERNUM >= 0x1240 ZEXTERN int ZEXPORT z_gzbuffer OF((gzFile file, unsigned size)) { if (!g_useZSTD) @@ -707,6 +716,31 @@ ZEXTERN int ZEXPORT z_gzbuffer OF((gzFile file, unsigned size)) } +ZEXTERN z_off_t ZEXPORT z_gzoffset OF((gzFile file)) +{ + if (!g_useZSTD) + return gzoffset(file); + FINISH_WITH_ERR("gzoffset is not supported!"); +} + + +ZEXTERN int ZEXPORT z_gzclose_r OF((gzFile file)) +{ + if (!g_useZSTD) + return gzclose_r(file); + FINISH_WITH_ERR("gzclose_r is not supported!"); +} + + +ZEXTERN int ZEXPORT z_gzclose_w OF((gzFile file)) +{ + if (!g_useZSTD) + return gzclose_w(file); + FINISH_WITH_ERR("gzclose_w is not supported!"); +} +#endif + + ZEXTERN int ZEXPORT z_gzsetparams OF((gzFile file, int level, int strategy)) { if (!g_useZSTD) @@ -732,7 +766,11 @@ ZEXTERN int ZEXPORT z_gzwrite OF((gzFile file, } +#if ZLIB_VERNUM >= 0x1260 ZEXTERN int ZEXPORTVA z_gzprintf Z_ARG((gzFile file, const char *format, ...)) +#else +ZEXTERN int ZEXPORTVA z_gzprintf OF((gzFile file, const char *format, ...)) +#endif { if (!g_useZSTD) { int ret; @@ -774,7 +812,11 @@ ZEXTERN int ZEXPORT z_gzputc OF((gzFile file, int c)) } +#if ZLIB_VERNUM == 0x1260 +ZEXTERN int ZEXPORT z_gzgetc_ OF((gzFile file)) +#else ZEXTERN int ZEXPORT z_gzgetc OF((gzFile file)) +#endif { if (!g_useZSTD) return gzgetc(file); @@ -822,14 +864,6 @@ ZEXTERN z_off_t ZEXPORT z_gztell OF((gzFile file)) } -ZEXTERN z_off_t ZEXPORT z_gzoffset OF((gzFile file)) -{ - if (!g_useZSTD) - return gzoffset(file); - FINISH_WITH_ERR("gzoffset is not supported!"); -} - - ZEXTERN int ZEXPORT z_gzeof OF((gzFile file)) { if (!g_useZSTD) @@ -854,22 +888,6 @@ ZEXTERN int ZEXPORT z_gzclose OF((gzFile file)) } -ZEXTERN int ZEXPORT z_gzclose_r OF((gzFile file)) -{ - if (!g_useZSTD) - return gzclose_r(file); - FINISH_WITH_ERR("gzclose_r is not supported!"); -} - - -ZEXTERN int ZEXPORT z_gzclose_w OF((gzFile file)) -{ - if (!g_useZSTD) - return gzclose_w(file); - FINISH_WITH_ERR("gzclose_w is not supported!"); -} - - ZEXTERN const char * ZEXPORT z_gzerror OF((gzFile file, int *errnum)) { if (!g_useZSTD) diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h index c2c908dd..e438789a 100644 --- a/zlibWrapper/zstd_zlibwrapper.h +++ b/zlibWrapper/zstd_zlibwrapper.h @@ -41,7 +41,11 @@ extern "C" { #include #if !defined(z_const) -# define z_const const +#if ZLIB_VERNUM >= 0x1260 + #define z_const const +#else + #define z_const +#endif #endif void useZSTD(int turn_on); From e02bf99fa4ec7f93b27204a33c545e1a7f18250b Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 2 Jun 2016 12:00:32 +0200 Subject: [PATCH 04/22] zlibWrapper: fixed memory deallocation bug with zstd decompression --- zlibWrapper/zstd_zlibwrapper.c | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index c9a8b1d1..ff85a861 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -125,7 +125,7 @@ ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } zwc->compressionLevel = level; - strm->opaque = (voidpf) zwc; + strm->state = (struct internal_state*) zwc; /* use state which in not used by user */ strm->total_in = 0; strm->total_out = 0; return Z_OK; @@ -151,7 +151,7 @@ ZEXTERN int ZEXPORT z_deflateSetDictionary OF((z_streamp strm, if (!g_useZSTD) return deflateSetDictionary(strm, dictionary, dictLength); - { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->opaque; + { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; LOG_WRAPPER("- deflateSetDictionary level=%d\n", (int)strm->data_type); { size_t const errorCode = ZBUFF_compressInitDictionary(zwc->zbc, dictionary, dictLength, zwc->compressionLevel); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } @@ -171,7 +171,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) return res; } - zwc = (ZWRAP_CCtx*) strm->opaque; + zwc = (ZWRAP_CCtx*) strm->state; LOG_WRAPPER("deflate flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); if (strm->avail_in > 0) { @@ -220,7 +220,7 @@ ZEXTERN int ZEXPORT z_deflateEnd OF((z_streamp strm)) return deflateEnd(strm); } LOG_WRAPPER("- deflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out)); - { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->opaque; + { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; size_t const errorCode = ZWRAP_freeCCtx(zwc); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } @@ -265,9 +265,6 @@ typedef struct { int stream_size; char *version; int windowBits; - alloc_func zalloc; /* used to allocate the internal state */ - free_func zfree; /* used to free the internal state */ - voidpf opaque; /* private data object passed to zalloc and zfree */ } ZWRAP_DCtx; @@ -299,11 +296,8 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, zwd->stream_size = stream_size; zwd->version = strdup(version); - zwd->zalloc = strm->zalloc; - zwd->zfree = strm->zfree; - zwd->opaque = strm->opaque; - strm->state = (struct internal_state*) zwd; + strm->state = (struct internal_state*) zwd; /* use state which in not used by user */ strm->total_in = 0; strm->total_out = 0; strm->reserved = 1; /* mark as unknown steam */ @@ -399,7 +393,11 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) strm->avail_in = strm2.avail_in; strm->next_out = strm2.next_out; strm->avail_out = strm2.avail_out; + strm->reserved = 0; /* mark as zlib stream */ + { size_t const errorCode = ZWRAP_freeDCtx(zwd); + if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } + if (flush == Z_INFLATE_SYNC) return inflateSync(strm); return inflate(strm, flush); } @@ -444,7 +442,7 @@ ZEXTERN int ZEXPORT z_inflateEnd OF((z_streamp strm)) { int ret = Z_OK; if (!strm->reserved) - ret = inflateEnd(strm); + return inflateEnd(strm); LOG_WRAPPER("- inflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out)); { ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; From 2866951558d8038dc872d4811f22cd0756c0d506 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 2 Jun 2016 13:04:18 +0200 Subject: [PATCH 05/22] opaque parameter for custom memory allocation functions --- lib/common/zstd_common.c | 18 ++++++++++++++++++ lib/common/zstd_static.h | 11 ++++++++--- lib/compress/zbuff_compress.c | 27 ++++++++++++--------------- lib/compress/zstd_compress.c | 31 ++++++++++++------------------- lib/decompress/zbuff_decompress.c | 27 ++++++++++++--------------- lib/decompress/zstd_decompress.c | 17 ++++++----------- programs/zbufftest.c | 14 ++++++++------ 7 files changed, 76 insertions(+), 69 deletions(-) diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c index 2a2c39d6..c4ec46fb 100644 --- a/lib/common/zstd_common.c +++ b/lib/common/zstd_common.c @@ -33,6 +33,7 @@ /*-************************************* * Dependencies ***************************************/ +#include /* malloc */ #include "error_private.h" #include "zstd_static.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode */ #include "zbuff.h" /* declaration of ZBUFF_isError, ZBUFF_getErrorName */ @@ -70,3 +71,20 @@ const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(c unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } + + + +void* ZSTD_defaultAllocFunction(void* opaque, size_t size) +{ + (void)opaque; + void* address = malloc(size); + /* DISPLAYLEVEL(4, "alloc %p, %d opaque=%d \n", address, (int)size, (int)opaque); */ + return address; +} + +void ZSTD_defaultFreeFunction(void* opaque, void* address) +{ + (void)opaque; + /* if (address) DISPLAYLEVEL(4, "free %p opaque=%d \n", address, (int)opaque); */ + free(address); +} diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h index 19118f46..b36e92c8 100644 --- a/lib/common/zstd_static.h +++ b/lib/common/zstd_static.h @@ -97,9 +97,14 @@ typedef struct { ZSTD_frameParameters fParams; } ZSTD_parameters; -typedef void* (*ZSTD_allocFunction) (size_t size); -typedef void (*ZSTD_freeFunction) (void* address); -typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; } ZSTD_customMem; +/* custom memory allocation functions */ +typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); +typedef void (*ZSTD_freeFunction) (void* opaque, void* address); +typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; + +void* ZSTD_defaultAllocFunction(void* opaque, size_t size); +void ZSTD_defaultFreeFunction(void* opaque, void* address); +static ZSTD_customMem const defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL }; /*-************************************* diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index 66deb495..97d87b99 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -95,13 +95,12 @@ struct ZBUFF_CCtx_s { size_t outBuffContentSize; size_t outBuffFlushedSize; ZBUFF_cStage stage; - ZSTD_allocFunction customAlloc; - ZSTD_freeFunction customFree; + ZSTD_customMem customMem; }; /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */ ZBUFF_CCtx* ZBUFF_createCCtx(void) { - ZSTD_customMem customMem = { NULL, NULL }; + ZSTD_customMem const customMem = { NULL, NULL, NULL }; return ZBUFF_createCCtx_advanced(customMem); } @@ -113,8 +112,7 @@ ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem) { zbc = (ZBUFF_CCtx*)calloc(1, sizeof(ZBUFF_CCtx)); if (zbc==NULL) return NULL; - zbc->customAlloc = malloc; - zbc->customFree = free; + memcpy(&zbc->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); zbc->zc = ZSTD_createCCtx(); return zbc; } @@ -122,11 +120,10 @@ ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem) if (!customMem.customAlloc || !customMem.customFree) return NULL; - zbc = (ZBUFF_CCtx*)customMem.customAlloc(sizeof(ZBUFF_CCtx)); + zbc = (ZBUFF_CCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_CCtx)); if (zbc==NULL) return NULL; memset(zbc, 0, sizeof(ZBUFF_CCtx)); - zbc->customAlloc = customMem.customAlloc; - zbc->customFree = customMem.customFree; + memcpy(&zbc->customMem, &customMem, sizeof(ZSTD_customMem)); zbc->zc = ZSTD_createCCtx_advanced(customMem); return zbc; } @@ -135,9 +132,9 @@ size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc) { if (zbc==NULL) return 0; /* support free on NULL */ ZSTD_freeCCtx(zbc->zc); - zbc->customFree(zbc->inBuff); - zbc->customFree(zbc->outBuff); - zbc->customFree(zbc); + zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff); + zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff); + zbc->customMem.customFree(zbc->customMem.opaque, zbc); return 0; } @@ -152,16 +149,16 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; if (zbc->inBuffSize < neededInBuffSize) { zbc->inBuffSize = neededInBuffSize; - zbc->customFree(zbc->inBuff); /* should not be necessary */ - zbc->inBuff = (char*)zbc->customAlloc(neededInBuffSize); + zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff); /* should not be necessary */ + zbc->inBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, neededInBuffSize); if (zbc->inBuff == NULL) return ERROR(memory_allocation); } zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize/2); } if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) { zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1; - zbc->customFree(zbc->outBuff); /* should not be necessary */ - zbc->outBuff = (char*)zbc->customAlloc(zbc->outBuffSize); + zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff); /* should not be necessary */ + zbc->outBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, zbc->outBuffSize); if (zbc->outBuff == NULL) return ERROR(memory_allocation); } diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index a9a2dc79..eb2df3b4 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -107,8 +107,7 @@ struct ZSTD_CCtx_s size_t workSpaceSize; size_t blockSize; XXH64_state_t xxhState; - ZSTD_allocFunction customAlloc; - ZSTD_freeFunction customFree; + ZSTD_customMem customMem; seqStore_t seqStore; /* sequences storage ptrs */ U32* hashTable; @@ -123,7 +122,7 @@ struct ZSTD_CCtx_s ZSTD_CCtx* ZSTD_createCCtx(void) { - ZSTD_customMem customMem = { NULL, NULL }; + ZSTD_customMem const customMem = { NULL, NULL, NULL }; return ZSTD_createCCtx_advanced(customMem); } @@ -135,28 +134,24 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) { ctx = (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx)); if (!ctx) return NULL; - - ctx->customAlloc = malloc; - ctx->customFree = free; + memcpy(&ctx->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); return ctx; } if (!customMem.customAlloc || !customMem.customFree) return NULL; - ctx = (ZSTD_CCtx*) customMem.customAlloc(sizeof(ZSTD_CCtx)); + ctx = (ZSTD_CCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTD_CCtx)); if (!ctx) return NULL; - memset(ctx, 0, sizeof(ZSTD_CCtx)); - ctx->customAlloc = customMem.customAlloc; - ctx->customFree = customMem.customFree; + memcpy(&ctx->customMem, &customMem, sizeof(ZSTD_customMem)); return ctx; } size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) { - cctx->customFree(cctx->workSpace); - cctx->customFree(cctx); + cctx->customMem.customFree(cctx->customMem.opaque, cctx->workSpace); + cctx->customMem.customFree(cctx->customMem.opaque, cctx); return 0; /* reserved as a potential error code in the future */ } @@ -262,8 +257,8 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace + ((params.cParams.strategy == ZSTD_btopt) ? optSpace : 0); if (zc->workSpaceSize < neededSpace) { - zc->customFree(zc->workSpace); - zc->workSpace = zc->customAlloc(neededSpace); + zc->customMem.customFree(zc->customMem.opaque, zc->workSpace); + zc->workSpace = zc->customMem.customAlloc(zc->customMem.opaque, neededSpace); if (zc->workSpace == NULL) return ERROR(memory_allocation); zc->workSpaceSize = neededSpace; } } @@ -322,8 +317,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) if (srcCCtx->stage!=1) return ERROR(stage_wrong); dstCCtx->hashLog3 = srcCCtx->hashLog3; /* must be before ZSTD_resetCCtx_advanced */ - dstCCtx->customAlloc = srcCCtx->customAlloc; - dstCCtx->customFree = srcCCtx->customFree; + memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, 0); dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */ @@ -2487,10 +2481,9 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS size_t result; ZSTD_CCtx ctxBody; memset(&ctxBody, 0, sizeof(ctxBody)); - ctxBody.customAlloc = malloc; - ctxBody.customFree = free; + memcpy(&ctxBody.customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel); - ctxBody.customFree(ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */ + ctxBody.customMem.customFree(ctxBody.customMem.opaque, ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */ return result; } diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index 69898fdb..c4321498 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -82,14 +82,13 @@ struct ZBUFF_DCtx_s { size_t blockSize; BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; size_t lhSize; - ZSTD_allocFunction customAlloc; - ZSTD_freeFunction customFree; + ZSTD_customMem customMem; }; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */ ZBUFF_DCtx* ZBUFF_createDCtx(void) { - ZSTD_customMem customMem = { NULL, NULL }; + ZSTD_customMem const customMem = { NULL, NULL, NULL }; return ZBUFF_createDCtx_advanced(customMem); } @@ -101,8 +100,7 @@ ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem) { zbd = (ZBUFF_DCtx*)calloc(1, sizeof(ZBUFF_DCtx)); if (zbd==NULL) return NULL; - zbd->customAlloc = malloc; - zbd->customFree = free; + memcpy(&zbd->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); zbd->zd = ZSTD_createDCtx(); zbd->stage = ZBUFFds_init; return zbd; @@ -111,11 +109,10 @@ ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem) if (!customMem.customAlloc || !customMem.customFree) return NULL; - zbd = (ZBUFF_DCtx*)customMem.customAlloc(sizeof(ZBUFF_DCtx)); + zbd = (ZBUFF_DCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_DCtx)); if (zbd==NULL) return NULL; memset(zbd, 0, sizeof(ZBUFF_DCtx)); - zbd->customAlloc = customMem.customAlloc; - zbd->customFree = customMem.customFree; + memcpy(&zbd->customMem, &customMem, sizeof(ZSTD_customMem)); zbd->zd = ZSTD_createDCtx_advanced(customMem); zbd->stage = ZBUFFds_init; return zbd; @@ -125,9 +122,9 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd) { if (zbd==NULL) return 0; /* support free on null */ ZSTD_freeDCtx(zbd->zd); - zbd->customFree(zbd->inBuff); - zbd->customFree(zbd->outBuff); - zbd->customFree(zbd); + zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); + zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); + zbd->customMem.customFree(zbd->customMem.opaque, zbd); return 0; } @@ -198,16 +195,16 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, { size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTD_BLOCKSIZE_MAX); zbd->blockSize = blockSize; if (zbd->inBuffSize < blockSize) { - zbd->customFree(zbd->inBuff); + zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); zbd->inBuffSize = blockSize; - zbd->inBuff = (char*)zbd->customAlloc(blockSize); + zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize); if (zbd->inBuff == NULL) return ERROR(memory_allocation); } { size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize; if (zbd->outBuffSize < neededOutSize) { - zbd->customFree(zbd->outBuff); + zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); zbd->outBuffSize = neededOutSize; - zbd->outBuff = (char*)zbd->customAlloc(neededOutSize); + zbd->outBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, neededOutSize); if (zbd->outBuff == NULL) return ERROR(memory_allocation); } } } zbd->stage = ZBUFFds_read; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 27dd780a..461741d9 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -120,8 +120,7 @@ struct ZSTD_DCtx_s size_t headerSize; ZSTD_frameParams fParams; XXH64_state_t xxhState; - ZSTD_allocFunction customAlloc; - ZSTD_freeFunction customFree; + ZSTD_customMem customMem; blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ ZSTD_dStage stage; U32 dictID; @@ -156,9 +155,7 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) if (!customMem.customAlloc && !customMem.customFree) { dctx = (ZSTD_DCtx*) malloc(sizeof(ZSTD_DCtx)); if (!dctx) return NULL; - dctx->customAlloc = malloc; - dctx->customFree = free; - + memcpy(&dctx->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); ZSTD_decompressBegin(dctx); return dctx; } @@ -166,25 +163,23 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) if (!customMem.customAlloc || !customMem.customFree) return NULL; - dctx = (ZSTD_DCtx*) customMem.customAlloc(sizeof(ZSTD_DCtx)); + dctx = (ZSTD_DCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTD_DCtx)); if (!dctx) return NULL; - dctx->customAlloc = customMem.customAlloc; - dctx->customFree = customMem.customFree; - + memcpy(&dctx->customMem, &customMem, sizeof(ZSTD_customMem)); ZSTD_decompressBegin(dctx); return dctx; } ZSTD_DCtx* ZSTD_createDCtx(void) { - ZSTD_customMem const customMem = { NULL, NULL }; + ZSTD_customMem const customMem = { NULL, NULL, NULL }; return ZSTD_createDCtx_advanced(customMem); } size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) { - dctx->customFree(dctx); + dctx->customMem.customFree(dctx->customMem.opaque, dctx); return 0; /* reserved as a potential error code in the future */ } diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 19a385b4..f57c4cdf 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -129,16 +129,18 @@ static unsigned FUZ_highbit32(U32 v32) } */ -void* ZBUFF_allocFunction(size_t size) +void* ZBUFF_allocFunction(void* opaque, size_t size) { + (void)opaque; void* address = malloc(size); - /* DISPLAYLEVEL(4, "alloc %p, %d \n", address, (int)size); */ + /* DISPLAYLEVEL(4, "alloc %p, %d opaque=%d \n", address, (int)size, (int)opaque); */ return address; } -void ZBUFF_freeFunction(void* address) +void ZBUFF_freeFunction(void* opaque, void* address) { - /* if (address) DISPLAYLEVEL(4, "free %p \n", address); */ + (void)opaque; + /* if (address) DISPLAYLEVEL(4, "free %p opaque=%d \n", address, (int)opaque); */ free(address); } @@ -512,8 +514,8 @@ int main(int argc, const char** argv) int result=0; U32 mainPause = 0; const char* programName = argv[0]; - ZSTD_customMem customMem = { ZBUFF_allocFunction, ZBUFF_freeFunction }; - ZSTD_customMem customNULL = { NULL, NULL }; + ZSTD_customMem customMem = { ZBUFF_allocFunction, ZBUFF_freeFunction, (void*)777 }; + ZSTD_customMem customNULL = { NULL, NULL, NULL }; /* Check command line */ for(argNb=1; argNb Date: Thu, 2 Jun 2016 15:11:39 +0200 Subject: [PATCH 06/22] default custom allocation functions moved to zstd_internal.h --- lib/common/zstd_common.c | 2 +- lib/common/zstd_internal.h | 5 +++++ lib/common/zstd_static.h | 4 ---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c index c4ec46fb..262eeb23 100644 --- a/lib/common/zstd_common.c +++ b/lib/common/zstd_common.c @@ -76,8 +76,8 @@ const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(error void* ZSTD_defaultAllocFunction(void* opaque, size_t size) { - (void)opaque; void* address = malloc(size); + (void)opaque; /* DISPLAYLEVEL(4, "alloc %p, %d opaque=%d \n", address, (int)size, (int)opaque); */ return address; } diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 63e56aaf..8c647b51 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -255,4 +255,9 @@ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq); int ZSTD_isSkipFrame(ZSTD_DCtx* dctx); +/* custom memory allocation functions */ +void* ZSTD_defaultAllocFunction(void* opaque, size_t size); +void ZSTD_defaultFreeFunction(void* opaque, void* address); +MEM_STATIC ZSTD_customMem const defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL }; + #endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h index b36e92c8..f452e696 100644 --- a/lib/common/zstd_static.h +++ b/lib/common/zstd_static.h @@ -102,10 +102,6 @@ typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); typedef void (*ZSTD_freeFunction) (void* opaque, void* address); typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; -void* ZSTD_defaultAllocFunction(void* opaque, size_t size); -void ZSTD_defaultFreeFunction(void* opaque, void* address); -static ZSTD_customMem const defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL }; - /*-************************************* * Advanced functions From ff9114aee3123a4e80effe848776b746c3e4cc91 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 2 Jun 2016 16:52:36 +0200 Subject: [PATCH 07/22] zlibWrapper: added support for custom memory allocation functions --- lib/common/zstd_common.c | 4 ++-- lib/common/zstd_internal.h | 2 +- programs/zbufftest.c | 4 ++-- zlibWrapper/examples/example.c | 5 +++- zlibWrapper/zstd_zlibwrapper.c | 43 +++++++++++++++++++++++++++++----- 5 files changed, 46 insertions(+), 12 deletions(-) diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c index 262eeb23..5d3db0ea 100644 --- a/lib/common/zstd_common.c +++ b/lib/common/zstd_common.c @@ -78,13 +78,13 @@ void* ZSTD_defaultAllocFunction(void* opaque, size_t size) { void* address = malloc(size); (void)opaque; - /* DISPLAYLEVEL(4, "alloc %p, %d opaque=%d \n", address, (int)size, (int)opaque); */ + /* printf("alloc %p, %d opaque=%p \n", address, (int)size, opaque); */ return address; } void ZSTD_defaultFreeFunction(void* opaque, void* address) { (void)opaque; - /* if (address) DISPLAYLEVEL(4, "free %p opaque=%d \n", address, (int)opaque); */ + /* if (address) printf("free %p opaque=%p \n", address, opaque); */ free(address); } diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 8c647b51..230ce79f 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -258,6 +258,6 @@ int ZSTD_isSkipFrame(ZSTD_DCtx* dctx); /* custom memory allocation functions */ void* ZSTD_defaultAllocFunction(void* opaque, size_t size); void ZSTD_defaultFreeFunction(void* opaque, void* address); -MEM_STATIC ZSTD_customMem const defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL }; +static ZSTD_customMem const defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL }; #endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/programs/zbufftest.c b/programs/zbufftest.c index f57c4cdf..18382abf 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -133,14 +133,14 @@ void* ZBUFF_allocFunction(void* opaque, size_t size) { (void)opaque; void* address = malloc(size); - /* DISPLAYLEVEL(4, "alloc %p, %d opaque=%d \n", address, (int)size, (int)opaque); */ + /* DISPLAYLEVEL(4, "alloc %p, %d opaque=%p \n", address, (int)size, opaque); */ return address; } void ZBUFF_freeFunction(void* opaque, void* address) { (void)opaque; - /* if (address) DISPLAYLEVEL(4, "free %p opaque=%d \n", address, (int)opaque); */ + /* if (address) DISPLAYLEVEL(4, "free %p opaque=%p \n", address, opaque); */ free(address); } diff --git a/zlibWrapper/examples/example.c b/zlibWrapper/examples/example.c index fe6c80ac..3fea9d2a 100644 --- a/zlibWrapper/examples/example.c +++ b/zlibWrapper/examples/example.c @@ -62,11 +62,14 @@ void *myalloc(q, n, m) unsigned n, m; { q = Z_NULL; - return calloc(n, m); + void *buf = calloc(n, m); + /* printf("myalloc %p n=%d m=%d\n", buf, n, m); */ + return buf; } void myfree(void *q, void *p) { + /* printf("myfree %p\n", p); */ q = Z_NULL; free(p); } diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index ff85a861..26f173b1 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -37,6 +37,7 @@ #include "zstd.h" #include "zstd_static.h" /* ZSTD_MAGICNUMBER */ #include "zbuff.h" +#include "zbuff_static.h" /* ZBUFF_createCCtx_advanced */ #define Z_INFLATE_SYNC 8 @@ -75,6 +76,22 @@ const char * zstdVersion() { return ZSTD_VERSION_STRING; } ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion(); } +void* ZWRAP_allocFunction(void* opaque, size_t size) +{ + z_streamp strm = (z_streamp) opaque; + void* address = strm->zalloc(strm->opaque, 1, size); + /* printf("ZWRAP alloc %p, %d \n", address, (int)size); */ + return address; +} + +void ZWRAP_freeFunction(void* opaque, void* address) +{ + z_streamp strm = (z_streamp) opaque; + strm->zfree(strm->opaque, address); + /* if (address) printf("ZWRAP free %p \n", address); */ +} + + /* *** Compression *** */ @@ -82,15 +99,22 @@ typedef struct { ZBUFF_CCtx* zbc; size_t bytesLeft; int compressionLevel; + z_stream allocFunc; /* copy of zalloc, zfree, opaque */ } ZWRAP_CCtx; -ZWRAP_CCtx* ZWRAP_createCCtx() +ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm) { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*)malloc(sizeof(ZWRAP_CCtx)); if (zwc==NULL) return NULL; memset(zwc, 0, sizeof(*zwc)); - zwc->zbc = ZBUFF_createCCtx(); + if (strm->zalloc && strm->zfree) { + ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwc->allocFunc }; + memcpy(&zwc->allocFunc, strm, sizeof(z_stream)); + zwc->zbc = ZBUFF_createCCtx_advanced(ZWRAP_customMem); + } + else + zwc->zbc = ZBUFF_createCCtx(); return zwc; } @@ -115,7 +139,7 @@ ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, } LOG_WRAPPER("- deflateInit level=%d\n", level); - zwc = ZWRAP_createCCtx(); + zwc = ZWRAP_createCCtx(strm); if (zwc == NULL) return Z_MEM_ERROR; if (level == Z_DEFAULT_COMPRESSION) @@ -265,14 +289,16 @@ typedef struct { int stream_size; char *version; int windowBits; + z_stream allocFunc; /* copy of zalloc, zfree, opaque */ } ZWRAP_DCtx; -ZWRAP_DCtx* ZWRAP_createDCtx(void) +ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm) { ZWRAP_DCtx* zwd = (ZWRAP_DCtx*)malloc(sizeof(ZWRAP_DCtx)); if (zwd==NULL) return NULL; memset(zwd, 0, sizeof(*zwd)); + memcpy(&zwd->allocFunc, strm, sizeof(z_stream)); return zwd; } @@ -290,7 +316,7 @@ size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd) ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, const char *version, int stream_size)) { - ZWRAP_DCtx* zwd = ZWRAP_createDCtx(); + ZWRAP_DCtx* zwd = ZWRAP_createDCtx(strm); LOG_WRAPPER("- inflateInit\n"); if (zwd == NULL) return Z_MEM_ERROR; @@ -402,7 +428,12 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) return inflate(strm, flush); } - zwd->zbd = ZBUFF_createDCtx(); + if (zwd->allocFunc.zalloc && zwd->allocFunc.zfree) { + ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwd->allocFunc }; + zwd->zbd = ZBUFF_createDCtx_advanced(ZWRAP_customMem); + } else + zwd->zbd = ZBUFF_createDCtx(); + { size_t const errorCode = ZBUFF_decompressInit(zwd->zbd); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } From 7cab86f6470184ca72dbfdeba6a9cbaad6d7b04d Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 2 Jun 2016 18:24:07 +0200 Subject: [PATCH 08/22] ZWRAP_allocFunction and ZWRAP_freeFunction are static --- Makefile | 2 +- programs/zbufftest.c | 6 +++--- zlibWrapper/zstd_zlibwrapper.c | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index db58d430..c8b5831b 100644 --- a/Makefile +++ b/Makefile @@ -106,7 +106,7 @@ ppctest: clean # for Travis CI ppcinstall: clean - sudo apt-get update -y -q + # sudo apt-get update -y -q sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu # doesn't work with Ubuntu 12.04 # for Travis CI diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 18382abf..b1d5b47c 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -129,7 +129,7 @@ static unsigned FUZ_highbit32(U32 v32) } */ -void* ZBUFF_allocFunction(void* opaque, size_t size) +static void* ZBUFF_allocFunction(void* opaque, size_t size) { (void)opaque; void* address = malloc(size); @@ -137,7 +137,7 @@ void* ZBUFF_allocFunction(void* opaque, size_t size) return address; } -void ZBUFF_freeFunction(void* opaque, void* address) +static void ZBUFF_freeFunction(void* opaque, void* address) { (void)opaque; /* if (address) DISPLAYLEVEL(4, "free %p opaque=%p \n", address, opaque); */ @@ -514,7 +514,7 @@ int main(int argc, const char** argv) int result=0; U32 mainPause = 0; const char* programName = argv[0]; - ZSTD_customMem customMem = { ZBUFF_allocFunction, ZBUFF_freeFunction, (void*)777 }; + ZSTD_customMem customMem = { ZBUFF_allocFunction, ZBUFF_freeFunction, NULL }; ZSTD_customMem customNULL = { NULL, NULL, NULL }; /* Check command line */ diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 26f173b1..b55b760a 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -76,7 +76,7 @@ const char * zstdVersion() { return ZSTD_VERSION_STRING; } ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion(); } -void* ZWRAP_allocFunction(void* opaque, size_t size) +static void* ZWRAP_allocFunction(void* opaque, size_t size) { z_streamp strm = (z_streamp) opaque; void* address = strm->zalloc(strm->opaque, 1, size); @@ -84,7 +84,7 @@ void* ZWRAP_allocFunction(void* opaque, size_t size) return address; } -void ZWRAP_freeFunction(void* opaque, void* address) +static void ZWRAP_freeFunction(void* opaque, void* address) { z_streamp strm = (z_streamp) opaque; strm->zfree(strm->opaque, address); From 614fddee6a71d460844de3f31057f5dcc9f9ea81 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 2 Jun 2016 18:40:41 +0200 Subject: [PATCH 09/22] zlibWrapper: support for stricter gcc warnings --- zlibWrapper/Makefile | 3 ++- zlibWrapper/examples/example.c | 2 +- zlibWrapper/zstd_zlibwrapper.c | 12 ++++++------ zlibWrapper/zstd_zlibwrapper.h | 4 ++-- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index 6c00622b..962f9294 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -18,7 +18,8 @@ endif ZLIBWRAPPER_PATH = . EXAMPLE_PATH = examples CC = gcc -CFLAGS = $(LOC) -I../lib/common -I$(ZLIBDIR) -I$(ZLIBWRAPPER_PATH) -O3 -Wall -std=gnu89 +CFLAGS = $(LOC) -I../lib/common -I$(ZLIBDIR) -I$(ZLIBWRAPPER_PATH) -O3 -std=gnu90 +CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wstrict-prototypes -Wundef LDFLAGS = $(LOC) RM = rm -f diff --git a/zlibWrapper/examples/example.c b/zlibWrapper/examples/example.c index 3fea9d2a..a2836cdb 100644 --- a/zlibWrapper/examples/example.c +++ b/zlibWrapper/examples/example.c @@ -61,8 +61,8 @@ void *myalloc(q, n, m) void *q; unsigned n, m; { - q = Z_NULL; void *buf = calloc(n, m); + q = Z_NULL; /* printf("myalloc %p n=%d m=%d\n", buf, n, m); */ return buf; } diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index b55b760a..4740e0db 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -69,9 +69,9 @@ static int g_useZSTD = ZWRAP_USE_ZSTD; /* 0 = don't use ZSTD */ void useZSTD(int turn_on) { g_useZSTD = turn_on; } -int isUsingZSTD() { return g_useZSTD; } +int isUsingZSTD(void) { return g_useZSTD; } -const char * zstdVersion() { return ZSTD_VERSION_STRING; } +const char * zstdVersion(void) { return ZSTD_VERSION_STRING; } ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion(); } @@ -421,8 +421,8 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) strm->avail_out = strm2.avail_out; strm->reserved = 0; /* mark as zlib stream */ - { size_t const errorCode = ZWRAP_freeDCtx(zwd); - if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } + errorCode = ZWRAP_freeDCtx(zwd); + if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; if (flush == Z_INFLATE_SYNC) return inflateSync(strm); return inflate(strm, flush); @@ -434,8 +434,8 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) } else zwd->zbd = ZBUFF_createDCtx(); - { size_t const errorCode = ZBUFF_decompressInit(zwd->zbd); - if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; } + errorCode = ZBUFF_decompressInit(zwd->zbd); + if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; srcSize = ZWRAP_HEADERSIZE; dstCapacity = 0; diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h index e438789a..d14c3a92 100644 --- a/zlibWrapper/zstd_zlibwrapper.h +++ b/zlibWrapper/zstd_zlibwrapper.h @@ -49,8 +49,8 @@ extern "C" { #endif void useZSTD(int turn_on); -int isUsingZSTD(); -const char * zstdVersion(); +int isUsingZSTD(void); +const char * zstdVersion(void); #if defined (__cplusplus) From ff2d18928d8a491624b29a6410fd410ae2eae0b0 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 2 Jun 2016 22:15:09 +0200 Subject: [PATCH 10/22] zlibWrapper: added gcc flag -Wdeclaration-after-statement --- programs/zbufftest.c | 2 +- zlibWrapper/Makefile | 2 +- zlibWrapper/zstd_zlibwrapper.c | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 355b6431..38711981 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -131,8 +131,8 @@ static unsigned FUZ_highbit32(U32 v32) static void* ZBUFF_allocFunction(void* opaque, size_t size) { - (void)opaque; void* address = malloc(size); + (void)opaque; /* DISPLAYLEVEL(4, "alloc %p, %d opaque=%p \n", address, (int)size, opaque); */ return address; } diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index 962f9294..edfcdf78 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -19,7 +19,7 @@ ZLIBWRAPPER_PATH = . EXAMPLE_PATH = examples CC = gcc CFLAGS = $(LOC) -I../lib/common -I$(ZLIBDIR) -I$(ZLIBWRAPPER_PATH) -O3 -std=gnu90 -CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wstrict-prototypes -Wundef +CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef LDFLAGS = $(LOC) RM = rm -f diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 4740e0db..a4e22a11 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -667,9 +667,9 @@ ZEXTERN int ZEXPORT z_compress OF((Bytef *dest, uLongf *destLen, if (!g_useZSTD) return compress(dest, destLen, source, sourceLen); - size_t dstCapacity = *destLen; - LOG_WRAPPER("z_compress sourceLen=%d dstCapacity=%d\n", (int)sourceLen, (int)dstCapacity); - { size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, -1); + { size_t dstCapacity = *destLen; + size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, -1); + LOG_WRAPPER("z_compress sourceLen=%d dstCapacity=%d\n", (int)sourceLen, (int)dstCapacity); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; *destLen = errorCode; } @@ -684,8 +684,8 @@ ZEXTERN int ZEXPORT z_compress2 OF((Bytef *dest, uLongf *destLen, if (!g_useZSTD) return compress2(dest, destLen, source, sourceLen, level); - size_t dstCapacity = *destLen; - { size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, level); + { size_t dstCapacity = *destLen; + size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, level); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; *destLen = errorCode; } @@ -709,8 +709,8 @@ ZEXTERN int ZEXPORT z_uncompress OF((Bytef *dest, uLongf *destLen, // if (!g_useZSTD) return uncompress(dest, destLen, source, sourceLen); - size_t dstCapacity = *destLen; - { size_t const errorCode = ZSTD_decompress(dest, dstCapacity, source, sourceLen); + { size_t dstCapacity = *destLen; + size_t const errorCode = ZSTD_decompress(dest, dstCapacity, source, sourceLen); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; *destLen = errorCode; } From aa224c0205eed233bd11cfeeb0b1e2f6d42eb26c Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 2 Jun 2016 22:24:29 +0200 Subject: [PATCH 11/22] VS projects: remove duplicated xxhash.c and xxhash.h --- projects/VS2008/fullbench/fullbench.vcproj | 10 +--------- projects/VS2010/fullbench/fullbench.vcxproj | 1 - 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/projects/VS2008/fullbench/fullbench.vcproj b/projects/VS2008/fullbench/fullbench.vcproj index bbb7da7f..50cbcc2c 100644 --- a/projects/VS2008/fullbench/fullbench.vcproj +++ b/projects/VS2008/fullbench/fullbench.vcproj @@ -360,10 +360,6 @@ RelativePath="..\..\..\lib\decompress\huf_decompress.c" > - - @@ -422,12 +418,8 @@ RelativePath="..\..\..\lib\common\xxhash.h" > - - - From 764deea5f87ac8b47bb3fe5d440a52e1d33628be Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 3 Jun 2016 10:11:26 +0200 Subject: [PATCH 12/22] cmake project: fixed "make install" --- projects/cmake/lib/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/cmake/lib/CMakeLists.txt b/projects/cmake/lib/CMakeLists.txt index 716ccbf3..3b7bc802 100644 --- a/projects/cmake/lib/CMakeLists.txt +++ b/projects/cmake/lib/CMakeLists.txt @@ -162,7 +162,8 @@ IF (UNIX) SET(INSTALL_INCLUDE_DIR ${PREFIX}/include) # install target - INSTALL(FILES ${LIBRARY_DIR}/zstd.h ${LIBRARY_DIR}/zstd_buffered.h ${LIBRARY_DIR}/dictBuilder.h DESTINATION ${INSTALL_INCLUDE_DIR}) + INSTALL(FILES ${LIBRARY_DIR}/common/zstd.h ${LIBRARY_DIR}/common/zstd_static.h ${LIBRARY_DIR}/common/zbuff.h ${LIBRARY_DIR}/common/zbuff_static.h + ${LIBRARY_DIR}/common/mem.h ${LIBRARY_DIR}/common/error_public.h ${LIBRARY_DIR}/dictBuilder/zdict.h ${LIBRARY_DIR}/dictBuilder/zdict_static.h DESTINATION ${INSTALL_INCLUDE_DIR}) INSTALL(TARGETS libzstd_static DESTINATION ${INSTALL_LIBRARY_DIR}) INSTALL(TARGETS libzstd_shared LIBRARY DESTINATION ${INSTALL_LIBRARY_DIR}) From d36ee9b845800dfbc6aa503aa6844cd98f6f80ed Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 3 Jun 2016 10:24:00 +0200 Subject: [PATCH 13/22] cmake project: added "make clean-all" --- projects/cmake/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/projects/cmake/CMakeLists.txt b/projects/cmake/CMakeLists.txt index ae8f7be6..9650eb4f 100644 --- a/projects/cmake/CMakeLists.txt +++ b/projects/cmake/CMakeLists.txt @@ -52,3 +52,8 @@ ADD_SUBDIRECTORY(programs) #----------------------------------------------------------------------------- INCLUDE(CMakeModules/AddExtraCompilationFlags.cmake) ADD_EXTRA_COMPILATION_FLAGS() + +ADD_CUSTOM_TARGET(clean-all + COMMAND ${CMAKE_BUILD_TOOL} clean + COMMAND rm -rf ${CMAKE_BINARY_DIR}/ +) From c7f008b069d25facb7b228c6a36c6dde497dd33d Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 3 Jun 2016 11:40:15 +0200 Subject: [PATCH 14/22] Travis cmaketest will also run "make install" and "make uninstall" --- Makefile | 2 +- projects/cmake/lib/CMakeLists.txt | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c8b5831b..755b0c1a 100644 --- a/Makefile +++ b/Makefile @@ -155,7 +155,7 @@ cmaketest: cmake --version rm -rf projects/cmake/build mkdir projects/cmake/build - cd projects/cmake/build ; cmake $(CMAKE_PARAMS) .. ; $(MAKE) + cd projects/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall c90test: clean CFLAGS="-std=c90" $(MAKE) all # will fail, due to // and long long diff --git a/projects/cmake/lib/CMakeLists.txt b/projects/cmake/lib/CMakeLists.txt index 3b7bc802..84568b20 100644 --- a/projects/cmake/lib/CMakeLists.txt +++ b/projects/cmake/lib/CMakeLists.txt @@ -157,13 +157,15 @@ SET_TARGET_PROPERTIES( OUTPUT_NAME ${SHARED_LIBRARY_OUTPUT_NAME}) IF (UNIX) - SET(PREFIX /usr/local) + IF ("${PREFIX}" STREQUAL "") + SET(PREFIX /usr/local) + ENDIF() + MESSAGE("the variable PREFIX=${PREFIX}") SET(INSTALL_LIBRARY_DIR ${PREFIX}/lib) SET(INSTALL_INCLUDE_DIR ${PREFIX}/include) # install target - INSTALL(FILES ${LIBRARY_DIR}/common/zstd.h ${LIBRARY_DIR}/common/zstd_static.h ${LIBRARY_DIR}/common/zbuff.h ${LIBRARY_DIR}/common/zbuff_static.h - ${LIBRARY_DIR}/common/mem.h ${LIBRARY_DIR}/common/error_public.h ${LIBRARY_DIR}/dictBuilder/zdict.h ${LIBRARY_DIR}/dictBuilder/zdict_static.h DESTINATION ${INSTALL_INCLUDE_DIR}) + INSTALL(FILES ${LIBRARY_DIR}/common/zstd.h ${LIBRARY_DIR}/common/zbuff.h ${LIBRARY_DIR}/dictBuilder/zdict.h DESTINATION ${INSTALL_INCLUDE_DIR}) INSTALL(TARGETS libzstd_static DESTINATION ${INSTALL_LIBRARY_DIR}) INSTALL(TARGETS libzstd_shared LIBRARY DESTINATION ${INSTALL_LIBRARY_DIR}) From db2f54041461e09dc0cb833fa999def85fa2bf73 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 3 Jun 2016 12:56:56 +0200 Subject: [PATCH 15/22] added defaultCustomNULL --- lib/common/zstd_internal.h | 3 ++- lib/compress/zbuff_compress.c | 3 +-- lib/compress/zstd_compress.c | 3 +-- lib/decompress/zbuff_decompress.c | 3 +-- lib/decompress/zstd_decompress.c | 3 +-- programs/zbufftest.c | 3 +-- zlibWrapper/Makefile | 4 ++-- 7 files changed, 9 insertions(+), 13 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 230ce79f..b56030ba 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -259,5 +259,6 @@ int ZSTD_isSkipFrame(ZSTD_DCtx* dctx); void* ZSTD_defaultAllocFunction(void* opaque, size_t size); void ZSTD_defaultFreeFunction(void* opaque, void* address); static ZSTD_customMem const defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL }; - +static ZSTD_customMem const defaultCustomNULL = { NULL, NULL, NULL }; + #endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index 6e40d3c0..0ec27ad7 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -100,8 +100,7 @@ struct ZBUFF_CCtx_s { ZBUFF_CCtx* ZBUFF_createCCtx(void) { - ZSTD_customMem const customMem = { NULL, NULL, NULL }; - return ZBUFF_createCCtx_advanced(customMem); + return ZBUFF_createCCtx_advanced(defaultCustomNULL); } ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 9de86a6b..93caee0d 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -122,8 +122,7 @@ struct ZSTD_CCtx_s ZSTD_CCtx* ZSTD_createCCtx(void) { - ZSTD_customMem const customMem = { NULL, NULL, NULL }; - return ZSTD_createCCtx_advanced(customMem); + return ZSTD_createCCtx_advanced(defaultCustomNULL); } ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index c4321498..4dfb2ada 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -88,8 +88,7 @@ struct ZBUFF_DCtx_s { ZBUFF_DCtx* ZBUFF_createDCtx(void) { - ZSTD_customMem const customMem = { NULL, NULL, NULL }; - return ZBUFF_createDCtx_advanced(customMem); + return ZBUFF_createDCtx_advanced(defaultCustomNULL); } ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 461741d9..d4426bcc 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -172,8 +172,7 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) ZSTD_DCtx* ZSTD_createDCtx(void) { - ZSTD_customMem const customMem = { NULL, NULL, NULL }; - return ZSTD_createDCtx_advanced(customMem); + return ZSTD_createDCtx_advanced(defaultCustomNULL); } diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 38711981..37a94179 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -521,7 +521,6 @@ int main(int argc, const char** argv) U32 mainPause = 0; const char* programName = argv[0]; ZSTD_customMem customMem = { ZBUFF_allocFunction, ZBUFF_freeFunction, NULL }; - ZSTD_customMem customNULL = { NULL, NULL, NULL }; /* Check command line */ for(argNb=1; argNb Date: Fri, 3 Jun 2016 13:23:04 +0200 Subject: [PATCH 16/22] removed calloc calls from lib/ --- lib/compress/huf_compress.c | 1 - lib/compress/zbuff_compress.c | 8 +------- lib/compress/zstd_compress.c | 8 +------- lib/decompress/huf_decompress.c | 1 - lib/decompress/zbuff_decompress.c | 9 +-------- lib/decompress/zstd_decompress.c | 10 ++-------- programs/zbufftest.c | 3 ++- 7 files changed, 7 insertions(+), 33 deletions(-) diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index a1004fba..05cf6965 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -59,7 +59,6 @@ /* ************************************************************** * Includes ****************************************************************/ -#include /* malloc, free, qsort */ #include /* memcpy, memset */ #include /* printf (debug) */ #include "huf_static.h" diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index 0ec27ad7..1681bc84 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -108,13 +108,7 @@ ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem) ZBUFF_CCtx* zbc; if (!customMem.customAlloc && !customMem.customFree) - { - zbc = (ZBUFF_CCtx*)calloc(1, sizeof(ZBUFF_CCtx)); - if (zbc==NULL) return NULL; - memcpy(&zbc->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); - zbc->zc = ZSTD_createCCtx(); - return zbc; - } + customMem = defaultCustomMem; if (!customMem.customAlloc || !customMem.customFree) return NULL; diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 93caee0d..3196916f 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -51,7 +51,6 @@ /*-************************************* * Dependencies ***************************************/ -#include /* malloc */ #include /* memset */ #include "mem.h" #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ @@ -130,12 +129,7 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) ZSTD_CCtx* ctx; if (!customMem.customAlloc && !customMem.customFree) - { - ctx = (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx)); - if (!ctx) return NULL; - memcpy(&ctx->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); - return ctx; - } + customMem = defaultCustomMem; if (!customMem.customAlloc || !customMem.customFree) return NULL; diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c index 01e9c07b..14afbb92 100644 --- a/lib/decompress/huf_decompress.c +++ b/lib/decompress/huf_decompress.c @@ -59,7 +59,6 @@ /* ************************************************************** * Includes ****************************************************************/ -#include /* malloc, free, qsort */ #include /* memcpy, memset */ #include /* printf (debug) */ #include "huf_static.h" diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index 4dfb2ada..01fcc217 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -96,14 +96,7 @@ ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem) ZBUFF_DCtx* zbd; if (!customMem.customAlloc && !customMem.customFree) - { - zbd = (ZBUFF_DCtx*)calloc(1, sizeof(ZBUFF_DCtx)); - if (zbd==NULL) return NULL; - memcpy(&zbd->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); - zbd->zd = ZSTD_createDCtx(); - zbd->stage = ZBUFFds_init; - return zbd; - } + customMem = defaultCustomMem; if (!customMem.customAlloc || !customMem.customFree) return NULL; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index d4426bcc..495e8d29 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -53,7 +53,6 @@ /*-******************************************************* * Dependencies *********************************************************/ -#include /* calloc */ #include /* memcpy, memmove */ #include /* debug only : printf */ #include "mem.h" /* low level memory routines */ @@ -152,13 +151,8 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) { ZSTD_DCtx* dctx; - if (!customMem.customAlloc && !customMem.customFree) { - dctx = (ZSTD_DCtx*) malloc(sizeof(ZSTD_DCtx)); - if (!dctx) return NULL; - memcpy(&dctx->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); - ZSTD_decompressBegin(dctx); - return dctx; - } + if (!customMem.customAlloc && !customMem.customFree) + customMem = defaultCustomMem; if (!customMem.customAlloc || !customMem.customFree) return NULL; diff --git a/programs/zbufftest.c b/programs/zbufftest.c index 37a94179..38711981 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -521,6 +521,7 @@ int main(int argc, const char** argv) U32 mainPause = 0; const char* programName = argv[0]; ZSTD_customMem customMem = { ZBUFF_allocFunction, ZBUFF_freeFunction, NULL }; + ZSTD_customMem customNULL = { NULL, NULL, NULL }; /* Check command line */ for(argNb=1; argNb Date: Fri, 3 Jun 2016 13:28:20 +0200 Subject: [PATCH 17/22] defaultCustomNULL replaced with defaultCustomMem --- lib/common/zstd_internal.h | 3 +-- lib/compress/zbuff_compress.c | 2 +- lib/compress/zstd_compress.c | 2 +- lib/decompress/zbuff_decompress.c | 2 +- lib/decompress/zstd_decompress.c | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index b56030ba..8432c1dd 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -259,6 +259,5 @@ int ZSTD_isSkipFrame(ZSTD_DCtx* dctx); void* ZSTD_defaultAllocFunction(void* opaque, size_t size); void ZSTD_defaultFreeFunction(void* opaque, void* address); static ZSTD_customMem const defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL }; -static ZSTD_customMem const defaultCustomNULL = { NULL, NULL, NULL }; - + #endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index 1681bc84..0340eeb3 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -100,7 +100,7 @@ struct ZBUFF_CCtx_s { ZBUFF_CCtx* ZBUFF_createCCtx(void) { - return ZBUFF_createCCtx_advanced(defaultCustomNULL); + return ZBUFF_createCCtx_advanced(defaultCustomMem); } ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 3196916f..68a6072e 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -121,7 +121,7 @@ struct ZSTD_CCtx_s ZSTD_CCtx* ZSTD_createCCtx(void) { - return ZSTD_createCCtx_advanced(defaultCustomNULL); + return ZSTD_createCCtx_advanced(defaultCustomMem); } ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index 01fcc217..e9e84ef7 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -88,7 +88,7 @@ struct ZBUFF_DCtx_s { ZBUFF_DCtx* ZBUFF_createDCtx(void) { - return ZBUFF_createDCtx_advanced(defaultCustomNULL); + return ZBUFF_createDCtx_advanced(defaultCustomMem); } ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 495e8d29..128d941e 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -166,7 +166,7 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) ZSTD_DCtx* ZSTD_createDCtx(void) { - return ZSTD_createDCtx_advanced(defaultCustomNULL); + return ZSTD_createDCtx_advanced(defaultCustomMem); } From 2a74609b90ceae29f5bfa8e8c651ee97c4ce3607 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 3 Jun 2016 14:53:51 +0200 Subject: [PATCH 18/22] zlibWrapper: ZWRAP_createCCtx and ZWRAP_freeCCtx use custom memory allocation functions --- lib/common/zstd_internal.h | 2 +- zlibWrapper/zstd_zlibwrapper.c | 30 +++++++++++++++++++----------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 8432c1dd..230ce79f 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -259,5 +259,5 @@ int ZSTD_isSkipFrame(ZSTD_DCtx* dctx); void* ZSTD_defaultAllocFunction(void* opaque, size_t size); void ZSTD_defaultFreeFunction(void* opaque, void* address); static ZSTD_customMem const defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL }; - + #endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index a4e22a11..937895cd 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -38,6 +38,7 @@ #include "zstd_static.h" /* ZSTD_MAGICNUMBER */ #include "zbuff.h" #include "zbuff_static.h" /* ZBUFF_createCCtx_advanced */ +#include "zstd_internal.h" /* defaultCustomMem */ #define Z_INFLATE_SYNC 8 @@ -47,8 +48,6 @@ #define LOG_WRAPPER(...) // printf(__VA_ARGS__) -#define MIN(a,b) ((a)<(b)?(a):(b)) - #define FINISH_WITH_ERR(msg) { \ fprintf(stderr, "ERROR: %s\n", msg); \ return Z_MEM_ERROR; \ @@ -99,22 +98,31 @@ typedef struct { ZBUFF_CCtx* zbc; size_t bytesLeft; int compressionLevel; + ZSTD_customMem customMem; z_stream allocFunc; /* copy of zalloc, zfree, opaque */ } ZWRAP_CCtx; ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm) { - ZWRAP_CCtx* zwc = (ZWRAP_CCtx*)malloc(sizeof(ZWRAP_CCtx)); - if (zwc==NULL) return NULL; - memset(zwc, 0, sizeof(*zwc)); + ZWRAP_CCtx* zwc; + if (strm->zalloc && strm->zfree) { - ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwc->allocFunc }; - memcpy(&zwc->allocFunc, strm, sizeof(z_stream)); - zwc->zbc = ZBUFF_createCCtx_advanced(ZWRAP_customMem); + zwc = (ZWRAP_CCtx*)strm->zalloc(strm->opaque, 1, sizeof(ZWRAP_CCtx)); + if (zwc==NULL) return NULL; + memset(zwc, 0, sizeof(ZWRAP_CCtx)); + memcpy(&zwc->allocFunc, strm, sizeof(z_stream)); + { ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwc->allocFunc }; + memcpy(&zwc->customMem, &ZWRAP_customMem, sizeof(ZSTD_customMem)); + } + } else { + zwc = (ZWRAP_CCtx*)defaultCustomMem.customAlloc(defaultCustomMem.opaque, sizeof(ZWRAP_CCtx)); + if (zwc==NULL) return NULL; + memset(zwc, 0, sizeof(ZWRAP_CCtx)); + memcpy(&zwc->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); } - else - zwc->zbc = ZBUFF_createCCtx(); + + zwc->zbc = ZBUFF_createCCtx_advanced(zwc->customMem); return zwc; } @@ -123,7 +131,7 @@ size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc) { if (zwc==NULL) return 0; /* support free on NULL */ ZBUFF_freeCCtx(zwc->zbc); - free(zwc); + zwc->customMem.customFree(zwc->customMem.opaque, zwc); return 0; } From f040be9850d8678682cfac6862dce6d469d4556d Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 3 Jun 2016 16:31:57 +0200 Subject: [PATCH 19/22] zlibWrapper: ZWRAP_createDCtx and ZWRAP_freeDCtx use custom memory allocation functions --- zlibWrapper/zstd_zlibwrapper.c | 61 +++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 937895cd..f3f6ea39 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -30,7 +30,6 @@ */ #include /* fprintf */ -#include /* malloc */ #include /* va_list */ #include #include "zstd_zlibwrapper.h" @@ -103,6 +102,15 @@ typedef struct { } ZWRAP_CCtx; +size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc) +{ + if (zwc==NULL) return 0; /* support free on NULL */ + ZBUFF_freeCCtx(zwc->zbc); + zwc->customMem.customFree(zwc->customMem.opaque, zwc); + return 0; +} + + ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm) { ZWRAP_CCtx* zwc; @@ -123,19 +131,11 @@ ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm) } zwc->zbc = ZBUFF_createCCtx_advanced(zwc->customMem); + if (zwc->zbc == NULL) { ZWRAP_freeCCtx(zwc); return NULL; } return zwc; } -size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc) -{ - if (zwc==NULL) return 0; /* support free on NULL */ - ZBUFF_freeCCtx(zwc->zbc); - zwc->customMem.customFree(zwc->customMem.opaque, zwc); - return 0; -} - - ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, const char *version, int stream_size)) { @@ -297,16 +297,30 @@ typedef struct { int stream_size; char *version; int windowBits; + ZSTD_customMem customMem; z_stream allocFunc; /* copy of zalloc, zfree, opaque */ } ZWRAP_DCtx; ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm) { - ZWRAP_DCtx* zwd = (ZWRAP_DCtx*)malloc(sizeof(ZWRAP_DCtx)); - if (zwd==NULL) return NULL; - memset(zwd, 0, sizeof(*zwd)); - memcpy(&zwd->allocFunc, strm, sizeof(z_stream)); + ZWRAP_DCtx* zwd; + + if (strm->zalloc && strm->zfree) { + zwd = (ZWRAP_DCtx*)strm->zalloc(strm->opaque, 1, sizeof(ZWRAP_DCtx)); + if (zwd==NULL) return NULL; + memset(zwd, 0, sizeof(ZWRAP_DCtx)); + memcpy(&zwd->allocFunc, strm, sizeof(z_stream)); + { ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwd->allocFunc }; + memcpy(&zwd->customMem, &ZWRAP_customMem, sizeof(ZSTD_customMem)); + } + } else { + zwd = (ZWRAP_DCtx*)defaultCustomMem.customAlloc(defaultCustomMem.opaque, sizeof(ZWRAP_DCtx)); + if (zwd==NULL) return NULL; + memset(zwd, 0, sizeof(ZWRAP_DCtx)); + memcpy(&zwd->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); + } + return zwd; } @@ -314,9 +328,9 @@ ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm) size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd) { if (zwd==NULL) return 0; /* support free on null */ - if (zwd->version) free(zwd->version); - if (zwd->zbd) ZBUFF_freeDCtx(zwd->zbd); - free(zwd); + ZBUFF_freeDCtx(zwd->zbd); + if (zwd->version) zwd->customMem.customFree(zwd->customMem.opaque, zwd->version); + zwd->customMem.customFree(zwd->customMem.opaque, zwd); return 0; } @@ -328,9 +342,11 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, LOG_WRAPPER("- inflateInit\n"); if (zwd == NULL) return Z_MEM_ERROR; - zwd->stream_size = stream_size; - zwd->version = strdup(version); + zwd->version = zwd->customMem.customAlloc(zwd->customMem.opaque, strlen(version) + 1); + if (zwd->version == NULL) { ZWRAP_freeDCtx(zwd); return Z_MEM_ERROR; } + strcpy(zwd->version, version); + zwd->stream_size = stream_size; strm->state = (struct internal_state*) zwd; /* use state which in not used by user */ strm->total_in = 0; strm->total_out = 0; @@ -436,11 +452,8 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) return inflate(strm, flush); } - if (zwd->allocFunc.zalloc && zwd->allocFunc.zfree) { - ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwd->allocFunc }; - zwd->zbd = ZBUFF_createDCtx_advanced(ZWRAP_customMem); - } else - zwd->zbd = ZBUFF_createDCtx(); + zwd->zbd = ZBUFF_createDCtx_advanced(zwd->customMem); + if (zwd->zbd == NULL) { ZWRAP_freeDCtx(zwd); return Z_MEM_ERROR; } errorCode = ZBUFF_decompressInit(zwd->zbd); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; From 3640396b1a4fac14d62658037165d8aacf494c56 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 3 Jun 2016 16:36:50 +0200 Subject: [PATCH 20/22] fixed: deallocation of structures in case of error in ZBUFF_createCCtx and ZBUFF_createDCtx --- lib/compress/zbuff_compress.c | 5 +++-- lib/compress/zstd_compress.c | 3 ++- lib/decompress/zbuff_decompress.c | 5 +++-- lib/decompress/zstd_decompress.c | 1 + zlibWrapper/Makefile | 8 ++------ 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index 0340eeb3..fe8c3801 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -118,6 +118,7 @@ ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem) memset(zbc, 0, sizeof(ZBUFF_CCtx)); memcpy(&zbc->customMem, &customMem, sizeof(ZSTD_customMem)); zbc->zc = ZSTD_createCCtx_advanced(customMem); + if (zbc->zc == NULL) { ZBUFF_freeCCtx(zbc); return NULL; } return zbc; } @@ -125,8 +126,8 @@ size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc) { if (zbc==NULL) return 0; /* support free on NULL */ ZSTD_freeCCtx(zbc->zc); - zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff); - zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff); + if (zbc->inBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff); + if (zbc->outBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff); zbc->customMem.customFree(zbc->customMem.opaque, zbc); return 0; } diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 68a6072e..dbc9dd20 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -143,7 +143,8 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) { - cctx->customMem.customFree(cctx->customMem.opaque, cctx->workSpace); + if (cctx==NULL) return 0; /* support free on NULL */ + if (cctx->workSpace) cctx->customMem.customFree(cctx->customMem.opaque, cctx->workSpace); cctx->customMem.customFree(cctx->customMem.opaque, cctx); return 0; /* reserved as a potential error code in the future */ } diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index e9e84ef7..01937bb0 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -106,6 +106,7 @@ ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem) memset(zbd, 0, sizeof(ZBUFF_DCtx)); memcpy(&zbd->customMem, &customMem, sizeof(ZSTD_customMem)); zbd->zd = ZSTD_createDCtx_advanced(customMem); + if (zbd->zd == NULL) { ZBUFF_freeDCtx(zbd); return NULL; } zbd->stage = ZBUFFds_init; return zbd; } @@ -114,8 +115,8 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd) { if (zbd==NULL) return 0; /* support free on null */ ZSTD_freeDCtx(zbd->zd); - zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); - zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); + if (zbd->inBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); + if (zbd->outBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); zbd->customMem.customFree(zbd->customMem.opaque, zbd); return 0; } diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 128d941e..cb2faab7 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -172,6 +172,7 @@ ZSTD_DCtx* ZSTD_createDCtx(void) size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) { + if (dctx==NULL) return 0; /* support free on NULL */ dctx->customMem.customFree(dctx->customMem.opaque, dctx); return 0; /* reserved as a potential error code in the future */ } diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index 245c57d6..21d56c5e 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -57,9 +57,5 @@ $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwra $(CC) $(CFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c clean: - -$(RM) $(ZLIBWRAPPER_PATH)/*.o - -$(RM) $(EXAMPLE_PATH)/*.o - -$(RM) *.o - -$(RM) *.exe - -$(RM) foo.gz - + -$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_d example_zstd + @echo Cleaning completed From 4af2c9de0cd90d952d2b53ceda376da696c895be Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 3 Jun 2016 17:39:31 +0200 Subject: [PATCH 21/22] zlibWrapper: improved error handling --- zlibWrapper/zstd_zlibwrapper.c | 84 ++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index f3f6ea39..42ac5c04 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -29,7 +29,6 @@ - zstd source repository : https://github.com/Cyan4973/zstd */ -#include /* fprintf */ #include /* va_list */ #include #include "zstd_zlibwrapper.h" @@ -47,13 +46,18 @@ #define LOG_WRAPPER(...) // printf(__VA_ARGS__) -#define FINISH_WITH_ERR(msg) { \ - fprintf(stderr, "ERROR: %s\n", msg); \ +#define FINISH_WITH_GZ_ERR(msg) { \ + (void)msg; \ + return Z_MEM_ERROR; \ +} + +#define FINISH_WITH_ERR(strm, message) { \ + strm->msg = message; \ return Z_MEM_ERROR; \ } #define FINISH_WITH_NULL_ERR(msg) { \ - fprintf(stderr, "ERROR: %s\n", msg); \ + (void)msg; \ return NULL; \ } @@ -220,7 +224,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) strm->avail_in -= srcSize; } - if (flush == Z_FULL_FLUSH) FINISH_WITH_ERR("Z_FULL_FLUSH is not supported!"); + if (flush == Z_FULL_FLUSH) FINISH_WITH_ERR(strm, "Z_FULL_FLUSH is not supported!"); if (flush == Z_FINISH || flush == Z_FULL_FLUSH) { size_t bytesLeft; @@ -519,7 +523,7 @@ ZEXTERN int ZEXPORT z_deflateCopy OF((z_streamp dest, { if (!g_useZSTD) return deflateCopy(dest, source); - FINISH_WITH_ERR("deflateCopy is not supported!"); + FINISH_WITH_ERR(source, "deflateCopy is not supported!"); } @@ -527,7 +531,7 @@ ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm)) { if (!g_useZSTD) return deflateReset(strm); - FINISH_WITH_ERR("deflateReset is not supported!"); + FINISH_WITH_ERR(strm, "deflateReset is not supported!"); } @@ -539,7 +543,7 @@ ZEXTERN int ZEXPORT z_deflateTune OF((z_streamp strm, { if (!g_useZSTD) return deflateTune(strm, good_length, max_lazy, nice_length, max_chain); - FINISH_WITH_ERR("deflateTune is not supported!"); + FINISH_WITH_ERR(strm, "deflateTune is not supported!"); } @@ -550,7 +554,7 @@ ZEXTERN int ZEXPORT z_deflatePending OF((z_streamp strm, { if (!g_useZSTD) return deflatePending(strm, pending, bits); - FINISH_WITH_ERR("deflatePending is not supported!"); + FINISH_WITH_ERR(strm, "deflatePending is not supported!"); } #endif @@ -561,7 +565,7 @@ ZEXTERN int ZEXPORT z_deflatePrime OF((z_streamp strm, { if (!g_useZSTD) return deflatePrime(strm, bits, value); - FINISH_WITH_ERR("deflatePrime is not supported!"); + FINISH_WITH_ERR(strm, "deflatePrime is not supported!"); } @@ -570,7 +574,7 @@ ZEXTERN int ZEXPORT z_deflateSetHeader OF((z_streamp strm, { if (!g_useZSTD) return deflateSetHeader(strm, head); - FINISH_WITH_ERR("deflateSetHeader is not supported!"); + FINISH_WITH_ERR(strm, "deflateSetHeader is not supported!"); } @@ -584,7 +588,7 @@ ZEXTERN int ZEXPORT z_inflateGetDictionary OF((z_streamp strm, { if (!strm->reserved) return inflateGetDictionary(strm, dictionary, dictLength); - FINISH_WITH_ERR("inflateGetDictionary is not supported!"); + FINISH_WITH_ERR(strm, "inflateGetDictionary is not supported!"); } #endif @@ -594,7 +598,7 @@ ZEXTERN int ZEXPORT z_inflateCopy OF((z_streamp dest, { if (!g_useZSTD) return inflateCopy(dest, source); - FINISH_WITH_ERR("inflateCopy is not supported!"); + FINISH_WITH_ERR(source, "inflateCopy is not supported!"); } @@ -602,7 +606,7 @@ ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm)) { if (!strm->reserved) return inflateReset(strm); - FINISH_WITH_ERR("inflateReset is not supported!"); + FINISH_WITH_ERR(strm, "inflateReset is not supported!"); } @@ -612,7 +616,7 @@ ZEXTERN int ZEXPORT z_inflateReset2 OF((z_streamp strm, { if (!strm->reserved) return inflateReset2(strm, windowBits); - FINISH_WITH_ERR("inflateReset2 is not supported!"); + FINISH_WITH_ERR(strm, "inflateReset2 is not supported!"); } #endif @@ -622,7 +626,7 @@ ZEXTERN long ZEXPORT z_inflateMark OF((z_streamp strm)) { if (!strm->reserved) return inflateMark(strm); - FINISH_WITH_ERR("inflateMark is not supported!"); + FINISH_WITH_ERR(strm, "inflateMark is not supported!"); } #endif @@ -633,7 +637,7 @@ ZEXTERN int ZEXPORT z_inflatePrime OF((z_streamp strm, { if (!strm->reserved) return inflatePrime(strm, bits, value); - FINISH_WITH_ERR("inflatePrime is not supported!"); + FINISH_WITH_ERR(strm, "inflatePrime is not supported!"); } @@ -642,7 +646,7 @@ ZEXTERN int ZEXPORT z_inflateGetHeader OF((z_streamp strm, { if (!strm->reserved) return inflateGetHeader(strm, head); - FINISH_WITH_ERR("inflateGetHeader is not supported!"); + FINISH_WITH_ERR(strm, "inflateGetHeader is not supported!"); } @@ -653,7 +657,7 @@ ZEXTERN int ZEXPORT z_inflateBackInit_ OF((z_streamp strm, int windowBits, { if (!strm->reserved) return inflateBackInit_(strm, windowBits, window, version, stream_size); - FINISH_WITH_ERR("inflateBackInit is not supported!"); + FINISH_WITH_ERR(strm, "inflateBackInit is not supported!"); } @@ -663,7 +667,7 @@ ZEXTERN int ZEXPORT z_inflateBack OF((z_streamp strm, { if (!strm->reserved) return inflateBack(strm, in, in_desc, out, out_desc); - FINISH_WITH_ERR("inflateBack is not supported!"); + FINISH_WITH_ERR(strm, "inflateBack is not supported!"); } @@ -671,7 +675,7 @@ ZEXTERN int ZEXPORT z_inflateBackEnd OF((z_streamp strm)) { if (!strm->reserved) return inflateBackEnd(strm); - FINISH_WITH_ERR("inflateBackEnd is not supported!"); + FINISH_WITH_ERR(strm, "inflateBackEnd is not supported!"); } @@ -762,7 +766,7 @@ ZEXTERN int ZEXPORT z_gzbuffer OF((gzFile file, unsigned size)) { if (!g_useZSTD) return gzbuffer(file, size); - FINISH_WITH_ERR("gzbuffer is not supported!"); + FINISH_WITH_GZ_ERR("gzbuffer is not supported!"); } @@ -770,7 +774,7 @@ ZEXTERN z_off_t ZEXPORT z_gzoffset OF((gzFile file)) { if (!g_useZSTD) return gzoffset(file); - FINISH_WITH_ERR("gzoffset is not supported!"); + FINISH_WITH_GZ_ERR("gzoffset is not supported!"); } @@ -778,7 +782,7 @@ ZEXTERN int ZEXPORT z_gzclose_r OF((gzFile file)) { if (!g_useZSTD) return gzclose_r(file); - FINISH_WITH_ERR("gzclose_r is not supported!"); + FINISH_WITH_GZ_ERR("gzclose_r is not supported!"); } @@ -786,7 +790,7 @@ ZEXTERN int ZEXPORT z_gzclose_w OF((gzFile file)) { if (!g_useZSTD) return gzclose_w(file); - FINISH_WITH_ERR("gzclose_w is not supported!"); + FINISH_WITH_GZ_ERR("gzclose_w is not supported!"); } #endif @@ -795,7 +799,7 @@ ZEXTERN int ZEXPORT z_gzsetparams OF((gzFile file, int level, int strategy)) { if (!g_useZSTD) return gzsetparams(file, level, strategy); - FINISH_WITH_ERR("gzsetparams is not supported!"); + FINISH_WITH_GZ_ERR("gzsetparams is not supported!"); } @@ -803,7 +807,7 @@ ZEXTERN int ZEXPORT z_gzread OF((gzFile file, voidp buf, unsigned len)) { if (!g_useZSTD) return gzread(file, buf, len); - FINISH_WITH_ERR("gzread is not supported!"); + FINISH_WITH_GZ_ERR("gzread is not supported!"); } @@ -812,7 +816,7 @@ ZEXTERN int ZEXPORT z_gzwrite OF((gzFile file, { if (!g_useZSTD) return gzwrite(file, buf, len); - FINISH_WITH_ERR("gzwrite is not supported!"); + FINISH_WITH_GZ_ERR("gzwrite is not supported!"); } @@ -834,7 +838,7 @@ ZEXTERN int ZEXPORTVA z_gzprintf OF((gzFile file, const char *format, ...)) // printf("gzprintf ret=%d\n", ret); return ret; } - FINISH_WITH_ERR("gzprintf is not supported!"); + FINISH_WITH_GZ_ERR("gzprintf is not supported!"); } @@ -842,7 +846,7 @@ ZEXTERN int ZEXPORT z_gzputs OF((gzFile file, const char *s)) { if (!g_useZSTD) return gzputs(file, s); - FINISH_WITH_ERR("gzputs is not supported!"); + FINISH_WITH_GZ_ERR("gzputs is not supported!"); } @@ -858,7 +862,7 @@ ZEXTERN int ZEXPORT z_gzputc OF((gzFile file, int c)) { if (!g_useZSTD) return gzputc(file, c); - FINISH_WITH_ERR("gzputc is not supported!"); + FINISH_WITH_GZ_ERR("gzputc is not supported!"); } @@ -870,7 +874,7 @@ ZEXTERN int ZEXPORT z_gzgetc OF((gzFile file)) { if (!g_useZSTD) return gzgetc(file); - FINISH_WITH_ERR("gzgetc is not supported!"); + FINISH_WITH_GZ_ERR("gzgetc is not supported!"); } @@ -878,7 +882,7 @@ ZEXTERN int ZEXPORT z_gzungetc OF((int c, gzFile file)) { if (!g_useZSTD) return gzungetc(c, file); - FINISH_WITH_ERR("gzungetc is not supported!"); + FINISH_WITH_GZ_ERR("gzungetc is not supported!"); } @@ -886,7 +890,7 @@ ZEXTERN int ZEXPORT z_gzflush OF((gzFile file, int flush)) { if (!g_useZSTD) return gzflush(file, flush); - FINISH_WITH_ERR("gzflush is not supported!"); + FINISH_WITH_GZ_ERR("gzflush is not supported!"); } @@ -894,7 +898,7 @@ ZEXTERN z_off_t ZEXPORT z_gzseek OF((gzFile file, z_off_t offset, int whence)) { if (!g_useZSTD) return gzseek(file, offset, whence); - FINISH_WITH_ERR("gzseek is not supported!"); + FINISH_WITH_GZ_ERR("gzseek is not supported!"); } @@ -902,7 +906,7 @@ ZEXTERN int ZEXPORT z_gzrewind OF((gzFile file)) { if (!g_useZSTD) return gzrewind(file); - FINISH_WITH_ERR("gzrewind is not supported!"); + FINISH_WITH_GZ_ERR("gzrewind is not supported!"); } @@ -910,7 +914,7 @@ ZEXTERN z_off_t ZEXPORT z_gztell OF((gzFile file)) { if (!g_useZSTD) return gztell(file); - FINISH_WITH_ERR("gztell is not supported!"); + FINISH_WITH_GZ_ERR("gztell is not supported!"); } @@ -918,7 +922,7 @@ ZEXTERN int ZEXPORT z_gzeof OF((gzFile file)) { if (!g_useZSTD) return gzeof(file); - FINISH_WITH_ERR("gzeof is not supported!"); + FINISH_WITH_GZ_ERR("gzeof is not supported!"); } @@ -926,7 +930,7 @@ ZEXTERN int ZEXPORT z_gzdirect OF((gzFile file)) { if (!g_useZSTD) return gzdirect(file); - FINISH_WITH_ERR("gzdirect is not supported!"); + FINISH_WITH_GZ_ERR("gzdirect is not supported!"); } @@ -934,7 +938,7 @@ ZEXTERN int ZEXPORT z_gzclose OF((gzFile file)) { if (!g_useZSTD) return gzclose(file); - FINISH_WITH_ERR("gzclose is not supported!"); + FINISH_WITH_GZ_ERR("gzclose is not supported!"); } From de2c92f6e09ebb4068ee74748c7ff5866ece89a6 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 3 Jun 2016 19:44:03 +0200 Subject: [PATCH 22/22] zlibWrapper: compress uses ZWRAP_DEFAULT_CLEVEL --- zlibWrapper/zstd_zlibwrapper.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 42ac5c04..6bd82045 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -29,11 +29,11 @@ - zstd source repository : https://github.com/Cyan4973/zstd */ -#include /* va_list */ +#include /* va_list */ #include #include "zstd_zlibwrapper.h" #include "zstd.h" -#include "zstd_static.h" /* ZSTD_MAGICNUMBER */ +#include "zstd_static.h" /* ZSTD_MAGICNUMBER */ #include "zbuff.h" #include "zbuff_static.h" /* ZBUFF_createCCtx_advanced */ #include "zstd_internal.h" /* defaultCustomMem */ @@ -693,7 +693,7 @@ ZEXTERN int ZEXPORT z_compress OF((Bytef *dest, uLongf *destLen, return compress(dest, destLen, source, sourceLen); { size_t dstCapacity = *destLen; - size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, -1); + size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, ZWRAP_DEFAULT_CLEVEL); LOG_WRAPPER("z_compress sourceLen=%d dstCapacity=%d\n", (int)sourceLen, (int)dstCapacity); if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; *destLen = errorCode;