diff --git a/Makefile b/Makefile index ac0c583f..20ae31e4 100644 --- a/Makefile +++ b/Makefile @@ -22,15 +22,18 @@ endif .PHONY: default all zlibwrapper zstd clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan -default: zstd +default: libzstd zstd all: $(MAKE) -C $(ZSTDDIR) $@ $(MAKE) -C $(PRGDIR) $@ zstd32 $(MAKE) -C $(TESTDIR) $@ all32 +libzstd: + @$(MAKE) -C $(ZSTDDIR) + zstd: - $(MAKE) -C $(PRGDIR) + @$(MAKE) -C $(PRGDIR) cp $(PRGDIR)/zstd . zlibwrapper: @@ -48,18 +51,18 @@ clean: @echo Cleaning completed -#---------------------------------------------------------------------------------- -#make install is validated only for Linux, OSX, kFreeBSD, Hurd and some BSD targets -#---------------------------------------------------------------------------------- +#------------------------------------------------------------------------------ +# make install is validated only for Linux, OSX, Hurd and some BSD targets +#------------------------------------------------------------------------------ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD)) HOST_OS = POSIX install: - $(MAKE) -C $(ZSTDDIR) $@ - $(MAKE) -C $(PRGDIR) $@ + @$(MAKE) -C $(ZSTDDIR) $@ + @$(MAKE) -C $(PRGDIR) $@ uninstall: - $(MAKE) -C $(ZSTDDIR) $@ - $(MAKE) -C $(PRGDIR) $@ + @$(MAKE) -C $(ZSTDDIR) $@ + @$(MAKE) -C $(PRGDIR) $@ travis-install: $(MAKE) install PREFIX=~/install_test_dir diff --git a/NEWS b/NEWS index ad56d17a..570d6679 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +v1.1.1 +New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption +Changed : zstd_errors.h is now part of include installation + v1.1.0 New : contrib/pzstd, parallel version of zstd, by Nick Terrell added : NetBSD install target (#338) diff --git a/build/VS2005/fullbench/fullbench.vcproj b/build/VS2005/fullbench/fullbench.vcproj index 17af9c81..c28d1f69 100644 --- a/build/VS2005/fullbench/fullbench.vcproj +++ b/build/VS2005/fullbench/fullbench.vcproj @@ -335,6 +335,10 @@ RelativePath="..\..\..\lib\common\entropy_common.c" > + + @@ -394,7 +398,7 @@ > + + @@ -398,7 +402,7 @@ > + + @@ -450,7 +454,7 @@ > + + @@ -426,7 +430,7 @@ > + + @@ -395,7 +399,7 @@ > + + @@ -399,7 +403,7 @@ > + + @@ -451,7 +455,7 @@ > + + @@ -427,7 +431,7 @@ > + @@ -175,6 +176,7 @@ + diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj index a436b038..020e521a 100644 --- a/build/VS2010/fuzzer/fuzzer.vcxproj +++ b/build/VS2010/fuzzer/fuzzer.vcxproj @@ -157,6 +157,7 @@ + @@ -176,6 +177,7 @@ + diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj index eb7e7b50..181bbe6d 100644 --- a/build/VS2010/zstd/zstd.vcxproj +++ b/build/VS2010/zstd/zstd.vcxproj @@ -20,6 +20,7 @@ + @@ -54,6 +55,7 @@ + @@ -220,4 +222,4 @@ - \ No newline at end of file + diff --git a/build/VS2010/zstdlib/zstdlib.vcxproj b/build/VS2010/zstdlib/zstdlib.vcxproj index b97808dd..d32b4861 100644 --- a/build/VS2010/zstdlib/zstdlib.vcxproj +++ b/build/VS2010/zstdlib/zstdlib.vcxproj @@ -20,6 +20,7 @@ + @@ -39,9 +40,11 @@ + + - + diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index c984145b..f970fe7e 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -59,6 +59,7 @@ MESSAGE("ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}") SET(Sources ${LIBRARY_DIR}/common/entropy_common.c ${LIBRARY_DIR}/common/zstd_common.c + ${LIBRARY_DIR}/common/error_private.c ${LIBRARY_DIR}/common/xxhash.c ${LIBRARY_DIR}/common/fse_decompress.c ${LIBRARY_DIR}/compress/fse_compress.c @@ -74,7 +75,7 @@ SET(Sources SET(Headers ${LIBRARY_DIR}/common/bitstream.h ${LIBRARY_DIR}/common/error_private.h - ${LIBRARY_DIR}/common/error_public.h + ${LIBRARY_DIR}/common/zstd_errors.h ${LIBRARY_DIR}/common/fse.h ${LIBRARY_DIR}/common/huf.h ${LIBRARY_DIR}/common/mem.h diff --git a/contrib/pzstd/test/OptionsTest.cpp b/contrib/pzstd/test/OptionsTest.cpp index e7d4b2b3..b3efe2b7 100644 --- a/contrib/pzstd/test/OptionsTest.cpp +++ b/contrib/pzstd/test/OptionsTest.cpp @@ -180,12 +180,6 @@ TEST(Options, GetOutputFile) { EXPECT_SUCCESS(options.parse(args.size(), args.data())); EXPECT_EQ("x.zst", options.getOutputFile(options.inputFiles[0])); } - { - Options options; - auto args = makeArray("-o-"); - EXPECT_FAILURE(options.parse(args.size(), args.data())); - EXPECT_EQ("-", options.getOutputFile(options.inputFiles[0])); - } { Options options; auto args = makeArray("x", "y", "-o", nullOutput); diff --git a/examples/Makefile b/examples/Makefile index 54602dfe..8ce6a258 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -1,26 +1,11 @@ -# ########################################################################## -# ZSTD educational examples - Makefile -# Copyright (C) Yann Collet 2016 +# ################################################################ +# Copyright (c) 2016-present, Yann Collet, Facebook, Inc. +# All rights reserved. # -# GPL v2 License -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# You can contact the author at : -# - zstd homepage : http://www.zstd.net/ -# ########################################################################## +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. +# ################################################################ # This Makefile presumes libzstd is installed, using `sudo make install` diff --git a/lib/Makefile b/lib/Makefile index 4fb8ed9d..1117b491 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -97,19 +97,21 @@ install: libzstd libzstd.pc @cp -a libzstd.pc $(DESTDIR)$(LIBDIR)/pkgconfig/ @install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h - @install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h + @install -m 644 common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h + @install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h # Deprecated streaming functions @install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h @echo zstd static and shared library installed uninstall: - $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT) - $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) - $(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/libzstd.pc - $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) - $(RM) $(DESTDIR)$(LIBDIR)/libzstd.a - $(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h - $(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h - $(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h + @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.a + @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT) + @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) + @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) + @$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/libzstd.pc + @$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h + @$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h + @$(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h # Deprecated streaming functions + @$(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h @echo zstd libraries successfully uninstalled endif diff --git a/lib/common/error_private.c b/lib/common/error_private.c new file mode 100644 index 00000000..a0fa1724 --- /dev/null +++ b/lib/common/error_private.c @@ -0,0 +1,43 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +/* The purpose of this file is to have a single list of error strings embedded in binary */ + +#include "error_private.h" + +const char* ERR_getErrorString(ERR_enum code) +{ + static const char* const notErrorCode = "Unspecified error code"; + switch( code ) + { + case PREFIX(no_error): return "No error detected"; + case PREFIX(GENERIC): return "Error (generic)"; + case PREFIX(prefix_unknown): return "Unknown frame descriptor"; + case PREFIX(version_unsupported): return "Version not supported"; + case PREFIX(parameter_unknown): return "Unknown parameter type"; + case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; + case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; + case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; + case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; + case PREFIX(init_missing): return "Context should be init first"; + case PREFIX(memory_allocation): return "Allocation error : not enough memory"; + case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; + case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; + case PREFIX(srcSize_wrong): return "Src size incorrect"; + case PREFIX(corruption_detected): return "Corrupted block detected"; + case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; + case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; + case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; + case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; + case PREFIX(dictionary_wrong): return "Dictionary mismatch"; + case PREFIX(maxCode): + default: return notErrorCode; + } +} diff --git a/lib/common/error_private.h b/lib/common/error_private.h index d27e15af..1bc2e495 100644 --- a/lib/common/error_private.h +++ b/lib/common/error_private.h @@ -21,7 +21,7 @@ extern "C" { * Dependencies ******************************************/ #include /* size_t */ -#include "error_public.h" /* enum list */ +#include "zstd_errors.h" /* enum list */ /* **************************************** @@ -62,35 +62,7 @@ ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) retu * Error Strings ******************************************/ -ERR_STATIC const char* ERR_getErrorString(ERR_enum code) -{ - static const char* notErrorCode = "Unspecified error code"; - switch( code ) - { - case PREFIX(no_error): return "No error detected"; - case PREFIX(GENERIC): return "Error (generic)"; - case PREFIX(prefix_unknown): return "Unknown frame descriptor"; - case PREFIX(version_unsupported): return "Version not supported"; - case PREFIX(parameter_unknown): return "Unknown parameter type"; - case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; - case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; - case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; - case PREFIX(init_missing): return "Context should be init first"; - case PREFIX(memory_allocation): return "Allocation error : not enough memory"; - case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; - case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; - case PREFIX(srcSize_wrong): return "Src size incorrect"; - case PREFIX(corruption_detected): return "Corrupted block detected"; - case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; - case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; - case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; - case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; - case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; - case PREFIX(dictionary_wrong): return "Dictionary mismatch"; - case PREFIX(maxCode): - default: return notErrorCode; - } -} +const char* ERR_getErrorString(ERR_enum code); /* error_private.c */ ERR_STATIC const char* ERR_getErrorName(size_t code) { diff --git a/lib/common/fse.h b/lib/common/fse.h index 720d54b1..cecb1aef 100644 --- a/lib/common/fse.h +++ b/lib/common/fse.h @@ -503,6 +503,7 @@ MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePt BIT_flushBits(bitC); } + /* ====== Decompression ====== */ typedef struct { @@ -581,14 +582,19 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) * Increasing memory usage improves compression ratio * Reduced memory usage can improve speed, due to cache effect * Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ -#define FSE_MAX_MEMORY_USAGE 14 -#define FSE_DEFAULT_MEMORY_USAGE 13 +#ifndef FSE_MAX_MEMORY_USAGE +# define FSE_MAX_MEMORY_USAGE 14 +#endif +#ifndef FSE_DEFAULT_MEMORY_USAGE +# define FSE_DEFAULT_MEMORY_USAGE 13 +#endif /*!FSE_MAX_SYMBOL_VALUE : * Maximum symbol value authorized. * Required for proper stack allocation */ -#define FSE_MAX_SYMBOL_VALUE 255 - +#ifndef FSE_MAX_SYMBOL_VALUE +# define FSE_MAX_SYMBOL_VALUE 255 +#endif /* ************************************************************** * template functions type & suffix diff --git a/lib/common/error_public.h b/lib/common/zstd_errors.h similarity index 91% rename from lib/common/error_public.h rename to lib/common/zstd_errors.h index d46abd2c..50dc4f72 100644 --- a/lib/common/error_public.h +++ b/lib/common/zstd_errors.h @@ -7,8 +7,8 @@ * of patent rights can be found in the PATENTS file in the same directory. */ -#ifndef ERROR_PUBLIC_H_MODULE -#define ERROR_PUBLIC_H_MODULE +#ifndef ZSTD_ERRORS_H_398273423 +#define ZSTD_ERRORS_H_398273423 #if defined (__cplusplus) extern "C" { @@ -29,6 +29,7 @@ typedef enum { ZSTD_error_parameter_unknown, ZSTD_error_frameParameter_unsupported, ZSTD_error_frameParameter_unsupportedBy32bits, + ZSTD_error_frameParameter_windowTooLarge, ZSTD_error_compressionParameter_unsupported, ZSTD_error_init_missing, ZSTD_error_memory_allocation, @@ -56,4 +57,4 @@ const char* ZSTD_getErrorString(ZSTD_ErrorCode code); } #endif -#endif /* ERROR_PUBLIC_H_MODULE */ +#endif /* ZSTD_ERRORS_H_398273423 */ diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index b7d3d77a..78784aa3 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -155,13 +155,14 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si } } /* fill val */ - { U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; - U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; + { U16 nbPerRank[HUF_TABLELOG_MAX+2] = {0}; /* support w=0=>n=tableLog+1 */ + U16 valPerRank[HUF_TABLELOG_MAX+2] = {0}; { U32 n; for (n=0; n0; n--) { - valPerRank[n] = min; /* get starting value within each rank */ + U32 n; for (n=tableLog; n>0; n--) { /* start at n=tablelog <-> w=1 */ + valPerRank[n] = min; /* get starting value within each rank */ min += nbPerRank[n]; min >>= 1; } } diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 94f4b5a2..e71873b0 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1458,7 +1458,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co const U32 dictLimit = zc->dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const prefixStart = base + dictLimit; - const BYTE* match = base + matchIndex; + const BYTE* match; const U32 current = (U32)(ip-base); const U32 btLow = btMask >= current ? 0 : current - btMask; U32* smallerPtr = bt + 2*(current&btMask); @@ -2235,7 +2235,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, BYTE* op = ostart; U32 const maxDist = 1 << cctx->params.cParams.windowLog; - if (cctx->params.fParams.checksumFlag) + if (cctx->params.fParams.checksumFlag && srcSize) XXH64_update(&cctx->xxhState, src, srcSize); while (remaining) { @@ -2688,7 +2688,9 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_pa return NULL; } - memcpy(dictContent, dict, dictSize); + if (dictSize) { + memcpy(dictContent, dict, dictSize); + } { size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0); if (ZSTD_isError(errorCode)) { ZSTD_free(dictContent, customMem); diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index cb587290..cea67056 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -401,7 +401,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, ZSTD_rescaleFreqs(seqStorePtr); ip += (ip==prefixStart); { U32 i; for (i=0; irep[i]; } - inr = ip; + //inr = ip; /* Match Loop */ while (ip < ilimit) { @@ -657,7 +657,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, ctx->nextToUpdate3 = ctx->nextToUpdate; ZSTD_rescaleFreqs(seqStorePtr); ip += (ip==prefixStart); - inr = ip; + //inr = ip; /* Match Loop */ while (ip < ilimit) { @@ -666,7 +666,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, U32 current = (U32)(ip-base); memset(opt, 0, sizeof(ZSTD_optimal_t)); last_pos = 0; - inr = ip; + //inr = ip; opt[0].litlen = (U32)(ip - anchor); /* check repCode */ diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 47b5f42c..00b069e9 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -248,7 +248,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t if (!singleSegment) { BYTE const wlByte = ip[pos++]; U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; - if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported); + if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_windowTooLarge); /* avoids issue with 1 << windowLog */ windowSize = (1U << windowLog); windowSize += (windowSize >> 3) * (wlByte&7); } @@ -270,7 +270,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t case 3 : frameContentSize = MEM_readLE64(ip+pos); break; } if (!windowSize) windowSize = (U32)frameContentSize; - if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported); + if (windowSize > windowSizeMax) return ERROR(frameParameter_windowTooLarge); fparamsPtr->frameContentSize = frameContentSize; fparamsPtr->windowSize = windowSize; fparamsPtr->dictID = dictID; @@ -807,7 +807,8 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState) if (ofCode <= 1) { offset += (llCode==0); if (offset) { - size_t const temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; seqState->prevOffset[1] = seqState->prevOffset[0]; seqState->prevOffset[0] = offset = temp; @@ -878,7 +879,12 @@ size_t ZSTD_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; + if (op > oend_w) { + memmove(op, match, sequence.matchLength); + return sequenceLength; + } } } + /* Requirement: op <= oend_w */ /* match within prefix */ if (sequence.offset < 8) { @@ -1397,7 +1403,9 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_cu return NULL; } - memcpy(dictContent, dict, dictSize); + if (dictSize) { + memcpy(dictContent, dict, dictSize); + } { size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize); if (ZSTD_isError(errorCode)) { ZSTD_free(dictContent, customMem); @@ -1568,7 +1576,7 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, switch(paramType) { default : return ERROR(parameter_unknown); - case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue; break; + case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); break; } return 0; } @@ -1649,7 +1657,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } } zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); - if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_unsupported); + if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge); /* Adapt buffer sizes to frame header instructions */ { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 47a82af1..b3f20b12 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -371,21 +371,22 @@ static dictItem ZDICT_analyzePos( static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip) { const U32 tableSize = table->pos; - const U32 max = elt.pos + (elt.length-1); + const U32 eltEnd = elt.pos + elt.length; /* tail overlap */ U32 u; for (u=1; u elt.pos) && (table[u].pos < max)) { /* overlap */ + if ((table[u].pos > elt.pos) && (table[u].pos <= eltEnd)) { /* overlap, existing > new */ /* append */ U32 addedLength = table[u].pos - elt.pos; table[u].length += addedLength; table[u].pos = elt.pos; table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */ - table[u].savings += elt.length / 8; /* rough approx */ + table[u].savings += elt.length / 8; /* rough approx bonus */ elt = table[u]; + /* sort : improve rank */ while ((u>1) && (table[u-1].savings < elt.savings)) - table[u] = table[u-1], u--; + table[u] = table[u-1], u--; table[u] = elt; return u; } } @@ -393,14 +394,15 @@ static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip) /* front overlap */ for (u=1; u elt.pos) && (table[u].pos < elt.pos)) { /* overlap */ + if ((table[u].pos + table[u].length >= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */ /* append */ - int addedLength = (elt.pos + elt.length) - (table[u].pos + table[u].length); - table[u].savings += elt.length / 8; /* rough approx */ - if (addedLength > 0) { /* otherwise, already included */ + int addedLength = (int)eltEnd - (table[u].pos + table[u].length); + table[u].savings += elt.length / 8; /* rough approx bonus */ + if (addedLength > 0) { /* otherwise, elt fully included into existing */ table[u].length += addedLength; table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */ } + /* sort : improve rank */ elt = table[u]; while ((u>1) && (table[u-1].savings < elt.savings)) table[u] = table[u-1], u--; @@ -811,7 +813,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, MEM_writeLE32(dstPtr+4, repStartValue[1]); MEM_writeLE32(dstPtr+8, repStartValue[2]); #endif - dstPtr += 12; + //dstPtr += 12; eSize += 12; _cleanup: diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index c9dcb94e..05e40aac 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -3107,8 +3107,13 @@ static size_t ZSTD_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; + if (op > oend_8) { + memmove(op, match, sequence.matchLength); + return sequenceLength; + } } } + /* Requirement: op <= oend_8 */ /* match within prefix */ if (sequence.offset < 8) diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 5027e2b8..3cb5bb4a 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -2032,13 +2032,14 @@ size_t HUFv05_decompress1X2_usingDTable( { BYTE* op = (BYTE*)dst; BYTE* const oend = op + dstSize; - size_t errorCode; const U32 dtLog = DTable[0]; const void* dtPtr = DTable; const HUFv05_DEltX2* const dt = ((const HUFv05_DEltX2*)dtPtr)+1; BITv05_DStream_t bitD; - errorCode = BITv05_initDStream(&bitD, cSrc, cSrcSize); - if (HUFv05_isError(errorCode)) return errorCode; + + if (dstSize <= cSrcSize) return ERROR(dstSize_tooSmall); + { size_t const errorCode = BITv05_initDStream(&bitD, cSrc, cSrcSize); + if (HUFv05_isError(errorCode)) return errorCode; } HUFv05_decodeStreamX2(op, &bitD, oend, dt, dtLog); @@ -3063,7 +3064,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx, size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr, FSEv05_DTable* DTableLL, FSEv05_DTable* DTableML, FSEv05_DTable* DTableOffb, - const void* src, size_t srcSize) + const void* src, size_t srcSize, U32 flagStaticTable) { const BYTE* const istart = (const BYTE* const)src; const BYTE* ip = istart; @@ -3118,6 +3119,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps FSEv05_buildDTable_raw(DTableLL, LLbits); break; case FSEv05_ENCODING_STATIC: + if (!flagStaticTable) return ERROR(corruption_detected); break; case FSEv05_ENCODING_DYNAMIC : default : /* impossible */ @@ -3141,6 +3143,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps FSEv05_buildDTable_raw(DTableOffb, Offbits); break; case FSEv05_ENCODING_STATIC: + if (!flagStaticTable) return ERROR(corruption_detected); break; case FSEv05_ENCODING_DYNAMIC : default : /* impossible */ @@ -3164,6 +3167,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps FSEv05_buildDTable_raw(DTableML, MLbits); break; case FSEv05_ENCODING_STATIC: + if (!flagStaticTable) return ERROR(corruption_detected); break; case FSEv05_ENCODING_DYNAMIC : default : /* impossible */ @@ -3312,7 +3316,12 @@ static size_t ZSTDv05_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; + if (op > oend_8) { + memmove(op, match, sequence.matchLength); + return sequenceLength; + } } } + /* Requirement: op <= oend_8 */ /* match within prefix */ if (sequence.offset < 8) { @@ -3371,7 +3380,7 @@ static size_t ZSTDv05_decompressSequences( /* Build Decoding Tables */ errorCode = ZSTDv05_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength, DTableLL, DTableML, DTableOffb, - ip, seqSize); + ip, seqSize, dctx->flagStaticTables); if (ZSTDv05_isError(errorCode)) return errorCode; ip += errorCode; diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index d9e89f80..96a84d3e 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -3466,7 +3466,12 @@ size_t ZSTDv06_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; + if (op > oend_8) { + memmove(op, match, sequence.matchLength); + return sequenceLength; + } } } + /* Requirement: op <= oend_8 */ /* match within prefix */ if (sequence.offset < 8) { diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index f4c8073f..62285238 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -3690,7 +3690,12 @@ size_t ZSTDv07_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; + if (op > oend_w) { + memmove(op, match, sequence.matchLength); + return sequenceLength; + } } } + /* Requirement: op <= oend_w */ /* match within prefix */ if (sequence.offset < 8) { diff --git a/lib/zstd.h b/lib/zstd.h index 2c140730..8039cc7d 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -55,7 +55,7 @@ ZSTDLIB_API unsigned ZSTD_versionNumber (void); /**< returns version number of #define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MINOR 1 -#define ZSTD_VERSION_RELEASE 0 +#define ZSTD_VERSION_RELEASE 1 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_QUOTE(str) #str diff --git a/programs/Makefile b/programs/Makefile index ed26f070..0ed7f3d0 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -76,17 +76,18 @@ default: zstd all: zstd - +$(ZSTDDECOMP_O): CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) +$(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP) $(ZSTDDECOMP_O): $(ZSTDDIR)/decompress/zstd_decompress.c - $(CC) $(ALIGN_LOOP) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -c -o $@ - -$(ZSTDDECOMP32_O): $(ZSTDDIR)/decompress/zstd_decompress.c - $(CC) -m32 $(ALIGN_LOOP) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -c -o $@ zstd : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \ zstdcli.c fileio.c bench.c datagen.c dibio.c $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(RES_FILE) -o $@$(EXT) + +$(ZSTDDECOMP32_O): $(ZSTDDIR)/decompress/zstd_decompress.c + $(CC) -m32 $(ALIGN_LOOP) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -c -o $@ + zstd32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \ zstdcli.c fileio.c bench.c datagen.c dibio.c $(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(RES32_FILE) -o $@$(EXT) @@ -153,11 +154,11 @@ install: zstd @echo zstd installation completed uninstall: - $(RM) $(DESTDIR)$(BINDIR)/zstdcat - $(RM) $(DESTDIR)$(BINDIR)/unzstd - $(RM) $(DESTDIR)$(BINDIR)/zstd$(EXT) - $(RM) $(DESTDIR)$(MANDIR)/zstdcat.1 - $(RM) $(DESTDIR)$(MANDIR)/unzstd.1 - $(RM) $(DESTDIR)$(MANDIR)/zstd.1 + @$(RM) $(DESTDIR)$(BINDIR)/zstdcat + @$(RM) $(DESTDIR)$(BINDIR)/unzstd + @$(RM) $(DESTDIR)$(BINDIR)/zstd$(EXT) + @$(RM) $(DESTDIR)$(MANDIR)/zstdcat.1 + @$(RM) $(DESTDIR)$(MANDIR)/unzstd.1 + @$(RM) $(DESTDIR)$(MANDIR)/zstd.1 @echo zstd programs successfully uninstalled endif diff --git a/programs/fileio.c b/programs/fileio.c index 56f22fe7..c4c308e0 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -8,13 +8,6 @@ */ -/* - Note : this file is part of zstd command line, which is not library. - The license of ZSTD library is BSD. - The license of this file is GPLv2. -*/ - - /* ************************************* * Tuning options ***************************************/ @@ -127,6 +120,9 @@ static U32 g_checksumFlag = 1; void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; } static U32 g_removeSrcFile = 0; void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); } +static U32 g_memLimit = 0; +void FIO_setMemLimit(unsigned memLimit) { g_memLimit = memLimit; } + /*-************************************* @@ -487,6 +483,7 @@ static dRess_t FIO_createDResources(const char* dictFileName) /* Allocation */ ress.dctx = ZSTD_createDStream(); if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZSTD_DStream"); + ZSTD_setDStreamParameter(ress.dctx, ZSTDdsp_maxWindowSize, g_memLimit); ress.srcBufferSize = ZSTD_DStreamInSize(); ress.srcBuffer = malloc(ress.srcBufferSize); ress.dstBufferSize = ZSTD_DStreamOutSize(); diff --git a/programs/fileio.h b/programs/fileio.h index 1e89aec2..60a7e0de 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -15,7 +15,6 @@ extern "C" { #endif - /* ************************************* * Special i/o constants **************************************/ @@ -37,6 +36,7 @@ void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdo void FIO_setDictIDFlag(unsigned dictIDFlag); void FIO_setChecksumFlag(unsigned checksumFlag); void FIO_setRemoveSrcFile(unsigned flag); +void FIO_setMemLimit(unsigned memLimit); /*-************************************* diff --git a/programs/zstdcli.c b/programs/zstdcli.c index fe97f965..9aa97cdd 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -8,13 +8,6 @@ */ -/* - Note : this is a user program, not part of libzstd. - The license of libzstd is BSD. - The license of this command line program is GPLv2. -*/ - - /*-************************************ * Tuning parameters **************************************/ @@ -32,7 +25,6 @@ **************************************/ #include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */ #include /* strcmp, strlen */ -#include /* toupper */ #include /* errno */ #include "fileio.h" #ifndef ZSTD_NOBENCH @@ -142,6 +134,7 @@ static int usage_advanced(const char* programName) DISPLAY( "--test : test compressed file integrity \n"); DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); #endif + DISPLAY( " -M# : Set a memory usage limit for decompression \n"); DISPLAY( "-- : All arguments after \"--\" are treated as files \n"); #ifndef ZSTD_NODICT DISPLAY( "\n"); @@ -179,14 +172,30 @@ static void waitEnter(void) } /*! readU32FromChar() : - @return : unsigned integer value reach from input in `char` format + @return : unsigned integer value read from input in `char` format + allows and interprets K, KB, KiB, M, MB and MiB suffix. Will also modify `*stringPtr`, advancing it to position where it stopped reading. - Note : this function can overflow if digit string > MAX_UINT */ + Note : function result can overflow if digit string > MAX_UINT */ static unsigned readU32FromChar(const char** stringPtr) { unsigned result = 0; while ((**stringPtr >='0') && (**stringPtr <='9')) result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + if ((**stringPtr=='K') || (**stringPtr=='M')) { + result <<= 10; + if (**stringPtr=='M') result <<= 10; + (*stringPtr)++ ; + if (**stringPtr=='i') (*stringPtr)++; + if (**stringPtr=='B') (*stringPtr)++; + } + return result; +} + +static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) +{ + size_t const comSize = strlen(longCommand); + unsigned const result = !strncmp(*stringPtr, longCommand, comSize); + if (result) *stringPtr += comSize; return result; } @@ -213,6 +222,7 @@ int main(int argCount, const char* argv[]) int cLevel = ZSTDCLI_CLEVEL_DEFAULT; int cLevelLast = 1; unsigned recursive = 0; + unsigned memLimit = 0; const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ unsigned filenameIdx = 0; const char* programName = argv[0]; @@ -231,8 +241,8 @@ int main(int argCount, const char* argv[]) /* init */ (void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */ (void)dictCLevel; (void)dictSelect; (void)dictID; /* not used when ZSTD_NODICT set */ - (void)decode; (void)cLevel; (void)testmode;/* not used when ZSTD_NOCOMPRESS set */ - (void)ultra; /* not used when ZSTD_NODECOMPRESS set */ + (void)decode; (void)cLevel; (void)testmode; /* not used when ZSTD_NOCOMPRESS set */ + (void)ultra; (void)memLimit; /* not used when ZSTD_NODECOMPRESS set */ if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); } filenameTable[0] = stdinmark; displayOut = stderr; @@ -247,16 +257,16 @@ int main(int argCount, const char* argv[]) if (!strcmp(programName, ZSTD_CAT)) { decode=1; forceStdout=1; displayLevel=1; outFileName=stdoutmark; } /* command switches */ - for(argNb=1; argNb /* clock_t */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */ #include "zstd.h" /* ZSTD_VERSION_STRING */ -#include "error_public.h" /* ZSTD_getErrorCode */ +#include "zstd_errors.h" /* ZSTD_getErrorCode */ #include "zdict.h" /* ZDICT_trainFromBuffer */ #include "datagen.h" /* RDG_genBuffer */ #include "mem.h" diff --git a/tests/playTests.sh b/tests/playTests.sh index d94d8fab..c5a58d62 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -81,6 +81,11 @@ $ZSTD -dc < tmp.zst > $INTOVOID # combine decompression, stdin & stdout $ZSTD -dc - < tmp.zst > $INTOVOID $ZSTD -d < tmp.zst > $INTOVOID # implicit stdout when stdin is used $ZSTD -d - < tmp.zst > $INTOVOID +$ECHO "test : impose memory limitation (must fail)" +$ZSTD -d -f tmp.zst -M2K -c > $INTOVOID && die "decompression needs more memory than allowed" +$ZSTD -d -f tmp.zst --memlimit=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command +$ZSTD -d -f tmp.zst --memory=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command +$ZSTD -d -f tmp.zst --memlimit-decompress=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command $ECHO "test : overwrite protection" $ZSTD -q tmp && die "overwrite check failed!" $ECHO "test : force overwrite"