diff --git a/Makefile b/Makefile
index ac0c583f..20ae31e4 100644
--- a/Makefile
+++ b/Makefile
@@ -22,15 +22,18 @@ endif
.PHONY: default all zlibwrapper zstd clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan
-default: zstd
+default: libzstd zstd
all:
$(MAKE) -C $(ZSTDDIR) $@
$(MAKE) -C $(PRGDIR) $@ zstd32
$(MAKE) -C $(TESTDIR) $@ all32
+libzstd:
+ @$(MAKE) -C $(ZSTDDIR)
+
zstd:
- $(MAKE) -C $(PRGDIR)
+ @$(MAKE) -C $(PRGDIR)
cp $(PRGDIR)/zstd .
zlibwrapper:
@@ -48,18 +51,18 @@ clean:
@echo Cleaning completed
-#----------------------------------------------------------------------------------
-#make install is validated only for Linux, OSX, kFreeBSD, Hurd and some BSD targets
-#----------------------------------------------------------------------------------
+#------------------------------------------------------------------------------
+# make install is validated only for Linux, OSX, Hurd and some BSD targets
+#------------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD))
HOST_OS = POSIX
install:
- $(MAKE) -C $(ZSTDDIR) $@
- $(MAKE) -C $(PRGDIR) $@
+ @$(MAKE) -C $(ZSTDDIR) $@
+ @$(MAKE) -C $(PRGDIR) $@
uninstall:
- $(MAKE) -C $(ZSTDDIR) $@
- $(MAKE) -C $(PRGDIR) $@
+ @$(MAKE) -C $(ZSTDDIR) $@
+ @$(MAKE) -C $(PRGDIR) $@
travis-install:
$(MAKE) install PREFIX=~/install_test_dir
diff --git a/NEWS b/NEWS
index ad56d17a..570d6679 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,7 @@
+v1.1.1
+New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption
+Changed : zstd_errors.h is now part of include installation
+
v1.1.0
New : contrib/pzstd, parallel version of zstd, by Nick Terrell
added : NetBSD install target (#338)
diff --git a/build/VS2005/fullbench/fullbench.vcproj b/build/VS2005/fullbench/fullbench.vcproj
index 17af9c81..c28d1f69 100644
--- a/build/VS2005/fullbench/fullbench.vcproj
+++ b/build/VS2005/fullbench/fullbench.vcproj
@@ -335,6 +335,10 @@
RelativePath="..\..\..\lib\common\entropy_common.c"
>
+
+
@@ -394,7 +398,7 @@
>
+
+
@@ -398,7 +402,7 @@
>
+
+
@@ -450,7 +454,7 @@
>
+
+
@@ -426,7 +430,7 @@
>
+
+
@@ -395,7 +399,7 @@
>
+
+
@@ -399,7 +403,7 @@
>
+
+
@@ -451,7 +455,7 @@
>
+
+
@@ -427,7 +431,7 @@
>
+
@@ -175,6 +176,7 @@
+
diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj
index a436b038..020e521a 100644
--- a/build/VS2010/fuzzer/fuzzer.vcxproj
+++ b/build/VS2010/fuzzer/fuzzer.vcxproj
@@ -157,6 +157,7 @@
+
@@ -176,6 +177,7 @@
+
diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj
index eb7e7b50..181bbe6d 100644
--- a/build/VS2010/zstd/zstd.vcxproj
+++ b/build/VS2010/zstd/zstd.vcxproj
@@ -20,6 +20,7 @@
+
@@ -54,6 +55,7 @@
+
@@ -220,4 +222,4 @@
-
\ No newline at end of file
+
diff --git a/build/VS2010/zstdlib/zstdlib.vcxproj b/build/VS2010/zstdlib/zstdlib.vcxproj
index b97808dd..d32b4861 100644
--- a/build/VS2010/zstdlib/zstdlib.vcxproj
+++ b/build/VS2010/zstdlib/zstdlib.vcxproj
@@ -20,6 +20,7 @@
+
@@ -39,9 +40,11 @@
+
+
-
+
diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt
index c984145b..f970fe7e 100644
--- a/build/cmake/lib/CMakeLists.txt
+++ b/build/cmake/lib/CMakeLists.txt
@@ -59,6 +59,7 @@ MESSAGE("ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}")
SET(Sources
${LIBRARY_DIR}/common/entropy_common.c
${LIBRARY_DIR}/common/zstd_common.c
+ ${LIBRARY_DIR}/common/error_private.c
${LIBRARY_DIR}/common/xxhash.c
${LIBRARY_DIR}/common/fse_decompress.c
${LIBRARY_DIR}/compress/fse_compress.c
@@ -74,7 +75,7 @@ SET(Sources
SET(Headers
${LIBRARY_DIR}/common/bitstream.h
${LIBRARY_DIR}/common/error_private.h
- ${LIBRARY_DIR}/common/error_public.h
+ ${LIBRARY_DIR}/common/zstd_errors.h
${LIBRARY_DIR}/common/fse.h
${LIBRARY_DIR}/common/huf.h
${LIBRARY_DIR}/common/mem.h
diff --git a/contrib/pzstd/test/OptionsTest.cpp b/contrib/pzstd/test/OptionsTest.cpp
index e7d4b2b3..b3efe2b7 100644
--- a/contrib/pzstd/test/OptionsTest.cpp
+++ b/contrib/pzstd/test/OptionsTest.cpp
@@ -180,12 +180,6 @@ TEST(Options, GetOutputFile) {
EXPECT_SUCCESS(options.parse(args.size(), args.data()));
EXPECT_EQ("x.zst", options.getOutputFile(options.inputFiles[0]));
}
- {
- Options options;
- auto args = makeArray("-o-");
- EXPECT_FAILURE(options.parse(args.size(), args.data()));
- EXPECT_EQ("-", options.getOutputFile(options.inputFiles[0]));
- }
{
Options options;
auto args = makeArray("x", "y", "-o", nullOutput);
diff --git a/examples/Makefile b/examples/Makefile
index 54602dfe..8ce6a258 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -1,26 +1,11 @@
-# ##########################################################################
-# ZSTD educational examples - Makefile
-# Copyright (C) Yann Collet 2016
+# ################################################################
+# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
#
-# GPL v2 License
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# You can contact the author at :
-# - zstd homepage : http://www.zstd.net/
-# ##########################################################################
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+# ################################################################
# This Makefile presumes libzstd is installed, using `sudo make install`
diff --git a/lib/Makefile b/lib/Makefile
index 4fb8ed9d..1117b491 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -97,19 +97,21 @@ install: libzstd libzstd.pc
@cp -a libzstd.pc $(DESTDIR)$(LIBDIR)/pkgconfig/
@install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a
@install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h
- @install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h
+ @install -m 644 common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h
+ @install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h # Deprecated streaming functions
@install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h
@echo zstd static and shared library installed
uninstall:
- $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
- $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
- $(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/libzstd.pc
- $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER)
- $(RM) $(DESTDIR)$(LIBDIR)/libzstd.a
- $(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h
- $(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h
- $(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h
+ @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.a
+ @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
+ @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
+ @$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER)
+ @$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/libzstd.pc
+ @$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h
+ @$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h
+ @$(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h # Deprecated streaming functions
+ @$(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h
@echo zstd libraries successfully uninstalled
endif
diff --git a/lib/common/error_private.c b/lib/common/error_private.c
new file mode 100644
index 00000000..a0fa1724
--- /dev/null
+++ b/lib/common/error_private.c
@@ -0,0 +1,43 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+/* The purpose of this file is to have a single list of error strings embedded in binary */
+
+#include "error_private.h"
+
+const char* ERR_getErrorString(ERR_enum code)
+{
+ static const char* const notErrorCode = "Unspecified error code";
+ switch( code )
+ {
+ case PREFIX(no_error): return "No error detected";
+ case PREFIX(GENERIC): return "Error (generic)";
+ case PREFIX(prefix_unknown): return "Unknown frame descriptor";
+ case PREFIX(version_unsupported): return "Version not supported";
+ case PREFIX(parameter_unknown): return "Unknown parameter type";
+ case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
+ case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
+ case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
+ case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
+ case PREFIX(init_missing): return "Context should be init first";
+ case PREFIX(memory_allocation): return "Allocation error : not enough memory";
+ case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
+ case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
+ case PREFIX(srcSize_wrong): return "Src size incorrect";
+ case PREFIX(corruption_detected): return "Corrupted block detected";
+ case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
+ case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
+ case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
+ case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
+ case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
+ case PREFIX(dictionary_wrong): return "Dictionary mismatch";
+ case PREFIX(maxCode):
+ default: return notErrorCode;
+ }
+}
diff --git a/lib/common/error_private.h b/lib/common/error_private.h
index d27e15af..1bc2e495 100644
--- a/lib/common/error_private.h
+++ b/lib/common/error_private.h
@@ -21,7 +21,7 @@ extern "C" {
* Dependencies
******************************************/
#include /* size_t */
-#include "error_public.h" /* enum list */
+#include "zstd_errors.h" /* enum list */
/* ****************************************
@@ -62,35 +62,7 @@ ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) retu
* Error Strings
******************************************/
-ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
-{
- static const char* notErrorCode = "Unspecified error code";
- switch( code )
- {
- case PREFIX(no_error): return "No error detected";
- case PREFIX(GENERIC): return "Error (generic)";
- case PREFIX(prefix_unknown): return "Unknown frame descriptor";
- case PREFIX(version_unsupported): return "Version not supported";
- case PREFIX(parameter_unknown): return "Unknown parameter type";
- case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
- case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
- case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
- case PREFIX(init_missing): return "Context should be init first";
- case PREFIX(memory_allocation): return "Allocation error : not enough memory";
- case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
- case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
- case PREFIX(srcSize_wrong): return "Src size incorrect";
- case PREFIX(corruption_detected): return "Corrupted block detected";
- case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
- case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
- case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
- case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
- case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
- case PREFIX(dictionary_wrong): return "Dictionary mismatch";
- case PREFIX(maxCode):
- default: return notErrorCode;
- }
-}
+const char* ERR_getErrorString(ERR_enum code); /* error_private.c */
ERR_STATIC const char* ERR_getErrorName(size_t code)
{
diff --git a/lib/common/fse.h b/lib/common/fse.h
index 720d54b1..cecb1aef 100644
--- a/lib/common/fse.h
+++ b/lib/common/fse.h
@@ -503,6 +503,7 @@ MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePt
BIT_flushBits(bitC);
}
+
/* ====== Decompression ====== */
typedef struct {
@@ -581,14 +582,19 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
* Increasing memory usage improves compression ratio
* Reduced memory usage can improve speed, due to cache effect
* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
-#define FSE_MAX_MEMORY_USAGE 14
-#define FSE_DEFAULT_MEMORY_USAGE 13
+#ifndef FSE_MAX_MEMORY_USAGE
+# define FSE_MAX_MEMORY_USAGE 14
+#endif
+#ifndef FSE_DEFAULT_MEMORY_USAGE
+# define FSE_DEFAULT_MEMORY_USAGE 13
+#endif
/*!FSE_MAX_SYMBOL_VALUE :
* Maximum symbol value authorized.
* Required for proper stack allocation */
-#define FSE_MAX_SYMBOL_VALUE 255
-
+#ifndef FSE_MAX_SYMBOL_VALUE
+# define FSE_MAX_SYMBOL_VALUE 255
+#endif
/* **************************************************************
* template functions type & suffix
diff --git a/lib/common/error_public.h b/lib/common/zstd_errors.h
similarity index 91%
rename from lib/common/error_public.h
rename to lib/common/zstd_errors.h
index d46abd2c..50dc4f72 100644
--- a/lib/common/error_public.h
+++ b/lib/common/zstd_errors.h
@@ -7,8 +7,8 @@
* of patent rights can be found in the PATENTS file in the same directory.
*/
-#ifndef ERROR_PUBLIC_H_MODULE
-#define ERROR_PUBLIC_H_MODULE
+#ifndef ZSTD_ERRORS_H_398273423
+#define ZSTD_ERRORS_H_398273423
#if defined (__cplusplus)
extern "C" {
@@ -29,6 +29,7 @@ typedef enum {
ZSTD_error_parameter_unknown,
ZSTD_error_frameParameter_unsupported,
ZSTD_error_frameParameter_unsupportedBy32bits,
+ ZSTD_error_frameParameter_windowTooLarge,
ZSTD_error_compressionParameter_unsupported,
ZSTD_error_init_missing,
ZSTD_error_memory_allocation,
@@ -56,4 +57,4 @@ const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
}
#endif
-#endif /* ERROR_PUBLIC_H_MODULE */
+#endif /* ZSTD_ERRORS_H_398273423 */
diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index b7d3d77a..78784aa3 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -155,13 +155,14 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si
} }
/* fill val */
- { U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0};
- U16 valPerRank[HUF_TABLELOG_MAX+1] = {0};
+ { U16 nbPerRank[HUF_TABLELOG_MAX+2] = {0}; /* support w=0=>n=tableLog+1 */
+ U16 valPerRank[HUF_TABLELOG_MAX+2] = {0};
{ U32 n; for (n=0; n0; n--) {
- valPerRank[n] = min; /* get starting value within each rank */
+ U32 n; for (n=tableLog; n>0; n--) { /* start at n=tablelog <-> w=1 */
+ valPerRank[n] = min; /* get starting value within each rank */
min += nbPerRank[n];
min >>= 1;
} }
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 94f4b5a2..e71873b0 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1458,7 +1458,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
const U32 dictLimit = zc->dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const prefixStart = base + dictLimit;
- const BYTE* match = base + matchIndex;
+ const BYTE* match;
const U32 current = (U32)(ip-base);
const U32 btLow = btMask >= current ? 0 : current - btMask;
U32* smallerPtr = bt + 2*(current&btMask);
@@ -2235,7 +2235,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
BYTE* op = ostart;
U32 const maxDist = 1 << cctx->params.cParams.windowLog;
- if (cctx->params.fParams.checksumFlag)
+ if (cctx->params.fParams.checksumFlag && srcSize)
XXH64_update(&cctx->xxhState, src, srcSize);
while (remaining) {
@@ -2688,7 +2688,9 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_pa
return NULL;
}
- memcpy(dictContent, dict, dictSize);
+ if (dictSize) {
+ memcpy(dictContent, dict, dictSize);
+ }
{ size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0);
if (ZSTD_isError(errorCode)) {
ZSTD_free(dictContent, customMem);
diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h
index cb587290..cea67056 100644
--- a/lib/compress/zstd_opt.h
+++ b/lib/compress/zstd_opt.h
@@ -401,7 +401,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
ZSTD_rescaleFreqs(seqStorePtr);
ip += (ip==prefixStart);
{ U32 i; for (i=0; irep[i]; }
- inr = ip;
+ //inr = ip;
/* Match Loop */
while (ip < ilimit) {
@@ -657,7 +657,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
ctx->nextToUpdate3 = ctx->nextToUpdate;
ZSTD_rescaleFreqs(seqStorePtr);
ip += (ip==prefixStart);
- inr = ip;
+ //inr = ip;
/* Match Loop */
while (ip < ilimit) {
@@ -666,7 +666,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
U32 current = (U32)(ip-base);
memset(opt, 0, sizeof(ZSTD_optimal_t));
last_pos = 0;
- inr = ip;
+ //inr = ip;
opt[0].litlen = (U32)(ip - anchor);
/* check repCode */
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 47b5f42c..00b069e9 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -248,7 +248,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
if (!singleSegment) {
BYTE const wlByte = ip[pos++];
U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
- if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported);
+ if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_windowTooLarge); /* avoids issue with 1 << windowLog */
windowSize = (1U << windowLog);
windowSize += (windowSize >> 3) * (wlByte&7);
}
@@ -270,7 +270,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
}
if (!windowSize) windowSize = (U32)frameContentSize;
- if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported);
+ if (windowSize > windowSizeMax) return ERROR(frameParameter_windowTooLarge);
fparamsPtr->frameContentSize = frameContentSize;
fparamsPtr->windowSize = windowSize;
fparamsPtr->dictID = dictID;
@@ -807,7 +807,8 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
if (ofCode <= 1) {
offset += (llCode==0);
if (offset) {
- size_t const temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
+ size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
+ temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */
if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1];
seqState->prevOffset[1] = seqState->prevOffset[0];
seqState->prevOffset[0] = offset = temp;
@@ -878,7 +879,12 @@ size_t ZSTD_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
+ if (op > oend_w) {
+ memmove(op, match, sequence.matchLength);
+ return sequenceLength;
+ }
} }
+ /* Requirement: op <= oend_w */
/* match within prefix */
if (sequence.offset < 8) {
@@ -1397,7 +1403,9 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_cu
return NULL;
}
- memcpy(dictContent, dict, dictSize);
+ if (dictSize) {
+ memcpy(dictContent, dict, dictSize);
+ }
{ size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize);
if (ZSTD_isError(errorCode)) {
ZSTD_free(dictContent, customMem);
@@ -1568,7 +1576,7 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
switch(paramType)
{
default : return ERROR(parameter_unknown);
- case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue; break;
+ case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); break;
}
return 0;
}
@@ -1649,7 +1657,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
} }
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
- if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_unsupported);
+ if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);
/* Adapt buffer sizes to frame header instructions */
{ size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c
index 47a82af1..b3f20b12 100644
--- a/lib/dictBuilder/zdict.c
+++ b/lib/dictBuilder/zdict.c
@@ -371,21 +371,22 @@ static dictItem ZDICT_analyzePos(
static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip)
{
const U32 tableSize = table->pos;
- const U32 max = elt.pos + (elt.length-1);
+ const U32 eltEnd = elt.pos + elt.length;
/* tail overlap */
U32 u; for (u=1; u elt.pos) && (table[u].pos < max)) { /* overlap */
+ if ((table[u].pos > elt.pos) && (table[u].pos <= eltEnd)) { /* overlap, existing > new */
/* append */
U32 addedLength = table[u].pos - elt.pos;
table[u].length += addedLength;
table[u].pos = elt.pos;
table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */
- table[u].savings += elt.length / 8; /* rough approx */
+ table[u].savings += elt.length / 8; /* rough approx bonus */
elt = table[u];
+ /* sort : improve rank */
while ((u>1) && (table[u-1].savings < elt.savings))
- table[u] = table[u-1], u--;
+ table[u] = table[u-1], u--;
table[u] = elt;
return u;
} }
@@ -393,14 +394,15 @@ static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip)
/* front overlap */
for (u=1; u elt.pos) && (table[u].pos < elt.pos)) { /* overlap */
+ if ((table[u].pos + table[u].length >= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */
/* append */
- int addedLength = (elt.pos + elt.length) - (table[u].pos + table[u].length);
- table[u].savings += elt.length / 8; /* rough approx */
- if (addedLength > 0) { /* otherwise, already included */
+ int addedLength = (int)eltEnd - (table[u].pos + table[u].length);
+ table[u].savings += elt.length / 8; /* rough approx bonus */
+ if (addedLength > 0) { /* otherwise, elt fully included into existing */
table[u].length += addedLength;
table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */
}
+ /* sort : improve rank */
elt = table[u];
while ((u>1) && (table[u-1].savings < elt.savings))
table[u] = table[u-1], u--;
@@ -811,7 +813,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
MEM_writeLE32(dstPtr+4, repStartValue[1]);
MEM_writeLE32(dstPtr+8, repStartValue[2]);
#endif
- dstPtr += 12;
+ //dstPtr += 12;
eSize += 12;
_cleanup:
diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c
index c9dcb94e..05e40aac 100644
--- a/lib/legacy/zstd_v04.c
+++ b/lib/legacy/zstd_v04.c
@@ -3107,8 +3107,13 @@ static size_t ZSTD_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
+ if (op > oend_8) {
+ memmove(op, match, sequence.matchLength);
+ return sequenceLength;
+ }
}
}
+ /* Requirement: op <= oend_8 */
/* match within prefix */
if (sequence.offset < 8)
diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c
index 5027e2b8..3cb5bb4a 100644
--- a/lib/legacy/zstd_v05.c
+++ b/lib/legacy/zstd_v05.c
@@ -2032,13 +2032,14 @@ size_t HUFv05_decompress1X2_usingDTable(
{
BYTE* op = (BYTE*)dst;
BYTE* const oend = op + dstSize;
- size_t errorCode;
const U32 dtLog = DTable[0];
const void* dtPtr = DTable;
const HUFv05_DEltX2* const dt = ((const HUFv05_DEltX2*)dtPtr)+1;
BITv05_DStream_t bitD;
- errorCode = BITv05_initDStream(&bitD, cSrc, cSrcSize);
- if (HUFv05_isError(errorCode)) return errorCode;
+
+ if (dstSize <= cSrcSize) return ERROR(dstSize_tooSmall);
+ { size_t const errorCode = BITv05_initDStream(&bitD, cSrc, cSrcSize);
+ if (HUFv05_isError(errorCode)) return errorCode; }
HUFv05_decodeStreamX2(op, &bitD, oend, dt, dtLog);
@@ -3063,7 +3064,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
FSEv05_DTable* DTableLL, FSEv05_DTable* DTableML, FSEv05_DTable* DTableOffb,
- const void* src, size_t srcSize)
+ const void* src, size_t srcSize, U32 flagStaticTable)
{
const BYTE* const istart = (const BYTE* const)src;
const BYTE* ip = istart;
@@ -3118,6 +3119,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
FSEv05_buildDTable_raw(DTableLL, LLbits);
break;
case FSEv05_ENCODING_STATIC:
+ if (!flagStaticTable) return ERROR(corruption_detected);
break;
case FSEv05_ENCODING_DYNAMIC :
default : /* impossible */
@@ -3141,6 +3143,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
FSEv05_buildDTable_raw(DTableOffb, Offbits);
break;
case FSEv05_ENCODING_STATIC:
+ if (!flagStaticTable) return ERROR(corruption_detected);
break;
case FSEv05_ENCODING_DYNAMIC :
default : /* impossible */
@@ -3164,6 +3167,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
FSEv05_buildDTable_raw(DTableML, MLbits);
break;
case FSEv05_ENCODING_STATIC:
+ if (!flagStaticTable) return ERROR(corruption_detected);
break;
case FSEv05_ENCODING_DYNAMIC :
default : /* impossible */
@@ -3312,7 +3316,12 @@ static size_t ZSTDv05_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
+ if (op > oend_8) {
+ memmove(op, match, sequence.matchLength);
+ return sequenceLength;
+ }
} }
+ /* Requirement: op <= oend_8 */
/* match within prefix */
if (sequence.offset < 8) {
@@ -3371,7 +3380,7 @@ static size_t ZSTDv05_decompressSequences(
/* Build Decoding Tables */
errorCode = ZSTDv05_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength,
DTableLL, DTableML, DTableOffb,
- ip, seqSize);
+ ip, seqSize, dctx->flagStaticTables);
if (ZSTDv05_isError(errorCode)) return errorCode;
ip += errorCode;
diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index d9e89f80..96a84d3e 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -3466,7 +3466,12 @@ size_t ZSTDv06_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
+ if (op > oend_8) {
+ memmove(op, match, sequence.matchLength);
+ return sequenceLength;
+ }
} }
+ /* Requirement: op <= oend_8 */
/* match within prefix */
if (sequence.offset < 8) {
diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c
index f4c8073f..62285238 100644
--- a/lib/legacy/zstd_v07.c
+++ b/lib/legacy/zstd_v07.c
@@ -3690,7 +3690,12 @@ size_t ZSTDv07_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
+ if (op > oend_w) {
+ memmove(op, match, sequence.matchLength);
+ return sequenceLength;
+ }
} }
+ /* Requirement: op <= oend_w */
/* match within prefix */
if (sequence.offset < 8) {
diff --git a/lib/zstd.h b/lib/zstd.h
index 2c140730..8039cc7d 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -55,7 +55,7 @@ ZSTDLIB_API unsigned ZSTD_versionNumber (void); /**< returns version number of
#define ZSTD_VERSION_MAJOR 1
#define ZSTD_VERSION_MINOR 1
-#define ZSTD_VERSION_RELEASE 0
+#define ZSTD_VERSION_RELEASE 1
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
#define ZSTD_QUOTE(str) #str
diff --git a/programs/Makefile b/programs/Makefile
index ed26f070..0ed7f3d0 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -76,17 +76,18 @@ default: zstd
all: zstd
-
+$(ZSTDDECOMP_O): CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
+$(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
$(ZSTDDECOMP_O): $(ZSTDDIR)/decompress/zstd_decompress.c
- $(CC) $(ALIGN_LOOP) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -c -o $@
-
-$(ZSTDDECOMP32_O): $(ZSTDDIR)/decompress/zstd_decompress.c
- $(CC) -m32 $(ALIGN_LOOP) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -c -o $@
zstd : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
zstdcli.c fileio.c bench.c datagen.c dibio.c
$(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(RES_FILE) -o $@$(EXT)
+
+$(ZSTDDECOMP32_O): $(ZSTDDIR)/decompress/zstd_decompress.c
+ $(CC) -m32 $(ALIGN_LOOP) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -c -o $@
+
zstd32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
zstdcli.c fileio.c bench.c datagen.c dibio.c
$(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(RES32_FILE) -o $@$(EXT)
@@ -153,11 +154,11 @@ install: zstd
@echo zstd installation completed
uninstall:
- $(RM) $(DESTDIR)$(BINDIR)/zstdcat
- $(RM) $(DESTDIR)$(BINDIR)/unzstd
- $(RM) $(DESTDIR)$(BINDIR)/zstd$(EXT)
- $(RM) $(DESTDIR)$(MANDIR)/zstdcat.1
- $(RM) $(DESTDIR)$(MANDIR)/unzstd.1
- $(RM) $(DESTDIR)$(MANDIR)/zstd.1
+ @$(RM) $(DESTDIR)$(BINDIR)/zstdcat
+ @$(RM) $(DESTDIR)$(BINDIR)/unzstd
+ @$(RM) $(DESTDIR)$(BINDIR)/zstd$(EXT)
+ @$(RM) $(DESTDIR)$(MANDIR)/zstdcat.1
+ @$(RM) $(DESTDIR)$(MANDIR)/unzstd.1
+ @$(RM) $(DESTDIR)$(MANDIR)/zstd.1
@echo zstd programs successfully uninstalled
endif
diff --git a/programs/fileio.c b/programs/fileio.c
index 56f22fe7..c4c308e0 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -8,13 +8,6 @@
*/
-/*
- Note : this file is part of zstd command line, which is not library.
- The license of ZSTD library is BSD.
- The license of this file is GPLv2.
-*/
-
-
/* *************************************
* Tuning options
***************************************/
@@ -127,6 +120,9 @@ static U32 g_checksumFlag = 1;
void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; }
static U32 g_removeSrcFile = 0;
void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); }
+static U32 g_memLimit = 0;
+void FIO_setMemLimit(unsigned memLimit) { g_memLimit = memLimit; }
+
/*-*************************************
@@ -487,6 +483,7 @@ static dRess_t FIO_createDResources(const char* dictFileName)
/* Allocation */
ress.dctx = ZSTD_createDStream();
if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZSTD_DStream");
+ ZSTD_setDStreamParameter(ress.dctx, ZSTDdsp_maxWindowSize, g_memLimit);
ress.srcBufferSize = ZSTD_DStreamInSize();
ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = ZSTD_DStreamOutSize();
diff --git a/programs/fileio.h b/programs/fileio.h
index 1e89aec2..60a7e0de 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -15,7 +15,6 @@
extern "C" {
#endif
-
/* *************************************
* Special i/o constants
**************************************/
@@ -37,6 +36,7 @@ void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdo
void FIO_setDictIDFlag(unsigned dictIDFlag);
void FIO_setChecksumFlag(unsigned checksumFlag);
void FIO_setRemoveSrcFile(unsigned flag);
+void FIO_setMemLimit(unsigned memLimit);
/*-*************************************
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index fe97f965..9aa97cdd 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -8,13 +8,6 @@
*/
-/*
- Note : this is a user program, not part of libzstd.
- The license of libzstd is BSD.
- The license of this command line program is GPLv2.
-*/
-
-
/*-************************************
* Tuning parameters
**************************************/
@@ -32,7 +25,6 @@
**************************************/
#include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */
#include /* strcmp, strlen */
-#include /* toupper */
#include /* errno */
#include "fileio.h"
#ifndef ZSTD_NOBENCH
@@ -142,6 +134,7 @@ static int usage_advanced(const char* programName)
DISPLAY( "--test : test compressed file integrity \n");
DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n");
#endif
+ DISPLAY( " -M# : Set a memory usage limit for decompression \n");
DISPLAY( "-- : All arguments after \"--\" are treated as files \n");
#ifndef ZSTD_NODICT
DISPLAY( "\n");
@@ -179,14 +172,30 @@ static void waitEnter(void)
}
/*! readU32FromChar() :
- @return : unsigned integer value reach from input in `char` format
+ @return : unsigned integer value read from input in `char` format
+ allows and interprets K, KB, KiB, M, MB and MiB suffix.
Will also modify `*stringPtr`, advancing it to position where it stopped reading.
- Note : this function can overflow if digit string > MAX_UINT */
+ Note : function result can overflow if digit string > MAX_UINT */
static unsigned readU32FromChar(const char** stringPtr)
{
unsigned result = 0;
while ((**stringPtr >='0') && (**stringPtr <='9'))
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
+ if ((**stringPtr=='K') || (**stringPtr=='M')) {
+ result <<= 10;
+ if (**stringPtr=='M') result <<= 10;
+ (*stringPtr)++ ;
+ if (**stringPtr=='i') (*stringPtr)++;
+ if (**stringPtr=='B') (*stringPtr)++;
+ }
+ return result;
+}
+
+static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
+{
+ size_t const comSize = strlen(longCommand);
+ unsigned const result = !strncmp(*stringPtr, longCommand, comSize);
+ if (result) *stringPtr += comSize;
return result;
}
@@ -213,6 +222,7 @@ int main(int argCount, const char* argv[])
int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
int cLevelLast = 1;
unsigned recursive = 0;
+ unsigned memLimit = 0;
const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */
unsigned filenameIdx = 0;
const char* programName = argv[0];
@@ -231,8 +241,8 @@ int main(int argCount, const char* argv[])
/* init */
(void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */
(void)dictCLevel; (void)dictSelect; (void)dictID; /* not used when ZSTD_NODICT set */
- (void)decode; (void)cLevel; (void)testmode;/* not used when ZSTD_NOCOMPRESS set */
- (void)ultra; /* not used when ZSTD_NODECOMPRESS set */
+ (void)decode; (void)cLevel; (void)testmode; /* not used when ZSTD_NOCOMPRESS set */
+ (void)ultra; (void)memLimit; /* not used when ZSTD_NODECOMPRESS set */
if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
filenameTable[0] = stdinmark;
displayOut = stderr;
@@ -247,16 +257,16 @@ int main(int argCount, const char* argv[])
if (!strcmp(programName, ZSTD_CAT)) { decode=1; forceStdout=1; displayLevel=1; outFileName=stdoutmark; }
/* command switches */
- for(argNb=1; argNb /* clock_t */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
#include "zstd.h" /* ZSTD_VERSION_STRING */
-#include "error_public.h" /* ZSTD_getErrorCode */
+#include "zstd_errors.h" /* ZSTD_getErrorCode */
#include "zdict.h" /* ZDICT_trainFromBuffer */
#include "datagen.h" /* RDG_genBuffer */
#include "mem.h"
diff --git a/tests/playTests.sh b/tests/playTests.sh
index d94d8fab..c5a58d62 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -81,6 +81,11 @@ $ZSTD -dc < tmp.zst > $INTOVOID # combine decompression, stdin & stdout
$ZSTD -dc - < tmp.zst > $INTOVOID
$ZSTD -d < tmp.zst > $INTOVOID # implicit stdout when stdin is used
$ZSTD -d - < tmp.zst > $INTOVOID
+$ECHO "test : impose memory limitation (must fail)"
+$ZSTD -d -f tmp.zst -M2K -c > $INTOVOID && die "decompression needs more memory than allowed"
+$ZSTD -d -f tmp.zst --memlimit=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command
+$ZSTD -d -f tmp.zst --memory=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command
+$ZSTD -d -f tmp.zst --memlimit-decompress=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command
$ECHO "test : overwrite protection"
$ZSTD -q tmp && die "overwrite check failed!"
$ECHO "test : force overwrite"