facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11

dev
Przemyslaw Skibinski 2016-10-20 11:01:52 +02:00
parent dafd66c7ac 52c1bf93fe
commit 575ab00db7
39 changed files with 280 additions and 186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
Makefile
View File

@ -22,15 +22,18 @@ endif
.PHONY: default all zlibwrapper zstd clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan
default: zstd
default: libzstd zstd
all:
$(MAKE) -C $(ZSTDDIR) $@
$(MAKE) -C $(PRGDIR) $@ zstd32
$(MAKE) -C $(TESTDIR) $@ all32
libzstd:
@$(MAKE) -C $(ZSTDDIR)
zstd:
$(MAKE) -C $(PRGDIR)
@$(MAKE) -C $(PRGDIR)
cp $(PRGDIR)/zstd .
zlibwrapper:
@ -48,18 +51,18 @@ clean:
@echo Cleaning completed
#----------------------------------------------------------------------------------
#make install is validated only for Linux, OSX, kFreeBSD, Hurd and some BSD targets
#----------------------------------------------------------------------------------
#------------------------------------------------------------------------------
# make install is validated only for Linux, OSX, Hurd and some BSD targets
#------------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD))
HOST_OS = POSIX
install:
$(MAKE) -C $(ZSTDDIR) $@
$(MAKE) -C $(PRGDIR) $@
@$(MAKE) -C $(ZSTDDIR) $@
@$(MAKE) -C $(PRGDIR) $@
uninstall:
$(MAKE) -C $(ZSTDDIR) $@
$(MAKE) -C $(PRGDIR) $@
@$(MAKE) -C $(ZSTDDIR) $@
@$(MAKE) -C $(PRGDIR) $@
travis-install:
$(MAKE) install PREFIX=~/install_test_dir

4
NEWS
View File

@ -1,3 +1,7 @@
v1.1.1
New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption
Changed : zstd_errors.h is now part of include installation
v1.1.0
New : contrib/pzstd, parallel version of zstd, by Nick Terrell
added : NetBSD install target (#338)

6
build/VS2005/fullbench/fullbench.vcproj
View File

@ -335,6 +335,10 @@
RelativePath="..\..\..\lib\common\entropy_common.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\error_private.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\fse_compress.c"
>
@ -394,7 +398,7 @@
>
</File>
<File
RelativePath="..\..\..\lib\common\error_public.h"
RelativePath="..\..\..\lib\common\zstd_errors.h"
>
</File>
<File

6
build/VS2005/fuzzer/fuzzer.vcproj
View File

@ -339,6 +339,10 @@
RelativePath="..\..\..\lib\common\entropy_common.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\error_private.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\fse_compress.c"
>
@ -398,7 +402,7 @@
>
</File>
<File
RelativePath="..\..\..\lib\common\error_public.h"
RelativePath="..\..\..\lib\common\zstd_errors.h"
>
</File>
<File

6
build/VS2005/zstd/zstd.vcproj
View File

@ -351,6 +351,10 @@
RelativePath="..\..\..\lib\common\entropy_common.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\error_private.c"
>
</File>
<File
RelativePath="..\..\..\programs\fileio.c"
>
@ -450,7 +454,7 @@
>
</File>
<File
RelativePath="..\..\..\lib\common\error_public.h"
RelativePath="..\..\..\lib\common\zstd_errors.h"
>
</File>
<File

6
build/VS2005/zstdlib/zstdlib.vcproj
View File

@ -335,6 +335,10 @@
RelativePath="..\..\..\lib\common\entropy_common.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\error_private.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\fse_compress.c"
>
@ -426,7 +430,7 @@
>
</File>
<File
RelativePath="..\..\..\lib\common\error_public.h"
RelativePath="..\..\..\lib\common\zstd_errors.h"
>
</File>
<File

6
build/VS2008/fullbench/fullbench.vcproj
View File

@ -336,6 +336,10 @@
RelativePath="..\..\..\lib\common\entropy_common.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\error_private.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\fse_compress.c"
>
@ -395,7 +399,7 @@
>
</File>
<File
RelativePath="..\..\..\lib\common\error_public.h"
RelativePath="..\..\..\lib\common\zstd_errors.h"
>
</File>
<File

6
build/VS2008/fuzzer/fuzzer.vcproj
View File

@ -340,6 +340,10 @@
RelativePath="..\..\..\lib\common\entropy_common.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\error_private.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\fse_compress.c"
>
@ -399,7 +403,7 @@
>
</File>
<File
RelativePath="..\..\..\lib\common\error_public.h"
RelativePath="..\..\..\lib\common\zstd_errors.h"
>
</File>
<File

6
build/VS2008/zstd/zstd.vcproj
View File

@ -352,6 +352,10 @@
RelativePath="..\..\..\lib\common\entropy_common.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\error_private.c"
>
</File>
<File
RelativePath="..\..\..\programs\fileio.c"
>
@ -451,7 +455,7 @@
>
</File>
<File
RelativePath="..\..\..\lib\common\error_public.h"
RelativePath="..\..\..\lib\common\zstd_errors.h"
>
</File>
<File

6
build/VS2008/zstdlib/zstdlib.vcproj
View File

@ -336,6 +336,10 @@
RelativePath="..\..\..\lib\common\entropy_common.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\error_private.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\fse_compress.c"
>
@ -427,7 +431,7 @@
>
</File>
<File
RelativePath="..\..\..\lib\common\error_public.h"
RelativePath="..\..\..\lib\common\zstd_errors.h"
>
</File>
<File

2
build/VS2010/fullbench/fullbench.vcxproj
View File

@ -158,6 +158,7 @@
<ItemGroup>
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
<ClCompile Include="..\..\..\lib\common\error_private.c" />
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
@ -175,6 +176,7 @@
<ClInclude Include="..\..\..\lib\common\huf.h" />
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
<ClInclude Include="..\..\..\lib\common\zbuff.h" />
<ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
<ClInclude Include="..\..\..\lib\zstd.h" />
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />

2
build/VS2010/fuzzer/fuzzer.vcxproj
View File

@ -157,6 +157,7 @@
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
<ClCompile Include="..\..\..\lib\common\error_private.c" />
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
@ -176,6 +177,7 @@
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
<ClInclude Include="..\..\..\lib\common\zbuff.h" />
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
<ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
<ClInclude Include="..\..\..\lib\zstd.h" />
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
<ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />

4
build/VS2010/zstd/zstd.vcxproj
View File

@ -20,6 +20,7 @@
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
<ClCompile Include="..\..\..\lib\common\error_private.c" />
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
@ -54,6 +55,7 @@
<ClInclude Include="..\..\..\lib\common\zbuff.h" />
<ClInclude Include="..\..\..\lib\zstd.h" />
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
<ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
<ClInclude Include="..\..\..\lib\legacy\zstd_v01.h" />
@ -220,4 +222,4 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>
</Project>

5
build/VS2010/zstdlib/zstdlib.vcxproj
View File

@ -20,6 +20,7 @@
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
<ClCompile Include="..\..\..\lib\common\error_private.c" />
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
@ -39,9 +40,11 @@
<ClCompile Include="..\..\..\lib\legacy\zstd_v05.c" />
<ClCompile Include="..\..\..\lib\legacy\zstd_v06.c" />
<ClCompile Include="..\..\..\lib\legacy\zstd_v07.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\..\lib\common\bitstream.h" />
<ClInclude Include="..\..\..\lib\common\error_private.h" />
<ClInclude Include="..\..\..\lib\common\error_public.h" />
<ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
<ClInclude Include="..\..\..\lib\common\mem.h" />
<ClInclude Include="..\..\..\lib\common\fse.h" />
<ClInclude Include="..\..\..\lib\common\huf.h" />

3
build/cmake/lib/CMakeLists.txt
View File

@ -59,6 +59,7 @@ MESSAGE("ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}")
SET(Sources
${LIBRARY_DIR}/common/entropy_common.c
${LIBRARY_DIR}/common/zstd_common.c
${LIBRARY_DIR}/common/error_private.c
${LIBRARY_DIR}/common/xxhash.c
${LIBRARY_DIR}/common/fse_decompress.c
${LIBRARY_DIR}/compress/fse_compress.c
@ -74,7 +75,7 @@ SET(Sources
SET(Headers
${LIBRARY_DIR}/common/bitstream.h
${LIBRARY_DIR}/common/error_private.h
${LIBRARY_DIR}/common/error_public.h
${LIBRARY_DIR}/common/zstd_errors.h
${LIBRARY_DIR}/common/fse.h
${LIBRARY_DIR}/common/huf.h
${LIBRARY_DIR}/common/mem.h

6
contrib/pzstd/test/OptionsTest.cpp
View File

@ -180,12 +180,6 @@ TEST(Options, GetOutputFile) {
EXPECT_SUCCESS(options.parse(args.size(), args.data()));
EXPECT_EQ("x.zst", options.getOutputFile(options.inputFiles[0]));
}
{
Options options;
auto args = makeArray("-o-");
EXPECT_FAILURE(options.parse(args.size(), args.data()));
EXPECT_EQ("-", options.getOutputFile(options.inputFiles[0]));
}
{
Options options;
auto args = makeArray("x", "y", "-o", nullOutput);

29
examples/Makefile
View File

@ -1,26 +1,11 @@
# ##########################################################################
# ZSTD educational examples - Makefile
# Copyright (C) Yann Collet 2016
# ################################################################
# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
# All rights reserved.
#
# GPL v2 License
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# You can contact the author at :
# - zstd homepage : http://www.zstd.net/
# ##########################################################################
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.
# ################################################################
# This Makefile presumes libzstd is installed, using `sudo make install`

20
lib/Makefile
View File

@ -97,19 +97,21 @@ install: libzstd libzstd.pc
@cp -a libzstd.pc $(DESTDIR)$(LIBDIR)/pkgconfig/
@install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a
@install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h
@install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h
@install -m 644 common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h
@install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h # Deprecated streaming functions
@install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h
@echo zstd static and shared library installed
uninstall:
$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/libzstd.pc
$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER)
$(RM) $(DESTDIR)$(LIBDIR)/libzstd.a
$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h
$(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h
$(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h
@$(RM) $(DESTDIR)$(LIBDIR)/libzstd.a
@$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
@$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
@$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER)
@$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/libzstd.pc
@$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h
@$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h
@$(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h # Deprecated streaming functions
@$(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h
@echo zstd libraries successfully uninstalled
endif

43
lib/common/error_private.c Normal file
View File

@ -0,0 +1,43 @@
/**
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
/* The purpose of this file is to have a single list of error strings embedded in binary */
#include "error_private.h"
const char* ERR_getErrorString(ERR_enum code)
{
static const char* const notErrorCode = "Unspecified error code";
switch( code )
{
case PREFIX(no_error): return "No error detected";
case PREFIX(GENERIC): return "Error (generic)";
case PREFIX(prefix_unknown): return "Unknown frame descriptor";
case PREFIX(version_unsupported): return "Version not supported";
case PREFIX(parameter_unknown): return "Unknown parameter type";
case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
case PREFIX(init_missing): return "Context should be init first";
case PREFIX(memory_allocation): return "Allocation error : not enough memory";
case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
case PREFIX(srcSize_wrong): return "Src size incorrect";
case PREFIX(corruption_detected): return "Corrupted block detected";
case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
case PREFIX(dictionary_wrong): return "Dictionary mismatch";
case PREFIX(maxCode):
default: return notErrorCode;
}
}

32
lib/common/error_private.h
View File

@ -21,7 +21,7 @@ extern "C" {
* Dependencies
******************************************/
#include <stddef.h> /* size_t */
#include "error_public.h" /* enum list */
#include "zstd_errors.h" /* enum list */
/* ****************************************
@ -62,35 +62,7 @@ ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) retu
* Error Strings
******************************************/
ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
{
static const char* notErrorCode = "Unspecified error code";
switch( code )
{
case PREFIX(no_error): return "No error detected";
case PREFIX(GENERIC): return "Error (generic)";
case PREFIX(prefix_unknown): return "Unknown frame descriptor";
case PREFIX(version_unsupported): return "Version not supported";
case PREFIX(parameter_unknown): return "Unknown parameter type";
case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
case PREFIX(init_missing): return "Context should be init first";
case PREFIX(memory_allocation): return "Allocation error : not enough memory";
case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
case PREFIX(srcSize_wrong): return "Src size incorrect";
case PREFIX(corruption_detected): return "Corrupted block detected";
case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
case PREFIX(dictionary_wrong): return "Dictionary mismatch";
case PREFIX(maxCode):
default: return notErrorCode;
}
}
const char* ERR_getErrorString(ERR_enum code); /* error_private.c */
ERR_STATIC const char* ERR_getErrorName(size_t code)
{

14
lib/common/fse.h
View File

@ -503,6 +503,7 @@ MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePt
BIT_flushBits(bitC);
}
/* ====== Decompression ====== */
typedef struct {
@ -581,14 +582,19 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
* Increasing memory usage improves compression ratio
* Reduced memory usage can improve speed, due to cache effect
* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
#define FSE_MAX_MEMORY_USAGE 14
#define FSE_DEFAULT_MEMORY_USAGE 13
#ifndef FSE_MAX_MEMORY_USAGE
# define FSE_MAX_MEMORY_USAGE 14
#endif
#ifndef FSE_DEFAULT_MEMORY_USAGE
# define FSE_DEFAULT_MEMORY_USAGE 13
#endif
/*!FSE_MAX_SYMBOL_VALUE :
* Maximum symbol value authorized.
* Required for proper stack allocation */
#define FSE_MAX_SYMBOL_VALUE 255
#ifndef FSE_MAX_SYMBOL_VALUE
# define FSE_MAX_SYMBOL_VALUE 255
#endif
/* **************************************************************
* template functions type & suffix

7
lib/common/error_public.h → lib/common/zstd_errors.h
View File

@ -7,8 +7,8 @@
* of patent rights can be found in the PATENTS file in the same directory.
*/
#ifndef ERROR_PUBLIC_H_MODULE
#define ERROR_PUBLIC_H_MODULE
#ifndef ZSTD_ERRORS_H_398273423
#define ZSTD_ERRORS_H_398273423
#if defined (__cplusplus)
extern "C" {
@ -29,6 +29,7 @@ typedef enum {
ZSTD_error_parameter_unknown,
ZSTD_error_frameParameter_unsupported,
ZSTD_error_frameParameter_unsupportedBy32bits,
ZSTD_error_frameParameter_windowTooLarge,
ZSTD_error_compressionParameter_unsupported,
ZSTD_error_init_missing,
ZSTD_error_memory_allocation,
@ -56,4 +57,4 @@ const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
}
#endif
#endif /* ERROR_PUBLIC_H_MODULE */
#endif /* ZSTD_ERRORS_H_398273423 */

9
lib/compress/huf_compress.c
View File

@ -155,13 +155,14 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si
} }
/* fill val */
{ U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0};
U16 valPerRank[HUF_TABLELOG_MAX+1] = {0};
{ U16 nbPerRank[HUF_TABLELOG_MAX+2] = {0}; /* support w=0=>n=tableLog+1 */
U16 valPerRank[HUF_TABLELOG_MAX+2] = {0};
{ U32 n; for (n=0; n<nbSymbols; n++) nbPerRank[CTable[n].nbBits]++; }
/* determine stating value per rank */
valPerRank[tableLog+1] = 0; /* for w==0 */
{ U16 min = 0;
U32 n; for (n=HUF_TABLELOG_MAX; n>0; n--) {
valPerRank[n] = min; /* get starting value within each rank */
U32 n; for (n=tableLog; n>0; n--) { /* start at n=tablelog <-> w=1 */
valPerRank[n] = min; /* get starting value within each rank */
min += nbPerRank[n];
min >>= 1;
} }

8
lib/compress/zstd_compress.c
View File

@ -1458,7 +1458,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
const U32 dictLimit = zc->dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const prefixStart = base + dictLimit;
const BYTE* match = base + matchIndex;
const BYTE* match;
const U32 current = (U32)(ip-base);
const U32 btLow = btMask >= current ? 0 : current - btMask;
U32* smallerPtr = bt + 2*(current&btMask);
@ -2235,7 +2235,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
BYTE* op = ostart;
U32 const maxDist = 1 << cctx->params.cParams.windowLog;
if (cctx->params.fParams.checksumFlag)
if (cctx->params.fParams.checksumFlag && srcSize)
XXH64_update(&cctx->xxhState, src, srcSize);
while (remaining) {
@ -2688,7 +2688,9 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_pa
return NULL;
}
memcpy(dictContent, dict, dictSize);
if (dictSize) {
memcpy(dictContent, dict, dictSize);
}
{ size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0);
if (ZSTD_isError(errorCode)) {
ZSTD_free(dictContent, customMem);

6
lib/compress/zstd_opt.h
View File

@ -401,7 +401,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
ZSTD_rescaleFreqs(seqStorePtr);
ip += (ip==prefixStart);
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=ctx->rep[i]; }
inr = ip;
//inr = ip;
/* Match Loop */
while (ip < ilimit) {
@ -657,7 +657,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
ctx->nextToUpdate3 = ctx->nextToUpdate;
ZSTD_rescaleFreqs(seqStorePtr);
ip += (ip==prefixStart);
inr = ip;
//inr = ip;
/* Match Loop */
while (ip < ilimit) {
@ -666,7 +666,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
U32 current = (U32)(ip-base);
memset(opt, 0, sizeof(ZSTD_optimal_t));
last_pos = 0;
inr = ip;
//inr = ip;
opt[0].litlen = (U32)(ip - anchor);
/* check repCode */

20
lib/decompress/zstd_decompress.c
View File

@ -248,7 +248,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
if (!singleSegment) {
BYTE const wlByte = ip[pos++];
U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported);
if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_windowTooLarge); /* avoids issue with 1 << windowLog */
windowSize = (1U << windowLog);
windowSize += (windowSize >> 3) * (wlByte&7);
}
@ -270,7 +270,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
}
if (!windowSize) windowSize = (U32)frameContentSize;
if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported);
if (windowSize > windowSizeMax) return ERROR(frameParameter_windowTooLarge);
fparamsPtr->frameContentSize = frameContentSize;
fparamsPtr->windowSize = windowSize;
fparamsPtr->dictID = dictID;
@ -807,7 +807,8 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
if (ofCode <= 1) {
offset += (llCode==0);
if (offset) {
size_t const temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */
if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1];
seqState->prevOffset[1] = seqState->prevOffset[0];
seqState->prevOffset[0] = offset = temp;
@ -878,7 +879,12 @@ size_t ZSTD_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
if (op > oend_w) {
memmove(op, match, sequence.matchLength);
return sequenceLength;
}
} }
/* Requirement: op <= oend_w */
/* match within prefix */
if (sequence.offset < 8) {
@ -1397,7 +1403,9 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_cu
return NULL;
}
memcpy(dictContent, dict, dictSize);
if (dictSize) {
memcpy(dictContent, dict, dictSize);
}
{ size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize);
if (ZSTD_isError(errorCode)) {
ZSTD_free(dictContent, customMem);
@ -1568,7 +1576,7 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
switch(paramType)
{
default : return ERROR(parameter_unknown);
case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue; break;
case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); break;
}
return 0;
}
@ -1649,7 +1657,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
} }
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_unsupported);
if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);
/* Adapt buffer sizes to frame header instructions */
{ size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);

20
lib/dictBuilder/zdict.c
View File

@ -371,21 +371,22 @@ static dictItem ZDICT_analyzePos(
static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip)
{
const U32 tableSize = table->pos;
const U32 max = elt.pos + (elt.length-1);
const U32 eltEnd = elt.pos + elt.length;
/* tail overlap */
U32 u; for (u=1; u<tableSize; u++) {
if (u==eltNbToSkip) continue;
if ((table[u].pos > elt.pos) && (table[u].pos < max)) { /* overlap */
if ((table[u].pos > elt.pos) && (table[u].pos <= eltEnd)) { /* overlap, existing > new */
/* append */
U32 addedLength = table[u].pos - elt.pos;
table[u].length += addedLength;
table[u].pos = elt.pos;
table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */
table[u].savings += elt.length / 8; /* rough approx */
table[u].savings += elt.length / 8; /* rough approx bonus */
elt = table[u];
/* sort : improve rank */
while ((u>1) && (table[u-1].savings < elt.savings))
table[u] = table[u-1], u--;
table[u] = table[u-1], u--;
table[u] = elt;
return u;
} }
@ -393,14 +394,15 @@ static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip)
/* front overlap */
for (u=1; u<tableSize; u++) {
if (u==eltNbToSkip) continue;
if ((table[u].pos + table[u].length > elt.pos) && (table[u].pos < elt.pos)) { /* overlap */
if ((table[u].pos + table[u].length >= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */
/* append */
int addedLength = (elt.pos + elt.length) - (table[u].pos + table[u].length);
table[u].savings += elt.length / 8; /* rough approx */
if (addedLength > 0) { /* otherwise, already included */
int addedLength = (int)eltEnd - (table[u].pos + table[u].length);
table[u].savings += elt.length / 8; /* rough approx bonus */
if (addedLength > 0) { /* otherwise, elt fully included into existing */
table[u].length += addedLength;
table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */
}
/* sort : improve rank */
elt = table[u];
while ((u>1) && (table[u-1].savings < elt.savings))
table[u] = table[u-1], u--;
@ -811,7 +813,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
MEM_writeLE32(dstPtr+4, repStartValue[1]);
MEM_writeLE32(dstPtr+8, repStartValue[2]);
#endif
dstPtr += 12;
//dstPtr += 12;
eSize += 12;
_cleanup:

5
lib/legacy/zstd_v04.c
View File

@ -3107,8 +3107,13 @@ static size_t ZSTD_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
if (op > oend_8) {
memmove(op, match, sequence.matchLength);
return sequenceLength;
}
}
}
/* Requirement: op <= oend_8 */
/* match within prefix */
if (sequence.offset < 8)

19
lib/legacy/zstd_v05.c
View File

@ -2032,13 +2032,14 @@ size_t HUFv05_decompress1X2_usingDTable(
{
BYTE* op = (BYTE*)dst;
BYTE* const oend = op + dstSize;
size_t errorCode;
const U32 dtLog = DTable[0];
const void* dtPtr = DTable;
const HUFv05_DEltX2* const dt = ((const HUFv05_DEltX2*)dtPtr)+1;
BITv05_DStream_t bitD;
errorCode = BITv05_initDStream(&bitD, cSrc, cSrcSize);
if (HUFv05_isError(errorCode)) return errorCode;
if (dstSize <= cSrcSize) return ERROR(dstSize_tooSmall);
{ size_t const errorCode = BITv05_initDStream(&bitD, cSrc, cSrcSize);
if (HUFv05_isError(errorCode)) return errorCode; }
HUFv05_decodeStreamX2(op, &bitD, oend, dt, dtLog);
@ -3063,7 +3064,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
FSEv05_DTable* DTableLL, FSEv05_DTable* DTableML, FSEv05_DTable* DTableOffb,
const void* src, size_t srcSize)
const void* src, size_t srcSize, U32 flagStaticTable)
{
const BYTE* const istart = (const BYTE* const)src;
const BYTE* ip = istart;
@ -3118,6 +3119,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
FSEv05_buildDTable_raw(DTableLL, LLbits);
break;
case FSEv05_ENCODING_STATIC:
if (!flagStaticTable) return ERROR(corruption_detected);
break;
case FSEv05_ENCODING_DYNAMIC :
default : /* impossible */
@ -3141,6 +3143,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
FSEv05_buildDTable_raw(DTableOffb, Offbits);
break;
case FSEv05_ENCODING_STATIC:
if (!flagStaticTable) return ERROR(corruption_detected);
break;
case FSEv05_ENCODING_DYNAMIC :
default : /* impossible */
@ -3164,6 +3167,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
FSEv05_buildDTable_raw(DTableML, MLbits);
break;
case FSEv05_ENCODING_STATIC:
if (!flagStaticTable) return ERROR(corruption_detected);
break;
case FSEv05_ENCODING_DYNAMIC :
default : /* impossible */
@ -3312,7 +3316,12 @@ static size_t ZSTDv05_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
if (op > oend_8) {
memmove(op, match, sequence.matchLength);
return sequenceLength;
}
} }
/* Requirement: op <= oend_8 */
/* match within prefix */
if (sequence.offset < 8) {
@ -3371,7 +3380,7 @@ static size_t ZSTDv05_decompressSequences(
/* Build Decoding Tables */
errorCode = ZSTDv05_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength,
DTableLL, DTableML, DTableOffb,
ip, seqSize);
ip, seqSize, dctx->flagStaticTables);
if (ZSTDv05_isError(errorCode)) return errorCode;
ip += errorCode;

5
lib/legacy/zstd_v06.c
View File

@ -3466,7 +3466,12 @@ size_t ZSTDv06_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
if (op > oend_8) {
memmove(op, match, sequence.matchLength);
return sequenceLength;
}
} }
/* Requirement: op <= oend_8 */
/* match within prefix */
if (sequence.offset < 8) {

5
lib/legacy/zstd_v07.c
View File

@ -3690,7 +3690,12 @@ size_t ZSTDv07_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
if (op > oend_w) {
memmove(op, match, sequence.matchLength);
return sequenceLength;
}
} }
/* Requirement: op <= oend_w */
/* match within prefix */
if (sequence.offset < 8) {

2
lib/zstd.h
View File

@ -55,7 +55,7 @@ ZSTDLIB_API unsigned ZSTD_versionNumber (void); /**< returns version number of
#define ZSTD_VERSION_MAJOR 1
#define ZSTD_VERSION_MINOR 1
#define ZSTD_VERSION_RELEASE 0
#define ZSTD_VERSION_RELEASE 1
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
#define ZSTD_QUOTE(str) #str

23
programs/Makefile
View File

@ -76,17 +76,18 @@ default: zstd
all: zstd
$(ZSTDDECOMP_O): CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
$(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
$(ZSTDDECOMP_O): $(ZSTDDIR)/decompress/zstd_decompress.c
$(CC) $(ALIGN_LOOP) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -c -o $@
$(ZSTDDECOMP32_O): $(ZSTDDIR)/decompress/zstd_decompress.c
$(CC) -m32 $(ALIGN_LOOP) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -c -o $@
zstd : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
zstdcli.c fileio.c bench.c datagen.c dibio.c
$(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(RES_FILE) -o $@$(EXT)
$(ZSTDDECOMP32_O): $(ZSTDDIR)/decompress/zstd_decompress.c
$(CC) -m32 $(ALIGN_LOOP) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -c -o $@
zstd32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
zstdcli.c fileio.c bench.c datagen.c dibio.c
$(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(RES32_FILE) -o $@$(EXT)
@ -153,11 +154,11 @@ install: zstd
@echo zstd installation completed
uninstall:
$(RM) $(DESTDIR)$(BINDIR)/zstdcat
$(RM) $(DESTDIR)$(BINDIR)/unzstd
$(RM) $(DESTDIR)$(BINDIR)/zstd$(EXT)
$(RM) $(DESTDIR)$(MANDIR)/zstdcat.1
$(RM) $(DESTDIR)$(MANDIR)/unzstd.1
$(RM) $(DESTDIR)$(MANDIR)/zstd.1
@$(RM) $(DESTDIR)$(BINDIR)/zstdcat
@$(RM) $(DESTDIR)$(BINDIR)/unzstd
@$(RM) $(DESTDIR)$(BINDIR)/zstd$(EXT)
@$(RM) $(DESTDIR)$(MANDIR)/zstdcat.1
@$(RM) $(DESTDIR)$(MANDIR)/unzstd.1
@$(RM) $(DESTDIR)$(MANDIR)/zstd.1
@echo zstd programs successfully uninstalled
endif

11
programs/fileio.c
View File

@ -8,13 +8,6 @@
*/
/*
Note : this file is part of zstd command line, which is not library.
The license of ZSTD library is BSD.
The license of this file is GPLv2.
*/
/* *************************************
* Tuning options
***************************************/
@ -127,6 +120,9 @@ static U32 g_checksumFlag = 1;
void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; }
static U32 g_removeSrcFile = 0;
void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); }
static U32 g_memLimit = 0;
void FIO_setMemLimit(unsigned memLimit) { g_memLimit = memLimit; }
/*-*************************************
@ -487,6 +483,7 @@ static dRess_t FIO_createDResources(const char* dictFileName)
/* Allocation */
ress.dctx = ZSTD_createDStream();
if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZSTD_DStream");
ZSTD_setDStreamParameter(ress.dctx, ZSTDdsp_maxWindowSize, g_memLimit);
ress.srcBufferSize = ZSTD_DStreamInSize();
ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = ZSTD_DStreamOutSize();

2
programs/fileio.h
View File

@ -15,7 +15,6 @@
extern "C" {
#endif
/* *************************************
* Special i/o constants
**************************************/
@ -37,6 +36,7 @@ void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdo
void FIO_setDictIDFlag(unsigned dictIDFlag);
void FIO_setChecksumFlag(unsigned checksumFlag);
void FIO_setRemoveSrcFile(unsigned flag);
void FIO_setMemLimit(unsigned memLimit);
/*-*************************************

59
programs/zstdcli.c
View File

@ -8,13 +8,6 @@
*/
/*
Note : this is a user program, not part of libzstd.
The license of libzstd is BSD.
The license of this command line program is GPLv2.
*/
/*-************************************
* Tuning parameters
**************************************/
@ -32,7 +25,6 @@
**************************************/
#include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */
#include <string.h> /* strcmp, strlen */
#include <ctype.h> /* toupper */
#include <errno.h> /* errno */
#include "fileio.h"
#ifndef ZSTD_NOBENCH
@ -142,6 +134,7 @@ static int usage_advanced(const char* programName)
DISPLAY( "--test : test compressed file integrity \n");
DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n");
#endif
DISPLAY( " -M# : Set a memory usage limit for decompression \n");
DISPLAY( "-- : All arguments after \"--\" are treated as files \n");
#ifndef ZSTD_NODICT
DISPLAY( "\n");
@ -179,14 +172,30 @@ static void waitEnter(void)
}
/*! readU32FromChar() :
@return : unsigned integer value reach from input in `char` format
@return : unsigned integer value read from input in `char` format
allows and interprets K, KB, KiB, M, MB and MiB suffix.
Will also modify `*stringPtr`, advancing it to position where it stopped reading.
Note : this function can overflow if digit string > MAX_UINT */
Note : function result can overflow if digit string > MAX_UINT */
static unsigned readU32FromChar(const char** stringPtr)
{
unsigned result = 0;
while ((**stringPtr >='0') && (**stringPtr <='9'))
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
if ((**stringPtr=='K') || (**stringPtr=='M')) {
result <<= 10;
if (**stringPtr=='M') result <<= 10;
(*stringPtr)++ ;
if (**stringPtr=='i') (*stringPtr)++;
if (**stringPtr=='B') (*stringPtr)++;
}
return result;
}
static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
{
size_t const comSize = strlen(longCommand);
unsigned const result = !strncmp(*stringPtr, longCommand, comSize);
if (result) *stringPtr += comSize;
return result;
}
@ -213,6 +222,7 @@ int main(int argCount, const char* argv[])
int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
int cLevelLast = 1;
unsigned recursive = 0;
unsigned memLimit = 0;
const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */
unsigned filenameIdx = 0;
const char* programName = argv[0];
@ -231,8 +241,8 @@ int main(int argCount, const char* argv[])
/* init */
(void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */
(void)dictCLevel; (void)dictSelect; (void)dictID; /* not used when ZSTD_NODICT set */
(void)decode; (void)cLevel; (void)testmode;/* not used when ZSTD_NOCOMPRESS set */
(void)ultra; /* not used when ZSTD_NODECOMPRESS set */
(void)decode; (void)cLevel; (void)testmode; /* not used when ZSTD_NOCOMPRESS set */
(void)ultra; (void)memLimit; /* not used when ZSTD_NODECOMPRESS set */
if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
filenameTable[0] = stdinmark;
displayOut = stderr;
@ -247,16 +257,16 @@ int main(int argCount, const char* argv[])
if (!strcmp(programName, ZSTD_CAT)) { decode=1; forceStdout=1; displayLevel=1; outFileName=stdoutmark; }
/* command switches */
for(argNb=1; argNb<argCount; argNb++) {
for (argNb=1; argNb<argCount; argNb++) {
const char* argument = argv[argNb];
if(!argument) continue; /* Protection if argument empty */
if (nextArgumentIsFile==0) {
/* long commands (--long-word) */
if (!strcmp(argument, "--")) { nextArgumentIsFile=1; continue; }
if (!strcmp(argument, "--")) { nextArgumentIsFile=1; continue; } /* only file names allowed from now on */
if (!strcmp(argument, "--decompress")) { decode=1; continue; }
if (!strcmp(argument, "--force")) { FIO_overwriteMode(); continue; }
if (!strcmp(argument, "--force")) { FIO_overwriteMode(); continue; }
if (!strcmp(argument, "--version")) { displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); }
if (!strcmp(argument, "--help")) { displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); }
if (!strcmp(argument, "--verbose")) { displayLevel++; continue; }
@ -275,6 +285,11 @@ int main(int argCount, const char* argv[])
if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
/* long commands with arguments */
if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
/* '-' means stdin/stdout */
if (!strcmp(argument, "-")){
if (!filenameIdx) {
@ -338,6 +353,12 @@ int main(int argCount, const char* argv[])
/* destination file name */
case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break;
/* limit decompression memory */
case 'M':
argument++;
memLimit = readU32FromChar(&argument);
break;
#ifdef UTIL_HAS_CREATEFILELIST
/* recursive */
case 'r': recursive=1; argument++; break;
@ -366,10 +387,7 @@ int main(int argCount, const char* argv[])
/* cut input into blocks (benchmark only) */
case 'B':
argument++;
{ size_t bSize = readU32FromChar(&argument);
if (toupper(*argument)=='K') bSize<<=10, argument++; /* allows using KB notation */
if (toupper(*argument)=='M') bSize<<=20, argument++;
if (toupper(*argument)=='B') argument++;
{ size_t const bSize = readU32FromChar(&argument);
BMK_setNotificationLevel(displayLevel);
BMK_SetBlockSize(bSize);
}
@ -402,8 +420,6 @@ int main(int argCount, const char* argv[])
nextArgumentIsMaxDict = 0;
lastCommand = 0;
maxDictSize = readU32FromChar(&argument);
if (toupper(*argument)=='K') maxDictSize <<= 10;
if (toupper(*argument)=='M') maxDictSize <<= 20;
continue;
}
@ -518,6 +534,7 @@ int main(int argCount, const char* argv[])
} else { /* decompression */
#ifndef ZSTD_NODECOMPRESS
if (testmode) { outFileName=nulmark; FIO_setRemoveSrcFile(0); } /* test mode */
FIO_setMemLimit(memLimit);
if (filenameIdx==1 && outFileName)
operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
else

25
tests/Makefile
View File

@ -1,25 +1,10 @@
# ##########################################################################
# ZSTD tests - Makefile
# Copyright (C) Yann Collet 2015-2016
# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
# All rights reserved.
#
# GPL v2 License
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# You can contact the author at :
# - zstd homepage : http://www.zstd.net/
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.
# ##########################################################################
# datagen : Synthetic and parametrable data generator, for tests
# fullbench : Precisely measure speed for each zstd inner functions

2
tests/fuzzer.c
View File

@ -27,7 +27,7 @@
#include <time.h> /* clock_t */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
#include "zstd.h" /* ZSTD_VERSION_STRING */
#include "error_public.h" /* ZSTD_getErrorCode */
#include "zstd_errors.h" /* ZSTD_getErrorCode */
#include "zdict.h" /* ZDICT_trainFromBuffer */
#include "datagen.h" /* RDG_genBuffer */
#include "mem.h"

5
tests/playTests.sh
View File

@ -81,6 +81,11 @@ $ZSTD -dc < tmp.zst > $INTOVOID # combine decompression, stdin & stdout
$ZSTD -dc - < tmp.zst > $INTOVOID
$ZSTD -d < tmp.zst > $INTOVOID # implicit stdout when stdin is used
$ZSTD -d - < tmp.zst > $INTOVOID
$ECHO "test : impose memory limitation (must fail)"
$ZSTD -d -f tmp.zst -M2K -c > $INTOVOID && die "decompression needs more memory than allowed"
$ZSTD -d -f tmp.zst --memlimit=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command
$ZSTD -d -f tmp.zst --memory=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command
$ZSTD -d -f tmp.zst --memlimit-decompress=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command
$ECHO "test : overwrite protection"
$ZSTD -q tmp && die "overwrite check failed!"
$ECHO "test : force overwrite"